diff --git a/CMakeLists.txt b/CMakeLists.txt index 6e16bf8c..4b68b793 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,19 +32,21 @@ endif () # if (CMAKE_INSTALL_LIBDIR) message(STATUS "CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_LIBDIR}") - set(DYAD_LIBDIR ${CMAKE_INSTALL_LIBDIR}) - set(DYAD_INSTALL_INCLUDE_DIR + set(DYAD_INSTALL_BINDIR ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}) + set(DYAD_INSTALL_LIBDIR ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}) + set(DYAD_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}) set(DYAD_INSTALL_DOCDIR ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_DOCDIR}) set(DYAD_INSTALL_SYSCONFDIR ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_SYSCONFDIR}/modulefiles) else () - set(DYAD_LIBDIR "lib") - set(DYAD_INSTALL_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/include") + set(DYAD_INSTALL_BINDIR "${CMAKE_INSTALL_PREFIX}/bin") + set(DYAD_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib") + set(DYAD_INSTALL_INCLUDEDIR "${CMAKE_INSTALL_PREFIX}/include") set(DYAD_INSTALL_DOCDIR "${CMAKE_INSTALL_PREFIX}/doc") set(DYAD_INSTALL_SYSCONFDIR "${CMAKE_INSTALL_PREFIX}/etc/modulefiles") - message(STATUS "DYAD_LIBDIR set to ${DYAD_LIBDIR}") + message(STATUS "DYAD_INSTALL_LIBDIR set to ${DYAD_INSTALL_LIBDIR}") endif () #------------------------------------------------------------------------------ @@ -54,17 +56,19 @@ endif () set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${CMAKE_BINARY_DIR} ${CMAKE_INSTALL_PREFIX}) # This sets where to look for dependent library's cmake files list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules) -list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/cmake) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/lib/cmake) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/lib64/cmake) list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/share/cmake) #------------------------------------------------------------------------------ -if (NOT DYAD_EXTERNALLY_CONFIGURED) - set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin CACHE PATH "Single Directory for all Executables.") - set(CMAKE_INCLUDE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/include CACHE PATH "Store the headers.") - set(EXECUTABLE_OUTPUT_PATH ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${DYAD_LIBDIR} CACHE PATH "Single Directory for all Libraries") - set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${DYAD_LIBDIR} CACHE PATH "Single Directory for all static libraries.") -endif () +# This doesn't really need to be in an 'if' statement since they're cache variables +# if (NOT DYAD_EXTERNALLY_CONFIGURED) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin CACHE PATH "Single Directory for all Executables.") +set(CMAKE_INCLUDE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/include CACHE PATH "Store the headers.") +set(EXECUTABLE_OUTPUT_PATH ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib CACHE PATH "Single Directory for all Libraries") +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib CACHE PATH "Single Directory for all static libraries.") +# endif () #----------------------------------------------------------------------------- # Targets built within this project are exported at Install time for use @@ -285,123 +289,65 @@ include_directories(${CMAKE_SOURCE_DIR}/include) # public header add_subdirectory(src/dyad) #cmake_policy(SET CMP0079 NEW) # In case that we need more control over the target building order - -#----------------------------------------------------------------------------- -# Configure the config.cmake file for the build directory -#----------------------------------------------------------------------------- -configure_file( - ${CMAKE_CURRENT_SOURCE_DIR}/CMake/${PROJECT_NAME}-config.cmake.build.in - ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/cmake/${PROJECT_NAME}/${PROJECT_NAME}-config.cmake @ONLY -) - -configure_file( - ${CMAKE_CURRENT_SOURCE_DIR}/CMake/${PROJECT_NAME}-config.cmake.install.in - ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/cmake/${PROJECT_NAME}/install/${PROJECT_NAME}-config.cmake @ONLY -) -install( - FILES - ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/cmake/${PROJECT_NAME}/install/${PROJECT_NAME}-config.cmake - DESTINATION - ${DYAD_LIBDIR}/cmake/${PROJECT_NAME} -) -#----------------------------------------------------------------------------- -# Configure the ${PROJECT_NAME}-config-version .cmake file for the install directory -#----------------------------------------------------------------------------- -configure_file( - ${CMAKE_CURRENT_SOURCE_DIR}/CMake/${PROJECT_NAME}-config-version.cmake.in - ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/cmake/${PROJECT_NAME}/${PROJECT_NAME}-config-version.cmake @ONLY -) - -install( - FILES - ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/cmake/${PROJECT_NAME}/${PROJECT_NAME}-config-version.cmake - DESTINATION - ${DYAD_LIBDIR}/cmake/${PROJECT_NAME} -) - -export(EXPORT ${DYAD_EXPORTED_TARGETS} - FILE "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/cmake/${PROJECT_NAME}/${PROJECT_NAME}Targets.cmake") - # Write the configure file configure_file("${CMAKE_SOURCE_DIR}/cmake/configure_files/dyad_config.hpp.in" "${CMAKE_INCLUDE_OUTPUT_DIRECTORY}/dyad/dyad_config.hpp" @ONLY) - ################################################################ # Install DYAD ################################################################ include(CMakePackageConfigHelpers) +set(DYAD_BUILD_CMAKE_DIR "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/cmake/${PROJECT_NAME}") +set(DYAD_INSTALL_CMAKE_DIR "${DYAD_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") +set(DYAD_INSTALL_TMP_CMAKE_DIR "${CMAKE_BINARY_DIR}/install_cmake_scripts") + # Write the version file. This is independent of build/install tree. write_basic_package_version_file( - DYADConfigVersion.cmake + ${DYAD_BUILD_CMAKE_DIR}/DYADConfigVersion.cmake VERSION "${DYAD_PACKAGE_VERSION}" COMPATIBILITY SameMajorVersion) +file(COPY ${DYAD_BUILD_CMAKE_DIR}/DYADConfigVersion.cmake + DESTINATION ${DYAD_INSTALL_TMP_CMAKE_DIR}) -#........................................... -# TODO: There are two different places where cmake config is handled currently. -# Need to clean up and merge - -if (WIP_package_config) # This is for the build tree -#set(INCLUDE_INSTALL_DIRS -# "${CMAKE_SOURCE_DIR}/src/dyad/common" -# "${CMAKE_SOURCE_DIR}/src/dyad/core" -# "${CMAKE_SOURCE_DIR}/src/dyad/dtl" -# "${CMAKE_SOURCE_DIR}/src/dyad/module" -# "${CMAKE_SOURCE_DIR}/src/dyad/wrapper" -# "${CMAKE_SOURCE_DIR}/src/dyad/utils" -# "${CMAKE_SOURCE_DIR}/src/dyad/utils/base64") -# "${CMAKE_SOURCE_DIR}/include/dyad/stream") set(EXTRA_CMAKE_MODULE_DIR "${CMAKE_SOURCE_DIR}/cmake/modules") - -configure_package_config_file(cmake/configure_files/DYADConfig.cmake.in - "${CMAKE_BINARY_DIR}/DYADConfig.cmake" - INSTALL_DESTINATION "${CMAKE_BINARY_DIR}/src/dyad/${DYAD_LIBDIR}/cmake/dyad" - PATH_VARS CMAKE_BINARY_DIR) - #PATH_VARS INCLUDE_INSTALL_DIRS CMAKE_BINARY_DIR) +set(INCLUDE_INSTALL_DIRS ${CMAKE_INCLUDE_OUTPUT_DIRECTORY}) +set(LIB_INSTALL_DIR ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}) +configure_package_config_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/configure_files/DYADConfig.cmake.in + "${DYAD_BUILD_CMAKE_DIR}/DYADConfig.cmake" + INSTALL_DESTINATION "${DYAD_BUILD_CMAKE_DIR}" + PATH_VARS INCLUDE_INSTALL_DIRS LIB_INSTALL_DIR) # Write the configure file for the install tree -set(INCLUDE_INSTALL_DIRS include) -set(EXTRA_CMAKE_MODULE_DIR) +set(EXTRA_CMAKE_MODULE_DIR "") +set(INCLUDE_INSTALL_DIRS "${DYAD_INSTALL_INCLUDEDIR}") +set(LIB_INSTALL_DIR "${DYAD_INSTALL_LIBDIR}") configure_package_config_file(cmake/configure_files/DYADConfig.cmake.in - "${CMAKE_BINARY_DIR}/DYADConfig.cmake.install" - INSTALL_DESTINATION "${DYAD_LIBDIR}/cmake/dyad" - PATH_VARS INCLUDE_INSTALL_DIRS DYAD_LIBDIR) + "${DYAD_INSTALL_TMP_CMAKE_DIR}/DYADConfig.cmake" + INSTALL_DESTINATION "${DYAD_INSTALL_CMAKE_DIR}" + PATH_VARS INCLUDE_INSTALL_DIRS LIB_INSTALL_DIR) # Install export -install(EXPORT DYADTargets +install(EXPORT ${DYAD_EXPORTED_TARGETS} # DYADTargets NAMESPACE DYAD:: FILE DYADTargets.cmake - DESTINATION "${DYAD_LIBDIR}/cmake/dyad") + DESTINATION "${DYAD_INSTALL_CMAKE_DIR}") # Install the cmake stuff -install(FILES "${PROJECT_BINARY_DIR}/DYADConfig.cmake.install" - DESTINATION "${DYAD_LIBDIR}/cmake/dyad" - RENAME "DYADConfig.cmake") - -install(FILES "${PROJECT_BINARY_DIR}/DYADConfigVersion.cmake" - DESTINATION "${DYAD_LIBDIR}/cmake/dyad") +install(DIRECTORY "${DYAD_INSTALL_TMP_CMAKE_DIR}/" # Trailing slash ensures directory structure is not copied + DESTINATION "${DYAD_INSTALL_CMAKE_DIR}" + FILES_MATCHING PATTERN "*.cmake") install(DIRECTORY cmake/modules - DESTINATION "${DYAD_LIBDIR}/cmake/dyad" + DESTINATION "${DYAD_INSTALL_CMAKE_DIR}" FILES_MATCHING PATTERN "*.cmake") -endif (WIP_package_config) -#........................................... install(FILES "${CMAKE_INCLUDE_OUTPUT_DIRECTORY}/dyad/dyad_config.hpp" DESTINATION - ${DYAD_INSTALL_INCLUDE_DIR}/dyad) - -install(EXPORT - ${DYAD_EXPORTED_TARGETS} - DESTINATION - ${DYAD_LIBDIR}/cmake/${PROJECT_NAME} - FILE - ${DYAD_EXPORTED_TARGETS}.cmake - ) + ${DYAD_INSTALL_INCLUDEDIR}/dyad) # Install license and readme install(FILES @@ -455,10 +401,11 @@ string(APPEND _str " DYAD_PACKAGE_VERSION: ${DYAD_PACKAGE_VERSION}\n") string(APPEND _str " DYAD_GIT_VERSION: ${DYAD_GIT_VERSION}\n") -string(APPEND _str - " DYAD_ENABLE_UCX_DATA: ${DYAD_ENABLE_UCX_DATA}\n") -string(APPEND _str - " DYAD_ENABLE_UCX_DATA_RMA: ${DYAD_ENABLE_UCX_DATA_RMA}\n") +# Comment out since they'll be printed again below +# string(APPEND _str +# " DYAD_ENABLE_UCX_DATA: ${DYAD_ENABLE_UCX_DATA}\n") +# string(APPEND _str +# " DYAD_ENABLE_UCX_DATA_RMA: ${DYAD_ENABLE_UCX_DATA_RMA}\n") string(APPEND _str " DYAD_ENABLE_TESTS: ${DYAD_ENABLE_TESTS}\n") string(APPEND _str diff --git a/cmake/configure_files/DYADConfig.cmake.in b/cmake/configure_files/DYADConfig.cmake.in index 4e805d32..f9b3c1b1 100644 --- a/cmake/configure_files/DYADConfig.cmake.in +++ b/cmake/configure_files/DYADConfig.cmake.in @@ -8,10 +8,11 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/modules") list(APPEND CMAKE_MODULE_PATH "@EXTRA_CMAKE_MODULE_DIR@") -#include(GNUInstallDirs) -include(ExternalProject) -include(DYADCMakeUtilities) -include(CMakePackageConfigHelpers) +# None of these are currently needed +# include(GNUInstallDirs) +# include(ExternalProject) +# include(DYADCMakeUtilities) +# include(CMakePackageConfigHelpers) set(DYAD_VERSION ${PACKAGE_VERSION}) @@ -26,8 +27,10 @@ set(DYAD_CXX_FLAGS "@CMAKE_CXX_FLAGS@") set(DYAD_C_STANDARD "@CMAKE_C_STANDARD@") set(DYAD_CXX_STANDARD "@CMAKE_CXX_STANDARD@") -set(CMAKE_C_STANDARD_REQUIRED TRUE) -set(CMAKE_CXX_STANDARD_REQUIRED TRUE) +# Using DYAD-specific versions of these variables so that we don't risk +# polluting user CMake scripts +set(DYAD_C_STANDARD_REQUIRED TRUE) +set(DYAD_CXX_STANDARD_REQUIRED TRUE) # Record the various flags and switches accumlated in DYAD set(DYAD_GNU_LINUX @DYAD_GNU_LINUX@) diff --git a/cmake/modules/SetupCompiler.cmake b/cmake/modules/SetupCompiler.cmake index 69855d1d..a708c40a 100644 --- a/cmake/modules/SetupCompiler.cmake +++ b/cmake/modules/SetupCompiler.cmake @@ -26,14 +26,20 @@ include(CheckIncludeFileCXX) MACRO (CHECK_GLIBC_VERSION) EXECUTE_PROCESS ( - COMMAND ${CMAKE_C_COMPILER} -print-file-name=libc.so.6 - OUTPUT_VARIABLE GLIBC - OUTPUT_STRIP_TRAILING_WHITESPACE) - - GET_FILENAME_COMPONENT (GLIBC ${GLIBC} REALPATH) - GET_FILENAME_COMPONENT (GLIBC_VERSION ${GLIBC} NAME) - STRING (REPLACE "libc-" "" GLIBC_VERSION ${GLIBC_VERSION}) - STRING (REPLACE ".so" "" GLIBC_VERSION ${GLIBC_VERSION}) + COMMAND ${CMAKE_C_COMPILER} ${CMAKE_SOURCE_DIR}/cmake/tests/check_glibc_version.c -o ${CMAKE_BINARY_DIR}/check_glibc_version + RESULT_VARIABLE COULD_COMPILE_GLIBC_CHECKER) + if (NOT COULD_COMPILE_GLIBC_CHECKER EQUAL 0) + message(FATAL_ERROR "Could not compile glibc version checker") + endif () + EXECUTE_PROCESS( + COMMAND ${CMAKE_BINARY_DIR}/check_glibc_version + OUTPUT_VARIABLE GLIBC_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE COULD_GET_GLIBC_VERSION) + if (NOT COULD_GET_GLIBC_VERSION EQUAL 0) + message(FATAL_ERROR "Could not get glibc version") + endif () + IF (NOT GLIBC_VERSION MATCHES "^[0-9.]+$") MESSAGE (FATAL_ERROR "Unknown glibc version: ${GLIBC_VERSION}") ENDIF (NOT GLIBC_VERSION MATCHES "^[0-9.]+$") @@ -89,11 +95,11 @@ macro(dyad_add_c_flags MY_FLAGS) endmacro() dyad_add_cxx_flags(CMAKE_CXX_FLAGS - -fPIC -Wall -Wextra -pedantic -Wno-unused-parameter -Wnon-virtual-dtor + -Wall -Wextra -pedantic -Wno-unused-parameter -Wnon-virtual-dtor -Wno-deprecated-declarations) dyad_add_c_flags(CMAKE_C_FLAGS - -fPIC -Wall -Wextra -pedantic -Wno-unused-parameter + -Wall -Wextra -pedantic -Wno-unused-parameter -Wno-deprecated-declarations) if (${GLIBC_VERSION} VERSION_GREATER_EQUAL "2.19") @@ -105,24 +111,13 @@ endif () # Promote a compiler warning as an error for project targets ################################################################ -if (DYAD_WARNINGS_AS_ERRORS) - dyad_add_cxx_flags(_WERROR_FLAGS -Werror) - separate_arguments(_WERROR_FLAGS NATIVE_COMMAND "${_WERROR_FLAGS}") - if (NOT TARGET DYAD_CXX_FLAGS_werror) - add_library(DYAD_CXX_FLAGS_werror INTERFACE) - set_property(TARGET DYAD_CXX_FLAGS_werror PROPERTY - INTERFACE_COMPILE_OPTIONS $<$:${_WERROR_FLAGS}>) - - add_library(DYAD_C_FLAGS_werror INTERFACE) - set_property(TARGET DYAD_C_FLAGS_werror PROPERTY - INTERFACE_COMPILE_OPTIONS $<$:${_WERROR_FLAGS}>) - - # Add the "library" to the export - install(TARGETS DYAD_C_FLAGS_werror EXPORT ${DYAD_EXPORTED_TARGETS}) - install(TARGETS DYAD_CXX_FLAGS_werror EXPORT ${DYAD_EXPORTED_TARGETS}) - endif () -endif () - +macro(dyad_add_werror_if_needed target) + if (DYAD_WARNINGS_AS_ERRORS) + target_compile_options(${target} PRIVATE + $<$:"-Werror"> + $<$:"-Werror">) + endif() +endmacro(dyad_add_werror_if_needed target) ################################################################ # Handle compiler dependent behaviors @@ -196,23 +191,15 @@ set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) # which point to directories outside the build tree to the install RPATH set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) -if (NOT DYAD_LIBDIR) - if (CMAKE_INSTALL_LIBDIR) - set(DYAD_LIBDIR ${CMAKE_INSTALL_LIBDIR}) - else () - set(DYAD_LIBDIR "lib") - endif () -endif () - -set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${DYAD_LIBDIR}") +set(CMAKE_INSTALL_RPATH "${DYAD_INSTALL_LIBDIR}") list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES - "${CMAKE_INSTALL_PREFIX}/${DYAD_LIBDIR}" _IS_SYSTEM_DIR) + "${DYAD_INSTALL_LIBDIR}" _IS_SYSTEM_DIR) if (${_IS_SYSTEM_DIR} STREQUAL "-1") # Set the install RPATH correctly list(APPEND CMAKE_INSTALL_RPATH - "${CMAKE_INSTALL_PREFIX}/${DYAD_LIBDIR}") + "${DYAD_INSTALL_LIBDIR}") endif () diff --git a/cmake/tests/check_glibc_version.c b/cmake/tests/check_glibc_version.c new file mode 100644 index 00000000..c5a51646 --- /dev/null +++ b/cmake/tests/check_glibc_version.c @@ -0,0 +1,8 @@ +#include +#include + +int main(void) +{ + puts(gnu_get_libc_version()); + return 0; +} \ No newline at end of file diff --git a/include/dyad/client/dyad_client.h b/include/dyad/client/dyad_client.h new file mode 100644 index 00000000..d0304f0e --- /dev/null +++ b/include/dyad/client/dyad_client.h @@ -0,0 +1,99 @@ +#ifndef DYAD_CORE_DYAD_CORE_H +#define DYAD_CORE_DYAD_CORE_H + +#if defined(DYAD_HAS_CONFIG) +#include +#else +#error "no config" +#endif + +#include +#include +#include +#include +#include + +#ifdef __cplusplus +#include +#include +#include +#else +#include +#include +#include +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif +#if DYAD_PERFFLOW +#define DYAD_CORE_FUNC_MODS __attribute__((annotate("@critical_path()"))) static +#else +#define DYAD_CORE_FUNC_MODS static inline +#endif + +struct dyad_metadata { + char *fpath; + uint32_t owner_rank; +}; +typedef struct dyad_metadata dyad_metadata_t; + +/** + * @brief Wrapper function that performs all the common tasks needed + * of a producer + * @param[in] ctx the DYAD context for the operation + * @param[in] fname the name of the file being "produced" + * + * @return An error code from dyad_rc.h + */ +DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_produce(dyad_ctx_t *ctx, + const char *fname); + +/** + * @brief Obtain DYAD metadata for a file in the consumer-managed directory + * @param[in] ctx the DYAD context for the operation + * @param[in] fname the name of the file for which metadata is obtained + * @param[in] should_wait if true, wait for the file to be produced before + * returning + * @param[out] mdata a dyad_metadata_t object containing the metadata for + * the file + * + * @return An error code from dyad_rc.h + */ +DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t +dyad_get_metadata(dyad_ctx_t *ctx, const char *fname, bool should_wait, + dyad_metadata_t **mdata); + +DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t +dyad_free_metadata(dyad_metadata_t **mdata); + +/** + * @brief Wrapper function that performs all the common tasks needed + * of a consumer + * @param[in] ctx the DYAD context for the operation + * @param[in] fname the name of the file being "consumed" + * + * @return An error code from dyad_rc.h + */ +DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_consume(dyad_ctx_t *ctx, + const char *fname); + +/** + * @brief Wrapper function that performs all the common tasks needed + * of a consumer + * @param[in] ctx the DYAD context for the operation + * @param[in] fname the name of the file being "consumed" + * @param[in] mdata a dyad_metadata_t object containing the metadata for the + * file User is responsible for deallocating this object + * + * @return An error code from dyad_rc.h + */ +DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_consume_w_metadata( + dyad_ctx_t *ctx, const char *fname, const dyad_metadata_t *mdata); + +#ifdef __cplusplus +} +#endif + +#endif /* DYAD_CORE_DYAD_CORE */ diff --git a/src/dyad/common/dyad_dtl.h b/include/dyad/common/dyad_dtl.h similarity index 100% rename from src/dyad/common/dyad_dtl.h rename to include/dyad/common/dyad_dtl.h diff --git a/src/dyad/common/dyad_envs.h b/include/dyad/common/dyad_envs.h similarity index 100% rename from src/dyad/common/dyad_envs.h rename to include/dyad/common/dyad_envs.h diff --git a/src/dyad/common/dyad_rc.h b/include/dyad/common/dyad_rc.h similarity index 100% rename from src/dyad/common/dyad_rc.h rename to include/dyad/common/dyad_rc.h diff --git a/include/dyad/common/dyad_structures.h b/include/dyad/common/dyad_structures.h new file mode 100644 index 00000000..cdf96370 --- /dev/null +++ b/include/dyad/common/dyad_structures.h @@ -0,0 +1,13 @@ +#ifndef DYAD_COMMON_STRUCTURES_H +#define DYAD_COMMON_STRUCTURES_H + +#if defined(DYAD_HAS_CONFIG) +#include +#else +#error "no config" +#endif + +struct dyad_ctx; +typedef struct dyad_ctx dyad_ctx_t; + +#endif /* DYAD_COMMON_STRUCTURES_H */ \ No newline at end of file diff --git a/src/dyad/core/dyad_ctx.h b/include/dyad/core/dyad_ctx.h similarity index 52% rename from src/dyad/core/dyad_ctx.h rename to include/dyad/core/dyad_ctx.h index e057f319..a09be760 100644 --- a/src/dyad/core/dyad_ctx.h +++ b/include/dyad/core/dyad_ctx.h @@ -7,18 +7,20 @@ #error "no config" #endif -#include #include +#include #include #ifdef __cplusplus extern "C" { #endif -DYAD_DLL_EXPORTED dyad_ctx_t* dyad_ctx_get (); -DYAD_DLL_EXPORTED void dyad_ctx_init (dyad_dtl_comm_mode_t dtl_comm_mode, - void* flux_handle); -DYAD_DLL_EXPORTED void dyad_ctx_fini (); +DYAD_DLL_EXPORTED extern const struct dyad_ctx dyad_ctx_default; + +DYAD_DLL_EXPORTED dyad_ctx_t *dyad_ctx_get(); +DYAD_DLL_EXPORTED void dyad_ctx_init(dyad_dtl_comm_mode_t dtl_comm_mode, + void *flux_handle); +DYAD_DLL_EXPORTED void dyad_ctx_fini(); /** * @brief Intialize the DYAD context @@ -34,42 +36,32 @@ DYAD_DLL_EXPORTED void dyad_ctx_fini (); * @param[in] service_mux number of brokers sharing node-local storage * @param[in] kvs_namespace Flux KVS namespace to be used for this * instance of DYAD - * @param[in] relative_to_managed_path relative path is relative to the managed path - * Which one of the managed path it will be relative to - * depends on the context and it could be either of - * producer or consumer's path. This is only applicable - * when there is only one of each. + * @param[in] relative_to_managed_path relative path is relative to the managed + * path Which one of the managed path it will be relative to depends on the + * context and it could be either of producer or consumer's path. This is only + * applicable when there is only one of each. * @param[in] dtl_mode_str DTL mode * @param[in] dtl_comm_mode DTL comm mode - * @param[in] flux_handle Flux handle (flux_t*). If NULL, it will be obtained via - * `flux_open()` + * @param[in] flux_handle Flux handle (flux_t*). If NULL, it will be + * obtained via `flux_open()` * * @return An error code from dyad_rc.h */ -DYAD_DLL_EXPORTED dyad_rc_t dyad_init (bool debug, - bool check, - bool shared_storage, - bool reinit, - bool async_publish, - bool fsync_write, - unsigned int key_depth, - unsigned int key_bins, - unsigned int service_mux, - const char* kvs_namespace, - const char* prod_managed_path, - const char* cons_managed_path, - bool relative_to_managed_path, - const char* dtl_mode_str, - const dyad_dtl_comm_mode_t dtl_comm_mode, - void* flux_handle); +DYAD_DLL_EXPORTED dyad_rc_t dyad_init( + bool debug, bool check, bool shared_storage, bool reinit, + bool async_publish, bool fsync_write, unsigned int key_depth, + unsigned int key_bins, unsigned int service_mux, const char *kvs_namespace, + const char *prod_managed_path, const char *cons_managed_path, + bool relative_to_managed_path, const char *dtl_mode_str, + const dyad_dtl_comm_mode_t dtl_comm_mode, void *flux_handle); /** * @brief Intialize the DYAD context using environment variables * * @return An error code from dyad_rc.h */ -DYAD_DLL_EXPORTED dyad_rc_t dyad_init_env (const dyad_dtl_comm_mode_t dtl_comm_mode, - void* flux_handle); +DYAD_DLL_EXPORTED dyad_rc_t +dyad_init_env(const dyad_dtl_comm_mode_t dtl_comm_mode, void *flux_handle); /** * @brief Reset producer path. Can be used by the module @@ -77,7 +69,7 @@ DYAD_DLL_EXPORTED dyad_rc_t dyad_init_env (const dyad_dtl_comm_mode_t dtl_comm_m * * @return An error code from dyad_rc.h */ -DYAD_DLL_EXPORTED dyad_rc_t dyad_set_prod_path (const char* path); +DYAD_DLL_EXPORTED dyad_rc_t dyad_set_prod_path(const char *path); /** * @brief Reset consumer path. Can be used by the module @@ -85,7 +77,7 @@ DYAD_DLL_EXPORTED dyad_rc_t dyad_set_prod_path (const char* path); * * @return An error code from dyad_rc.h */ -DYAD_DLL_EXPORTED dyad_rc_t dyad_set_cons_path (const char* path); +DYAD_DLL_EXPORTED dyad_rc_t dyad_set_cons_path(const char *path); /** * @brief Reset dtl mode. Can be used by the module @@ -94,8 +86,8 @@ DYAD_DLL_EXPORTED dyad_rc_t dyad_set_cons_path (const char* path); * * @return An error code from dyad_rc.h */ -DYAD_DLL_EXPORTED dyad_rc_t dyad_set_and_init_dtl_mode (const char* dtl_mode_name, - dyad_dtl_comm_mode_t dtl_comm_mode); +DYAD_DLL_EXPORTED dyad_rc_t dyad_set_and_init_dtl_mode( + const char *dtl_mode_name, dyad_dtl_comm_mode_t dtl_comm_mode); /** * Reset the contents of the ctx to the default values and deallocate @@ -103,14 +95,14 @@ DYAD_DLL_EXPORTED dyad_rc_t dyad_set_and_init_dtl_mode (const char* dtl_mode_nam * This is needed for wrapper to handle dyad exceptions as the wrapper requires * ctx for it's lifetime */ -DYAD_DLL_EXPORTED dyad_rc_t dyad_clear (); +DYAD_DLL_EXPORTED dyad_rc_t dyad_clear(); /** * @brief Finalizes the DYAD instance and deallocates the context * * @return An error code from dyad_rc.h */ -DYAD_DLL_EXPORTED dyad_rc_t dyad_finalize (); +DYAD_DLL_EXPORTED dyad_rc_t dyad_finalize(); #ifdef __cplusplus } diff --git a/src/dyad/CMakeLists.txt b/src/dyad/CMakeLists.txt index fe73effa..cc20a321 100644 --- a/src/dyad/CMakeLists.txt +++ b/src/dyad/CMakeLists.txt @@ -3,6 +3,7 @@ include_directories(${CMAKE_BINARY_DIR}/include) add_subdirectory(utils) add_subdirectory(dtl) add_subdirectory(core) +add_subdirectory(client) add_subdirectory(modules) add_subdirectory(wrapper) add_subdirectory(stream) diff --git a/src/dyad/client/CMakeLists.txt b/src/dyad/client/CMakeLists.txt new file mode 100644 index 00000000..87dc2ed8 --- /dev/null +++ b/src/dyad/client/CMakeLists.txt @@ -0,0 +1,54 @@ +set(DYAD_CLIENT_SRC ${CMAKE_CURRENT_SOURCE_DIR}/dyad_client.c) +set(DYAD_CLIENT_PRIVATE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_logging.h + ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_profiler.h + ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_structures_int.h + ${CMAKE_CURRENT_SOURCE_DIR}/../dtl/dyad_dtl_api.h + ${CMAKE_CURRENT_SOURCE_DIR}/../utils/utils.h + ${CMAKE_CURRENT_SOURCE_DIR}/../utils/murmur3.h + ${CMAKE_CURRENT_SOURCE_DIR}/dyad_client_int.h) +set(DYAD_CLIENT_PUBLIC_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/common/dyad_rc.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/common/dyad_dtl.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/common/dyad_envs.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/common/dyad_structures.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/client/dyad_client.h) + +add_library(${PROJECT_NAME}_client SHARED ${DYAD_CLIENT_SRC} + ${DYAD_CLIENT_PUBLIC_HEADERS} ${DYAD_CLIENT_PRIVATE_HEADERS}) +set_target_properties(${PROJECT_NAME}_client PROPERTIES CMAKE_INSTALL_RPATH + "${CMAKE_INSTALL_PREFIX}/${DYAD_LIBDIR}") +target_link_libraries(${PROJECT_NAME}_client PRIVATE Jansson::Jansson flux::core) +target_link_libraries(${PROJECT_NAME}_client PRIVATE ${PROJECT_NAME}_utils + ${PROJECT_NAME}_murmur3 ${PROJECT_NAME}_dtl) +target_link_libraries(${PROJECT_NAME}_client PUBLIC ${PROJECT_NAME}_ctx) + +target_compile_definitions(${PROJECT_NAME}_client PUBLIC BUILDING_DYAD=1) +target_compile_definitions(${PROJECT_NAME}_client PUBLIC DYAD_HAS_CONFIG) +target_include_directories(${PROJECT_NAME}_client PUBLIC + $ # will be used for generated code + $) # wil be used for sub projects +target_include_directories(${PROJECT_NAME}_client SYSTEM PRIVATE ${JANSSON_INCLUDE_DIRS}) +target_include_directories(${PROJECT_NAME}_client SYSTEM PRIVATE ${FluxCore_INCLUDE_DIRS}) + +if (TARGET DYAD_C_FLAGS_werror) + target_link_libraries(${PROJECT_NAME}_client PRIVATE DYAD_C_FLAGS_werror) +endif () + +if(DYAD_PROFILER STREQUAL "PERFFLOW_ASPECT") + target_link_libraries(${PROJECT_NAME}_client PRIVATE perfflowaspect::perfflowaspect) + target_include_directories(${PROJECT_NAME} SYSTEM PRIVATE ${perfflowaspect_INCLUDE_DIRS}) + target_compile_definitions(${PROJECT_NAME}_client PRIVATE DYAD_PERFFLOW=1) +endif() +if(DYAD_PROFILER STREQUAL "DFTRACER") + target_link_libraries(${PROJECT_NAME}_client PRIVATE ${DFTRACER_LIBRARIES}) +endif() + +install( + TARGETS ${PROJECT_NAME}_client + EXPORT ${DYAD_EXPORTED_TARGETS} + LIBRARY DESTINATION ${DYAD_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${DYAD_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${DYAD_INSTALL_BIN_DIR} +) +if(NOT "${DYAD_CLIENT_PUBLIC_HEADERS}" STREQUAL "") + dyad_install_headers("${DYAD_CLIENT_PUBLIC_HEADERS}" ${CMAKE_CURRENT_SOURCE_DIR}) +endif() \ No newline at end of file diff --git a/src/dyad/client/dyad_client.c b/src/dyad/client/dyad_client.c new file mode 100644 index 00000000..b6537920 --- /dev/null +++ b/src/dyad/client/dyad_client.c @@ -0,0 +1,956 @@ +#if defined(DYAD_HAS_CONFIG) +#include +#else +#error "no config" +#endif + +// clang-format off +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +// clang-format on + +#ifdef __cplusplus +#include +#include +#else +#include +#include +#include +#endif + +DYAD_DLL_EXPORTED int gen_path_key(const char *restrict str, + char *restrict path_key, const size_t len, + const uint32_t depth, const uint32_t width) { + DYAD_C_FUNCTION_START(); + static const uint32_t seeds[10] = {104677u, 104681u, 104683u, 104693u, + 104701u, 104707u, 104711u, 104717u, + 104723u, 104729u}; + + uint32_t seed = 57u; + uint32_t hash[4] = {0u}; // Output for the hash + size_t cx = 0ul; + int n = 0; + if (str == NULL || path_key == NULL || len == 0ul) { + DYAD_C_FUNCTION_END(); + return -1; + } + size_t str_len = strlen(str); + if (str_len == 0ul) { + DYAD_C_FUNCTION_END(); + return -1; + } + const char *str_long = str; + + path_key[0] = '\0'; + + // Just append the string so that it can be as large as 128 bytes. + if (str_len < 128ul) { + char buf[256] = {'\0'}; + memcpy(buf, str, str_len); + memset(buf + str_len, '@', 128ul - str_len); + buf[128u] = '\0'; + str_len = 128ul; + str_long = buf; + } + + for (uint32_t d = 0u; d < depth; d++) { + seed += seeds[d % 10]; + MurmurHash3_x64_128(str_long, str_len, seed, hash); + uint32_t bin = (hash[0] ^ hash[1] ^ hash[2] ^ hash[3]) % width; + n = snprintf(path_key + cx, len - cx, "%x.", bin); + cx += n; + if (cx >= len || n < 0) { + DYAD_C_FUNCTION_END(); + return -1; + } + } + n = snprintf(path_key + cx, len - cx, "%s", str); + // FIXME: cx + n >= len fails for str_len > 256 + if (n < 0) { + DYAD_C_FUNCTION_END(); + return -1; + } + DYAD_C_FUNCTION_UPDATE_STR("path_key", path_key); + DYAD_C_FUNCTION_END(); + return 0; +} + +static void future_cleanup_cb(flux_future_t *f, void *arg) { + if (flux_future_get(f, NULL) < 0) { + DYAD_LOG_STDERR("future_cleanup: future error detected with.%s", ""); + } + flux_future_destroy(f); +} + +DYAD_CORE_FUNC_MODS dyad_rc_t dyad_kvs_commit(const dyad_ctx_t *restrict ctx, + flux_kvs_txn_t *restrict txn) { + DYAD_C_FUNCTION_START(); + flux_future_t *f = NULL; + dyad_rc_t rc = DYAD_RC_OK; + DYAD_LOG_INFO(ctx, "Committing transaction to KVS"); + // Commit the transaction to the Flux KVS + f = flux_kvs_commit((flux_t *)ctx->h, ctx->kvs_namespace, 0, txn); + // If the commit failed, log an error and return DYAD_BADCOMMIT + if (f == NULL) { + DYAD_LOG_ERROR(ctx, "Could not commit transaction to Flux KVS"); + rc = DYAD_RC_BADCOMMIT; + goto kvs_commit_region_finish; + } + if (ctx->async_publish) { + if (flux_future_then(f, -1, future_cleanup_cb, NULL) < 0) { + DYAD_LOG_ERROR(ctx, "Error with flux_future_then"); + } + } else { + // If the commit is pending, wait for it to complete + flux_future_wait_for(f, -1.0); + // Once the commit is complete, destroy the future and transaction + flux_future_destroy(f); + f = NULL; + } + rc = DYAD_RC_OK; +kvs_commit_region_finish:; + DYAD_C_FUNCTION_END(); + return rc; +} + +DYAD_CORE_FUNC_MODS dyad_rc_t publish_via_flux(const dyad_ctx_t *restrict ctx, + const char *restrict upath) { + DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_UPDATE_STR("fname", ctx->fname); + DYAD_C_FUNCTION_UPDATE_STR("upath", upath); + dyad_rc_t rc = DYAD_RC_OK; + flux_kvs_txn_t *txn = NULL; + const size_t topic_len = PATH_MAX; + char topic[PATH_MAX + 1] = {'\0'}; + memset(topic, 0, topic_len + 1); + memset(topic, '\0', topic_len + 1); + // Generate the KVS key from the file path relative to + // the producer-managed directory + DYAD_LOG_INFO(ctx, "Generating KVS key from path (%s)", upath); + gen_path_key(upath, topic, topic_len, ctx->key_depth, ctx->key_bins); + // Crete and pack a Flux KVS transaction. + // The transaction will contain a single key-value pair + // with the previously generated key as the key and the + // producer's rank as the value + DYAD_LOG_INFO(ctx, "Creating KVS transaction under the key %s", topic); + txn = flux_kvs_txn_create(); + if (txn == NULL) { + DYAD_LOG_ERROR(ctx, "Could not create Flux KVS transaction"); + rc = DYAD_RC_FLUXFAIL; + goto publish_done; + } + if (flux_kvs_txn_pack(txn, 0, topic, "i", ctx->rank) < 0) { + DYAD_LOG_ERROR(ctx, "Could not pack Flux KVS transaction"); + rc = DYAD_RC_FLUXFAIL; + goto publish_done; + } + // Call dyad_kvs_commit to commit the transaction into the Flux KVS + rc = dyad_kvs_commit(ctx, txn); + // If dyad_kvs_commit failed, log an error and forward the return code + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(ctx, "dyad_kvs_commit failed!"); + goto publish_done; + } + rc = DYAD_RC_OK; +publish_done:; + if (txn != NULL) { + flux_kvs_txn_destroy(txn); + } + DYAD_C_FUNCTION_END(); + return rc; +} + +DYAD_DLL_EXPORTED dyad_rc_t dyad_commit(dyad_ctx_t *restrict ctx, + const char *restrict fname) { + DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_UPDATE_STR("fname", ctx->fname); + dyad_rc_t rc = DYAD_RC_OK; + char upath[PATH_MAX + 1] = {'\0'}; +#if 0 + if (fname == NULL || strlen (fname) > PATH_MAX) { + rc = DYAD_RC_SYSFAIL; + goto get_metadata_done; + } +#endif + // As this is a function called for DYAD producer, ctx->prod_managed_path + // must be a valid string (!NULL). ctx->delim_len is verified to be greater + // than 0 during initialization. + if (ctx->relative_to_managed_path && + //(strlen (fname) > 0ul) && // checked where get_path() was + (strncmp(fname, DYAD_PATH_DELIM, ctx->delim_len) != + 0)) { // fname is a relative path that is relative to the + // prod_managed_path + memcpy(upath, fname, strlen(fname)); + } else if (!cmp_canonical_path_prefix(ctx, true, fname, upath, PATH_MAX)) { + // Extract the path to the file specified by fname relative to the + // producer-managed path + // This relative path will be stored in upath + DYAD_LOG_INFO(ctx, "%s is not in the Producer's managed path", fname); + rc = DYAD_RC_OK; + goto commit_done; + } + DYAD_C_FUNCTION_UPDATE_STR("upath", upath); + DYAD_LOG_INFO(ctx, "Obtained file path relative to producer directory: %s", + upath); + // Call publish_via_flux to actually store information about the file into + // the Flux KVS + // Fence this call with reassignments of reenter so that, if intercepting + // file I/O API calls, we will not get stuck in infinite recursion + ctx->reenter = false; + rc = publish_via_flux(ctx, upath); + ctx->reenter = true; + +commit_done:; + // If "check" is set and the operation was successful, set the + // DYAD_CHECK_ENV environment variable to "ok" + if (rc == DYAD_RC_OK && (ctx && ctx->check)) { + setenv(DYAD_CHECK_ENV, "ok", 1); + } + DYAD_C_FUNCTION_END(); + return rc; +} + +static void print_mdata(const dyad_ctx_t *restrict ctx, + const dyad_metadata_t *restrict mdata) { + if (mdata == NULL) { + DYAD_LOG_INFO(ctx, "Cannot print a NULL metadata object!"); + } else { + DYAD_LOG_INFO(ctx, "Printing contents of DYAD Metadata object"); + DYAD_LOG_INFO(ctx, "fpath = %s", mdata->fpath); + DYAD_LOG_INFO(ctx, "owner_rank = %u", mdata->owner_rank); + } +} + +DYAD_DLL_EXPORTED dyad_rc_t dyad_kvs_read(const dyad_ctx_t *restrict ctx, + const char *restrict topic, + const char *restrict upath, + bool should_wait, + dyad_metadata_t **restrict mdata) { + DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_UPDATE_STR("upath", upath); + dyad_rc_t rc = DYAD_RC_OK; + int kvs_lookup_flags = 0; + flux_future_t *f = NULL; + if (mdata == NULL) { + DYAD_LOG_ERROR(ctx, "Metadata double pointer is NULL. " + "Cannot correctly create metadata object"); + rc = DYAD_RC_NOTFOUND; + goto kvs_read_end; + } + // Lookup information about the desired file (represented by kvs_topic) + // from the Flux KVS. If there is no information, wait for it to be + // made available + if (should_wait) + kvs_lookup_flags = FLUX_KVS_WAITCREATE; + DYAD_LOG_INFO(ctx, "Retrieving information from KVS under the key %s", topic); + f = flux_kvs_lookup((flux_t *)ctx->h, ctx->kvs_namespace, kvs_lookup_flags, + topic); + // If the KVS lookup failed, log an error and return DYAD_BADLOOKUP + if (f == NULL) { + DYAD_LOG_ERROR(ctx, "KVS lookup failed!\n"); + rc = DYAD_RC_NOTFOUND; + goto kvs_read_end; + } + // Extract the rank of the producer from the KVS response + DYAD_LOG_INFO(ctx, "Building metadata object from KVS entry\n"); + if (*mdata != NULL) { + DYAD_LOG_INFO(ctx, + "Metadata object is already allocated. Skipping allocation"); + } else { + *mdata = (dyad_metadata_t *)malloc(sizeof(struct dyad_metadata)); + if (*mdata == NULL) { + DYAD_LOG_ERROR(ctx, "Cannot allocate memory for metadata object"); + rc = DYAD_RC_SYSFAIL; + goto kvs_read_end; + } + } + size_t upath_len = strlen(upath); + (*mdata)->fpath = (char *)malloc(upath_len + 1); + if ((*mdata)->fpath == NULL) { + DYAD_LOG_ERROR(ctx, "Cannot allocate memory for fpath in metadata object"); + rc = DYAD_RC_SYSFAIL; + goto kvs_read_end; + } + memset((*mdata)->fpath, '\0', upath_len + 1); + memcpy((*mdata)->fpath, upath, upath_len); + rc = flux_kvs_lookup_get_unpack(f, "i", &((*mdata)->owner_rank)); + // If the extraction did not work, log an error and return DYAD_BADFETCH + if (rc < 0) { + DYAD_LOG_ERROR(ctx, "Could not unpack owner's rank from KVS response\n"); + rc = DYAD_RC_BADMETADATA; + goto kvs_read_end; + } + DYAD_LOG_INFO(ctx, "Successfully created DYAD Metadata object"); + print_mdata(ctx, *mdata); + DYAD_C_FUNCTION_UPDATE_STR("fpath", (*mdata)->fpath); + DYAD_C_FUNCTION_UPDATE_INT("owner_rank", (*mdata)->owner_rank); + rc = DYAD_RC_OK; + +kvs_read_end:; + if (DYAD_IS_ERROR(rc) && mdata != NULL && *mdata != NULL) { + dyad_free_metadata(mdata); + } + if (f != NULL) { + flux_future_destroy(f); + f = NULL; + } + DYAD_C_FUNCTION_END(); + return rc; +} + +DYAD_CORE_FUNC_MODS dyad_rc_t dyad_fetch_metadata( + const dyad_ctx_t *restrict ctx, const char *restrict fname, + const char *restrict upath, dyad_metadata_t **restrict mdata) { + DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_UPDATE_STR("fname", fname); + dyad_rc_t rc = DYAD_RC_OK; + const size_t topic_len = PATH_MAX; + char topic[PATH_MAX + 1] = {'\0'}; + *mdata = NULL; +#if 0 + if (fname == NULL || upath == NULL || strlen (fname) == 0ul || strlen (upath) == 0ul) { + rc = DYAD_RC_BADFIO; + goto get_metadata_done; + } +#endif + // Set reenter to false to avoid recursively performing DYAD operations + DYAD_LOG_INFO(ctx, "Obtained file path relative to consumer directory: %s\n", + upath); + DYAD_C_FUNCTION_UPDATE_STR("upath", upath); + // Generate the KVS key from the file path relative to + // the consumer-managed directory + gen_path_key(upath, topic, topic_len, ctx->key_depth, ctx->key_bins); + DYAD_LOG_INFO(ctx, "Generated KVS key for consumer: %s\n", topic); + // Call dyad_kvs_read to retrieve infromation about the file + // from the Flux KVS + rc = dyad_kvs_read(ctx, topic, upath, true, mdata); + // If an error occured in dyad_kvs_read, log it and propagate the return + // code + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(ctx, "dyad_kvs_read failed!\n"); + goto fetch_done; + } + // There are two cases where we do not want to perform file transfer: + // 1. if the shared storage feature is enabled + // 2. if the producer and consumer share the same rank + // In either of these cases, skip the creation of the dyad_kvs_response_t + // object, and return DYAD_OK. This will cause the file transfer step to be + // skipped + DYAD_C_FUNCTION_UPDATE_INT("owner_rank", (*mdata)->owner_rank); + DYAD_C_FUNCTION_UPDATE_INT("node_idx", ctx->node_idx); + if (((*mdata)->owner_rank / ctx->service_mux) == ctx->node_idx) { + DYAD_LOG_INFO( + ctx, + "Either shared-storage is indicated or the producer rank (%u) is the" + " same as the consumer rank (%u)", + (*mdata)->owner_rank, ctx->rank); + if (mdata != NULL && *mdata != NULL) { + dyad_free_metadata(mdata); + } + rc = DYAD_RC_OK; + DYAD_C_FUNCTION_UPDATE_INT("is_local", 1); + goto fetch_done; + } + DYAD_C_FUNCTION_UPDATE_INT("is_local", 0); + rc = DYAD_RC_OK; + +fetch_done:; + DYAD_C_FUNCTION_END(); + return rc; +} + +DYAD_DLL_EXPORTED dyad_rc_t dyad_get_data(const dyad_ctx_t *restrict ctx, + const dyad_metadata_t *restrict mdata, + char **restrict file_data, + size_t *restrict file_len) { + DYAD_C_FUNCTION_START(); + dyad_rc_t rc = DYAD_RC_OK; + flux_future_t *f = NULL; + json_t *rpc_payload = NULL; + DYAD_LOG_INFO(ctx, "Packing payload for RPC to DYAD module"); + DYAD_C_FUNCTION_UPDATE_INT("owner_rank", mdata->owner_rank); + DYAD_C_FUNCTION_UPDATE_STR("fpath", mdata->fpath); + rc = ctx->dtl_handle->rpc_pack(ctx, mdata->fpath, mdata->owner_rank, + &rpc_payload); + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(ctx, "Cannot create JSON payload for Flux RPC to " + "DYAD module\n"); + goto get_done; + } + DYAD_LOG_INFO(ctx, "Sending payload for RPC to DYAD module"); + f = flux_rpc_pack((flux_t *)ctx->h, DYAD_DTL_RPC_NAME, mdata->owner_rank, + FLUX_RPC_STREAMING, "o", rpc_payload); + if (f == NULL) { + DYAD_LOG_ERROR(ctx, "Cannot send RPC to producer module\n"); + rc = DYAD_RC_BADRPC; + goto get_done; + } + DYAD_LOG_INFO(ctx, "Receive RPC response from DYAD module"); + rc = ctx->dtl_handle->rpc_recv_response(ctx, f); + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(ctx, "Cannot receive and/or parse the RPC response\n"); + goto get_done; + } + DYAD_LOG_INFO(ctx, "Establish DTL connection with DYAD module"); + rc = ctx->dtl_handle->establish_connection(ctx); + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(ctx, + "Cannot establish connection with DYAD module on broker " + "%u\n", + mdata->owner_rank); + goto get_done; + } + DYAD_LOG_INFO(ctx, "Receive file data via DTL"); + rc = ctx->dtl_handle->recv(ctx, (void **)file_data, file_len); + DYAD_LOG_INFO(ctx, "Close DTL connection with DYAD module"); + ctx->dtl_handle->close_connection(ctx); + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(ctx, "Cannot receive data from producer module\n"); + goto get_done; + } + DYAD_C_FUNCTION_UPDATE_INT("file_len", *file_len); + + rc = DYAD_RC_OK; + +get_done:; + // There are two return codes that have special meaning when coming from the + // DTL: + // * DYAD_RC_RPC_FINISHED: occurs when an ENODATA error occurs + // * DYAD_RC_BADRPC: occurs when a previous RPC operation fails + // In either of these cases, we do not need to wait for the end of stream + // because the RPC is already completely messed up. If we do not have either + // of these cases, we will wait for one more RPC message. If everything went + // well in the module, this last message will set errno to ENODATA (i.e., + // end of stream). Otherwise, something went wrong, so we'll return + // DYAD_RC_BADRPC. + DYAD_LOG_INFO( + ctx, "Wait for end-of-stream message from module (current RC = %d)\n", + rc); + if (rc != DYAD_RC_RPC_FINISHED && rc != DYAD_RC_BADRPC) { + if (!(flux_rpc_get(f, NULL) < 0 && errno == ENODATA)) { + DYAD_LOG_ERROR(ctx, + "An error occured at end of getting data! Either the " + "module sent too many responses, or the module " + "failed with a bad error (errno = %d)\n", + errno); + rc = DYAD_RC_BADRPC; + } + } +#ifdef DYAD_ENABLE_UCX_RMA + ctx->dtl_handle->get_buffer(ctx, 0, (void **)file_data); + ssize_t read_len = 0l; + memcpy(&read_len, *file_data, sizeof(read_len)); + if (read_len < 0l) { + *file_len = 0ul; + DYAD_LOG_DEBUG(ctx, "Not able to read from %s file", mdata->fpath); + rc = DYAD_RC_BADFIO; + } else { + *file_len = (size_t)read_len; + } + *file_data = ((char *)*file_data) + sizeof(read_len); + DYAD_LOG_INFO(ctx, "Read %zd bytes from %s file", *file_len, mdata->fpath); +#endif + DYAD_LOG_INFO(ctx, "Destroy the Flux future for the RPC\n"); + flux_future_destroy(f); + DYAD_C_FUNCTION_END(); + return rc; +} + +DYAD_CORE_FUNC_MODS dyad_rc_t dyad_cons_store( + const dyad_ctx_t *restrict ctx, const dyad_metadata_t *restrict mdata, + int fd, const size_t data_len, char *restrict file_data) { + DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_UPDATE_INT("fd", fd); + dyad_rc_t rc = DYAD_RC_OK; + const char *odir = NULL; + char file_path[PATH_MAX + 1] = {'\0'}; + char file_path_copy[PATH_MAX + 1] = {'\0'}; + mode_t m = (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH | S_ISGID); + size_t written_len = 0; + memset(file_path, 0, PATH_MAX + 1); + memset(file_path_copy, 0, PATH_MAX + 1); + + // Build the full path to the file being consumed + strncpy(file_path, ctx->cons_managed_path, PATH_MAX - 1); + concat_str(file_path, mdata->fpath, "/", PATH_MAX); + memcpy(file_path_copy, file_path, PATH_MAX); // dirname modifies the arg + DYAD_C_FUNCTION_UPDATE_STR("cons_managed_path", ctx->cons_managed_path); + DYAD_C_FUNCTION_UPDATE_STR("fpath", mdata->fpath); + DYAD_C_FUNCTION_UPDATE_STR("file_path_copy", file_path_copy); + + DYAD_LOG_INFO(ctx, "Saving retrieved data to %s\n", file_path); + // Create the directory as needed + // TODO: Need to be consistent with the mode at the source + odir = dirname(file_path_copy); + if ((strncmp(odir, ".", strlen(".")) != 0) && + (mkdir_as_needed(odir, m) < 0)) { + DYAD_LOG_ERROR(ctx, "Cannot create needed directories for pulled file\n"); + rc = DYAD_RC_BADFIO; + goto pull_done; + } + + // Write the file contents to the location specified by the user + written_len = write(fd, file_data, data_len); + if (written_len != data_len) { + DYAD_LOG_ERROR(ctx, "cons store write of pulled file failed!\n"); + rc = DYAD_RC_BADFIO; + goto pull_done; + } + DYAD_C_FUNCTION_UPDATE_INT("data_len", data_len); + rc = DYAD_RC_OK; + +pull_done:; + // If "check" is set and the operation was successful, set the + // DYAD_CHECK_ENV environment variable to "ok" + if (rc == DYAD_RC_OK && (ctx && ctx->check)) + setenv(DYAD_CHECK_ENV, "ok", 1); + DYAD_C_FUNCTION_END(); + return rc; +} + +dyad_rc_t dyad_produce(dyad_ctx_t *restrict ctx, const char *restrict fname) { + DYAD_C_FUNCTION_START(); + ctx->fname = fname; + DYAD_C_FUNCTION_UPDATE_STR("fname", ctx->fname); + DYAD_LOG_DEBUG(ctx, "Executing dyad_produce"); + dyad_rc_t rc = DYAD_RC_OK; + // If the context is not defined, then it is not valid. + // So, return DYAD_NOCTX + if (!ctx || !ctx->h) { + DYAD_LOG_ERROR(ctx, "No CTX found in dyad_produce"); + rc = DYAD_RC_NOCTX; + goto produce_done; + } + // If the producer-managed path is NULL or empty, then the context is not + // valid for a producer operation. So, return DYAD_BADMANAGEDPATH + if (ctx->prod_managed_path == NULL) { + DYAD_LOG_ERROR(ctx, "No or empty producer managed path was found %s", + ctx->prod_managed_path); + rc = DYAD_RC_BADMANAGEDPATH; + goto produce_done; + } + // If the context is valid, call dyad_commit to perform + // the producer operation + rc = dyad_commit(ctx, fname); +produce_done:; + DYAD_C_FUNCTION_END(); + return rc; +} + +/** This function is coupled with Python API. This populates `mdata' which + * is used by `dyad_consume_w_metadata ()' + */ +dyad_rc_t dyad_get_metadata(dyad_ctx_t *restrict ctx, + const char *restrict fname, bool should_wait, + dyad_metadata_t **restrict mdata) { + DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_UPDATE_STR("fname", fname); + DYAD_C_FUNCTION_UPDATE_INT("should_wait", should_wait); + dyad_rc_t rc = DYAD_RC_OK; + +#if 0 + if (fname == NULL || strlen (fname) > PATH_MAX) { + rc = DYAD_RC_SYSFAIL; + goto get_metadata_done; + } +#endif + const size_t fname_len = strlen(fname); + char upath[PATH_MAX + 1] = {'\0'}; + + DYAD_LOG_INFO(ctx, "Obtaining file path relative to consumer directory: %s", + upath); + + if (fname_len == 0ul) { + rc = DYAD_RC_BADFIO; + goto get_metadata_done; + } + if (ctx->relative_to_managed_path && + (strncmp(fname, DYAD_PATH_DELIM, ctx->delim_len) != + 0)) { // fname is a relative path that is relative to the + // cons_managed_path + memcpy(upath, fname, fname_len); + } else if (!cmp_canonical_path_prefix(ctx, false, fname, upath, PATH_MAX)) { + // Extract the path to the file specified by fname relative to the + // producer-managed path + // This relative path will be stored in upath + // DYAD_LOG_TRACE (ctx, "%s is not in the Consumer's managed path\n", + // fname); + // NOTE: This is different from what dyad_fetch/commit returns, + // which is DYAD_RC_OK such that dyad does not interfere accesses on + // non-managed directories. + rc = DYAD_RC_UNTRACKED; + goto get_metadata_done; + } + ctx->reenter = false; + DYAD_C_FUNCTION_UPDATE_STR("upath", upath); + + // check if file exist locally, if so skip kvs + int fd = open(fname, O_RDONLY); + if (fd != -1) { + close(fd); + if (mdata == NULL) { + DYAD_LOG_ERROR(ctx, "Metadata double pointer is NULL. " + "Cannot correctly create metadata object"); + rc = DYAD_RC_NOTFOUND; + goto get_metadata_done; + } + if (*mdata != NULL) { + DYAD_LOG_INFO( + ctx, "Metadata object is already allocated. Skipping allocation"); + } else { + *mdata = (dyad_metadata_t *)malloc(sizeof(struct dyad_metadata)); + if (*mdata == NULL) { + DYAD_LOG_ERROR(ctx, "Cannot allocate memory for metadata object"); + rc = DYAD_RC_SYSFAIL; + goto get_metadata_done; + } + } + (*mdata)->fpath = (char *)malloc(fname_len + 1); + if ((*mdata)->fpath == NULL) { + DYAD_LOG_ERROR(ctx, + "Cannot allocate memory for fpath in metadata object"); + rc = DYAD_RC_SYSFAIL; + goto get_metadata_done; + } + memset((*mdata)->fpath, '\0', fname_len + 1); + memcpy((*mdata)->fpath, fname, fname_len); + (*mdata)->owner_rank = ctx->rank; + rc = DYAD_RC_OK; + goto get_metadata_done; + } + + const size_t topic_len = PATH_MAX; + char topic[PATH_MAX + 1] = {'\0'}; + DYAD_LOG_INFO(ctx, "Generating KVS key: %s", topic); + gen_path_key(upath, topic, topic_len, ctx->key_depth, ctx->key_bins); + rc = dyad_kvs_read(ctx, topic, upath, should_wait, mdata); + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(ctx, "Could not read data from the KVS"); + goto get_metadata_done; + } + rc = DYAD_RC_OK; + +get_metadata_done:; + if (DYAD_IS_ERROR(rc) && mdata != NULL && *mdata != NULL) { + dyad_free_metadata(mdata); + } + ctx->reenter = true; + DYAD_C_FUNCTION_END(); + return rc; +} + +dyad_rc_t dyad_free_metadata(dyad_metadata_t **mdata) { + DYAD_C_FUNCTION_START(); + if (mdata == NULL || *mdata == NULL) { + return DYAD_RC_OK; + } + if ((*mdata)->fpath != NULL) + free((*mdata)->fpath); + free(*mdata); + *mdata = NULL; + DYAD_C_FUNCTION_END(); + return DYAD_RC_OK; +} + +dyad_rc_t dyad_consume(dyad_ctx_t *restrict ctx, const char *restrict fname) { + DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_UPDATE_STR("fname", fname); + dyad_rc_t rc = DYAD_RC_OK; + int lock_fd = -1, io_fd = -1; + ssize_t file_size = -1; + char *file_data = NULL; + size_t data_len = 0ul; + dyad_metadata_t *mdata = NULL; + struct flock exclusive_lock; + char upath[PATH_MAX + 1] = {'\0'}; + + // If the context is not defined, then it is not valid. + // So, return DYAD_NOCTX + if (!ctx || !ctx->h) { + rc = DYAD_RC_NOCTX; + goto consume_close; + } + // If the consumer-managed path is NULL or empty, then the context is not + // valid for a consumer operation. So, return DYAD_BADMANAGEDPATH + if (ctx->cons_managed_path == NULL) { + rc = DYAD_RC_BADMANAGEDPATH; + goto consume_close; + } + + if (ctx->relative_to_managed_path && (strlen(fname) > 0ul) && + (strncmp(fname, DYAD_PATH_DELIM, ctx->delim_len) != + 0)) { // fname is a relative path that is relative to the + // cons_managed_path + memcpy(upath, fname, strlen(fname)); + } else if (!cmp_canonical_path_prefix(ctx, false, fname, upath, PATH_MAX)) { + // Extract the path to the file specified by fname relative to the + // consumer-managed path + // This relative path will be stored in upath + // DYAD_LOG_TRACE (ctx, "%s is not in the Consumer's managed path\n", + // fname); + rc = DYAD_RC_OK; + goto consume_close; + } + ctx->reenter = false; + + lock_fd = open(fname, O_RDWR | O_CREAT, 0666); + if (lock_fd == -1) { + // This could be a system file on which users have no write permission + DYAD_LOG_ERROR(ctx, "Cannot create file (%s) for dyad_consume!\n", fname); + rc = DYAD_RC_BADFIO; + goto consume_close; + } + rc = dyad_excl_flock(ctx, lock_fd, &exclusive_lock); + if (DYAD_IS_ERROR(rc)) { + dyad_release_flock(ctx, lock_fd, &exclusive_lock); + goto consume_done; + } + file_size = get_file_size(lock_fd); + if (ctx->shared_storage) { + dyad_release_flock(ctx, lock_fd, &exclusive_lock); + if (!ctx->use_fs_locks || file_size <= 0) { + // as file size was zero that means consumer won the lock first so has to + // wait for kvs. or we cannot use file lock based synchronization as it + // does not work with the files managed by c++ fstream. + rc = dyad_fetch_metadata(ctx, fname, upath, &mdata); + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(ctx, + "dyad_fetch_metadata failed fore shared storage!\n"); + goto consume_done; + } + } + } else { + if (file_size <= 0) { + DYAD_LOG_INFO(ctx, + "[node %u rank %u pid %d] File (%s with lock_fd %d) is not " + "fetched yet", + ctx->node_idx, ctx->rank, ctx->pid, fname, lock_fd); + // Call dyad_fetch to get (and possibly wait on) + // data from the Flux KVS + rc = dyad_fetch_metadata(ctx, fname, upath, &mdata); + // If an error occured in dyad_fetch_metadata, log an error + // and return the corresponding DYAD return code + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(ctx, "dyad_fetch_metadata failed!\n"); + dyad_release_flock(ctx, lock_fd, &exclusive_lock); + goto consume_done; + } + // If dyad_fetch_metadata was successful, but mdata is still NULL, + // then we need to skip data transfer. + // This is either because producer and consumer share storage + // or because the file is not on the managed directory. + if (mdata == NULL) { + DYAD_LOG_INFO(ctx, "File '%s' is local!\n", fname); + rc = DYAD_RC_OK; + dyad_release_flock(ctx, lock_fd, &exclusive_lock); + goto consume_done; + } + + // Call dyad_get_data to dispatch a RPC to the producer's Flux broker + // and retrieve the data associated with the file + rc = dyad_get_data(ctx, mdata, &file_data, &data_len); + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(ctx, "dyad_get_data failed!\n"); + dyad_release_flock(ctx, lock_fd, &exclusive_lock); + goto consume_done; + } + DYAD_C_FUNCTION_UPDATE_INT("data_len", data_len); + io_fd = open(fname, O_WRONLY); + DYAD_C_FUNCTION_UPDATE_INT("io_fd", io_fd); + if (io_fd == -1) { + DYAD_LOG_ERROR( + ctx, "Cannot open file (%s) in write mode for dyad_consume!\n", + fname); + rc = DYAD_RC_BADFIO; + goto consume_close; + } + // Call dyad_pull to fetch the data from the producer's + // Flux broker + rc = dyad_cons_store(ctx, mdata, io_fd, data_len, file_data); + // Regardless if there was an error in dyad_pull, + // free the KVS response object + if (mdata != NULL) { + dyad_free_metadata(&mdata); + } + if (close(io_fd) != 0) { + rc = DYAD_RC_BADFIO; + dyad_release_flock(ctx, lock_fd, &exclusive_lock); + goto consume_done; + } + // If an error occured in dyad_pull, log it + // and return the corresponding DYAD return code + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(ctx, "dyad_cons_store failed!\n"); + dyad_release_flock(ctx, lock_fd, &exclusive_lock); + goto consume_done; + }; + } + dyad_release_flock(ctx, lock_fd, &exclusive_lock); + } + DYAD_C_FUNCTION_UPDATE_INT("file_size", file_size); +consume_done:; + if (close(lock_fd) != 0) { + rc = DYAD_RC_BADFIO; + } + if (file_data != NULL) { + ctx->dtl_handle->return_buffer(ctx, (void **)&file_data); + } + // Set reenter to true to allow additional intercepting +consume_close:; + ctx->reenter = true; + DYAD_C_FUNCTION_END(); + return rc; +} + +dyad_rc_t dyad_consume_w_metadata(dyad_ctx_t *restrict ctx, const char *fname, + const dyad_metadata_t *restrict mdata) { + DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_UPDATE_STR("fname", fname); + dyad_rc_t rc = DYAD_RC_OK; + int lock_fd = -1, io_fd = -1; + ssize_t file_size = -1; + char *file_data = NULL; + size_t data_len = 0ul; + struct flock exclusive_lock; + // If the context is not defined, then it is not valid. + // So, return DYAD_NOCTX + if (!ctx || !ctx->h) { + rc = DYAD_RC_NOCTX; + goto consume_close; + } + // If dyad_get_metadata was successful, but mdata is still NULL, + // then we need to skip data transfer. + if (mdata == NULL) { + DYAD_LOG_INFO(ctx, "File '%s' is local!\n", fname); + rc = DYAD_RC_OK; + goto consume_close; + } + // If the consumer-managed path is NULL or empty, then the context is not + // valid for a consumer operation. So, return DYAD_BADMANAGEDPATH + if (ctx->cons_managed_path == NULL) { + rc = DYAD_RC_BADMANAGEDPATH; + goto consume_close; + } + // Set reenter to false to avoid recursively performing DYAD operations + ctx->reenter = false; + lock_fd = open(fname, O_RDWR | O_CREAT, 0666); + DYAD_C_FUNCTION_UPDATE_INT("lock_fd", lock_fd); + if (lock_fd == -1) { + DYAD_LOG_ERROR( + ctx, "Cannot create file (%s) for dyad_consume_w_metadata!\n", fname); + rc = DYAD_RC_BADFIO; + goto consume_close; + } + rc = dyad_excl_flock(ctx, lock_fd, &exclusive_lock); + if (DYAD_IS_ERROR(rc)) { + dyad_release_flock(ctx, lock_fd, &exclusive_lock); + goto consume_close; + } + if ((file_size = get_file_size(lock_fd)) <= 0) { + DYAD_LOG_INFO( + ctx, "[node %u rank %u pid %d] File (%s with fd %d) is not fetched yet", + ctx->node_idx, ctx->rank, ctx->pid, fname, lock_fd); + + // Call dyad_get_data to dispatch a RPC to the producer's Flux broker + // and retrieve the data associated with the file + rc = dyad_get_data(ctx, mdata, &file_data, &data_len); + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(ctx, "dyad_get_data failed!\n"); + dyad_release_flock(ctx, lock_fd, &exclusive_lock); + goto consume_done; + } + DYAD_C_FUNCTION_UPDATE_INT("data_len", data_len); + io_fd = open(fname, O_WRONLY); + DYAD_C_FUNCTION_UPDATE_INT("io_fd", io_fd); + if (io_fd == -1) { + DYAD_LOG_ERROR(ctx, + "Cannot open file (%s) in write mode for dyad_consume!\n", + fname); + rc = DYAD_RC_BADFIO; + goto consume_close; + } + // Call dyad_pull to fetch the data from the producer's + // Flux broker + rc = dyad_cons_store(ctx, mdata, io_fd, data_len, file_data); + + if (close(io_fd) != 0) { + rc = DYAD_RC_BADFIO; + dyad_release_flock(ctx, lock_fd, &exclusive_lock); + goto consume_done; + } + // If an error occured in dyad_pull, log it + // and return the corresponding DYAD return code + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(ctx, "dyad_cons_store failed!\n"); + dyad_release_flock(ctx, io_fd, &exclusive_lock); + goto consume_done; + }; + } + dyad_release_flock(ctx, lock_fd, &exclusive_lock); + DYAD_C_FUNCTION_UPDATE_INT("file_size", file_size); + + if (close(lock_fd) != 0) { + rc = DYAD_RC_BADFIO; + goto consume_done; + } + rc = DYAD_RC_OK; +consume_done:; + if (file_data != NULL) { + ctx->dtl_handle->return_buffer(ctx, (void **)&file_data); + } +consume_close:; + // Set reenter to true to allow additional intercepting + ctx->reenter = true; + DYAD_C_FUNCTION_END(); + return rc; +} + +#if DYAD_SYNC_DIR +int dyad_sync_directory(dyad_ctx_t *restrict ctx, const char *restrict path) { + DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_UPDATE_STR("path", path); + // Flush new directory entry https://lwn.net/Articles/457671/ + char path_copy[PATH_MAX + 1] = {'\0'}; + int odir_fd = -1; + char *odir = NULL; + bool reenter = false; + int rc = 0; + memset(path_copy, 0, PATH_MAX + 1); + + strncpy(path_copy, path, PATH_MAX); + odir = dirname(path_copy); + + reenter = ctx->reenter; // backup ctx->reenter + if (ctx != NULL) + ctx->reenter = false; + + if ((odir_fd = open(odir, O_RDONLY)) < 0) { + IPRINTF(ctx, "Cannot open the directory \"%s\"\n", odir); + rc = -1; + } else { + if (fsync(odir_fd) < 0) { + IPRINTF(ctx, "Cannot flush the directory \"%s\"\n", odir); + rc = -1; + } + if (close(odir_fd) < 0) { + IPRINTF(ctx, "Cannot close the directory \"%s\"\n", odir); + rc = -1; + } + } + if (ctx != NULL) + ctx->reenter = reenter; + DYAD_C_FUNCTION_END(); + return rc; +} +#endif diff --git a/src/dyad/client/dyad_client_int.h b/src/dyad/client/dyad_client_int.h new file mode 100644 index 00000000..c8fc8288 --- /dev/null +++ b/src/dyad/client/dyad_client_int.h @@ -0,0 +1,61 @@ +#ifndef DYAD_CORE_DYAD_CORE_INT_H +#define DYAD_CORE_DYAD_CORE_INT_H + +// clang-format off +#include +#include +// clang-format on + +// Debug message +#ifndef DPRINTF +#if VA_OPT_SUPPORTED +#define DPRINTF(curr_dyad_ctx, fmt, ...) \ + do { \ + if ((curr_dyad_ctx) && (curr_dyad_ctx)->debug) \ + fprintf(stderr, (fmt)__VA_OPT__(, ) __VA_ARGS__); \ + } while (0) +#else +#define DPRINTF(curr_dyad_ctx, fmt, ...) \ + do { \ + if ((curr_dyad_ctx) && (curr_dyad_ctx)->debug) \ + fprintf(stderr, (fmt), ##__VA_ARGS__); \ + } while (0) +#endif +#endif // DPRINTF + +#define TIME_DIFF(Tstart, Tend) \ + ((double)(1000000000L * ((Tend).tv_sec - (Tstart).tv_sec) + (Tend).tv_nsec - \ + (Tstart).tv_nsec) / \ + 1000000000L) + +// Detailed information message that can be omitted +#if DYAD_FULL_DEBUG +#define IPRINTF DPRINTF +#define IPRINTF_DEFINED +#else +#define IPRINTF(curr_dyad_ctx, fmt, ...) +#endif // DYAD_FULL_DEBUG + +DYAD_DLL_EXPORTED int gen_path_key(const char *str, char *path_key, + const size_t len, const uint32_t depth, + const uint32_t width); + +/** + * Private Function definitions + */ +DYAD_DLL_EXPORTED dyad_rc_t dyad_get_data(const dyad_ctx_t *ctx, + const dyad_metadata_t *mdata, + char **file_data, size_t *file_len); +DYAD_DLL_EXPORTED dyad_rc_t dyad_commit(dyad_ctx_t *ctx, const char *fname); + +DYAD_DLL_EXPORTED dyad_rc_t dyad_kvs_read(const dyad_ctx_t *ctx, + const char *topic, const char *upath, + bool should_wait, + dyad_metadata_t **mdata); + +#if DYAD_SYNC_DIR +DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED int dyad_sync_directory(dyad_ctx_t *ctx, + const char *path); +#endif + +#endif /* DYAD_CORE_DYAD_CORE_INT_H */ \ No newline at end of file diff --git a/src/dyad/common/dyad_logging.h b/src/dyad/common/dyad_logging.h index 0b395662..31204fad 100644 --- a/src/dyad/common/dyad_logging.h +++ b/src/dyad/common/dyad_logging.h @@ -6,7 +6,7 @@ #error "no config" #endif -#include +#include #include #include @@ -14,18 +14,18 @@ extern "C" { #endif -//#define DYAD_NOOP_MACRO do {} while (0) +// #define DYAD_NOOP_MACRO do {} while (0) //============================================================================= #ifdef DYAD_LOGGER_NO_LOG //============================================================================= -#define DYAD_LOGGER_INIT() DYAD_NOOP_MACRO -#define DYAD_LOG_STDERR(...) DYAD_NOOP_MACRO -#define DYAD_LOG_STDOUT(...) DYAD_NOOP_MACRO -#define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO -#define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO -#define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO -#define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO +#define DYAD_LOGGER_INIT() DYAD_NOOP_MACRO +#define DYAD_LOG_STDERR(...) DYAD_NOOP_MACRO +#define DYAD_LOG_STDOUT(...) DYAD_NOOP_MACRO +#define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO +#define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO +#define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO +#define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO #define DYAD_LOG_STDOUT_REDIRECT(fpath) DYAD_NOOP_MACRO #define DYAD_LOG_STDERR_REDIRECT(fpath) DYAD_NOOP_MACRO //============================================================================= @@ -35,101 +35,112 @@ extern "C" { #define DYAD_LOG_STDOUT(...) fprintf(stdout, __VA_ARGS__); #ifdef DYAD_LOGGER_FLUX // FLUX ----------------------------------------------- - #define DYAD_LOGGER_INIT() ; - #define DYAD_LOG_STDOUT_REDIRECT(fpath) freopen ((fpath), "a+", stdout); - #define DYAD_LOG_STDERR_REDIRECT(fpath) freopen ((fpath), "a+", stderr); - - #ifdef DYAD_UTIL_LOGGER - #ifdef DYAD_LOGGER_LEVEL_DEBUG - #define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_LOG_STDERR(__VA_ARGS__) - #else - #define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - - #ifdef DYAD_LOGGER_LEVEL_INFO - #define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_LOG_STDOUT(__VA_ARGS__) - #else - #define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - - #ifdef DYAD_LOGGER_LEVEL_WARN - #define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_LOG_STDOUT(__VA_ARGS__) - #else - #define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - - #ifdef DYAD_LOGGER_LEVEL_ERROR - #define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_LOG_STDERR(__VA_ARGS__) - #else - #define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - #else // DYAD_UTIL_LOGGER - #include - #ifdef DYAD_LOGGER_LEVEL_DEBUG - #define DYAD_LOG_DEBUG(dyad_ctx, ...) flux_log ((flux_t*)(((dyad_ctx_t*) dyad_ctx)->h), LOG_DEBUG, __VA_ARGS__); - #else - #define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - - #ifdef DYAD_LOGGER_LEVEL_INFO - #define DYAD_LOG_INFO(dyad_ctx, ...) flux_log ((flux_t*)(((dyad_ctx_t*) dyad_ctx)->h), LOG_INFO, __VA_ARGS__); - #else - #define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - - #ifdef DYAD_LOGGER_LEVEL_WARN - #define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO - #else - #define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - - #ifdef DYAD_LOGGER_LEVEL_ERROR - #define DYAD_LOG_ERROR(dyad_ctx, ...) flux_log_error ((flux_t*)(((dyad_ctx_t*) dyad_ctx)->h), __VA_ARGS__); - #else - #define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - #endif // DYAD_UTIL_LOGGER +#define DYAD_LOGGER_INIT() ; +#define DYAD_LOG_STDOUT_REDIRECT(fpath) freopen((fpath), "a+", stdout); +#define DYAD_LOG_STDERR_REDIRECT(fpath) freopen((fpath), "a+", stderr); + +#ifdef DYAD_UTIL_LOGGER +#ifdef DYAD_LOGGER_LEVEL_DEBUG +#define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_LOG_STDERR(__VA_ARGS__) +#else +#define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif + +#ifdef DYAD_LOGGER_LEVEL_INFO +#define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_LOG_STDOUT(__VA_ARGS__) +#else +#define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif + +#ifdef DYAD_LOGGER_LEVEL_WARN +#define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_LOG_STDOUT(__VA_ARGS__) +#else +#define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif + +#ifdef DYAD_LOGGER_LEVEL_ERROR +#define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_LOG_STDERR(__VA_ARGS__) +#else +#define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif +#else // DYAD_UTIL_LOGGER +#include +#ifdef DYAD_LOGGER_LEVEL_DEBUG +#define DYAD_LOG_DEBUG(dyad_ctx, ...) \ + flux_log((flux_t *)(((dyad_ctx_t *)dyad_ctx)->h), LOG_DEBUG, __VA_ARGS__); +#else +#define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif + +#ifdef DYAD_LOGGER_LEVEL_INFO +#define DYAD_LOG_INFO(dyad_ctx, ...) \ + flux_log((flux_t *)(((dyad_ctx_t *)dyad_ctx)->h), LOG_INFO, __VA_ARGS__); +#else +#define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif + +#ifdef DYAD_LOGGER_LEVEL_WARN +#define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO +#else +#define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif + +#ifdef DYAD_LOGGER_LEVEL_ERROR +#define DYAD_LOG_ERROR(dyad_ctx, ...) \ + flux_log_error((flux_t *)(((dyad_ctx_t *)dyad_ctx)->h), __VA_ARGS__); +#else +#define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif +#endif // DYAD_UTIL_LOGGER #elif defined(DYAD_LOGGER_CPP_LOGGER) // CPP_LOGGER --------------------------- - #include - - #define DYAD_LOGGER_NAME "DYAD" - - #ifdef DYAD_LOGGER_LEVEL_DEBUG - #define DYAD_LOGGER_INIT() cpp_logger_clog_level(CPP_LOGGER_DEBUG, DYAD_LOGGER_NAME); - #elif defined(DYAD_LOGGER_LEVEL_INFO) - #define DYAD_LOGGER_INIT() cpp_logger_clog_level(CPP_LOGGER_INFO, DYAD_LOGGER_NAME); - #elif defined(DYAD_LOGGER_LEVEL_WARN) - #define DYAD_LOGGER_INIT() cpp_logger_clog_level(CPP_LOGGER_WARN, DYAD_LOGGER_NAME); - #else - #define DYAD_LOGGER_INIT() cpp_logger_clog_level(CPP_LOGGER_ERROR, DYAD_LOGGER_NAME); - #endif - - #define DYAD_LOG_STDOUT_REDIRECT(fpath) freopen ((fpath), "a+", stdout); - #define DYAD_LOG_STDERR_REDIRECT(fpath) freopen ((fpath), "a+", stderr); - - #ifdef DYAD_LOGGER_LEVEL_DEBUG - #define DYAD_LOG_DEBUG(dyad_ctx, ...) cpp_logger_clog(CPP_LOGGER_DEBUG, DYAD_LOGGER_NAME, __VA_ARGS__); - #else - #define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - - #ifdef DYAD_LOGGER_LEVEL_INFO - #define DYAD_LOG_INFO(dyad_ctx, ...) cpp_logger_clog(CPP_LOGGER_INFO, DYAD_LOGGER_NAME, __VA_ARGS__); - #else - #define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - - #ifdef DYAD_LOGGER_LEVEL_WARN - #define DYAD_LOG_WARN(dyad_ctx, ...) cpp_logger_clog(CPP_LOGGER_WARN, DYAD_LOGGER_NAME, __VA_ARGS__); - #else - #define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif - - #ifdef DYAD_LOGGER_LEVEL_ERROR - #define DYAD_LOG_ERROR(dyad_ctx, ...) cpp_logger_clog(CPP_LOGGER_ERROR, DYAD_LOGGER_NAME, __VA_ARGS__); - #else - #define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO - #endif +#include + +#define DYAD_LOGGER_NAME "DYAD" + +#ifdef DYAD_LOGGER_LEVEL_DEBUG +#define DYAD_LOGGER_INIT() \ + cpp_logger_clog_level(CPP_LOGGER_DEBUG, DYAD_LOGGER_NAME); +#elif defined(DYAD_LOGGER_LEVEL_INFO) +#define DYAD_LOGGER_INIT() \ + cpp_logger_clog_level(CPP_LOGGER_INFO, DYAD_LOGGER_NAME); +#elif defined(DYAD_LOGGER_LEVEL_WARN) +#define DYAD_LOGGER_INIT() \ + cpp_logger_clog_level(CPP_LOGGER_WARN, DYAD_LOGGER_NAME); +#else +#define DYAD_LOGGER_INIT() \ + cpp_logger_clog_level(CPP_LOGGER_ERROR, DYAD_LOGGER_NAME); +#endif + +#define DYAD_LOG_STDOUT_REDIRECT(fpath) freopen((fpath), "a+", stdout); +#define DYAD_LOG_STDERR_REDIRECT(fpath) freopen((fpath), "a+", stderr); + +#ifdef DYAD_LOGGER_LEVEL_DEBUG +#define DYAD_LOG_DEBUG(dyad_ctx, ...) \ + cpp_logger_clog(CPP_LOGGER_DEBUG, DYAD_LOGGER_NAME, __VA_ARGS__); +#else +#define DYAD_LOG_DEBUG(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif + +#ifdef DYAD_LOGGER_LEVEL_INFO +#define DYAD_LOG_INFO(dyad_ctx, ...) \ + cpp_logger_clog(CPP_LOGGER_INFO, DYAD_LOGGER_NAME, __VA_ARGS__); +#else +#define DYAD_LOG_INFO(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif + +#ifdef DYAD_LOGGER_LEVEL_WARN +#define DYAD_LOG_WARN(dyad_ctx, ...) \ + cpp_logger_clog(CPP_LOGGER_WARN, DYAD_LOGGER_NAME, __VA_ARGS__); +#else +#define DYAD_LOG_WARN(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif + +#ifdef DYAD_LOGGER_LEVEL_ERROR +#define DYAD_LOG_ERROR(dyad_ctx, ...) \ + cpp_logger_clog(CPP_LOGGER_ERROR, DYAD_LOGGER_NAME, __VA_ARGS__); +#else +#define DYAD_LOG_ERROR(dyad_ctx, ...) DYAD_NOOP_MACRO +#endif #endif // DYAD_LOGGER_FLUX ---------------------------------------------------- //============================================================================= #endif // DYAD_LOGGER_NO_LOG diff --git a/src/dyad/common/dyad_structures.h b/src/dyad/common/dyad_structures.h deleted file mode 100644 index 0f603bc1..00000000 --- a/src/dyad/common/dyad_structures.h +++ /dev/null @@ -1,69 +0,0 @@ -#ifndef DYAD_COMMON_STRUCTURES_H -#define DYAD_COMMON_STRUCTURES_H - -#if defined(DYAD_HAS_CONFIG) -#include -#else -#error "no config" -#endif - -#ifdef __cplusplus -#include -#else -#include -#include -#endif - - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * @struct dyad_ctx - */ -struct dyad_ctx { - // Internal - void* h; // the Flux handle for DYAD - struct dyad_dtl* dtl_handle; // Opaque handle to DTL info - const char* fname; // Used to track which file is getting processed. - bool use_fs_locks; // Used to track if fs locks should be used. - char* prod_real_path; // producer managed real path - char* cons_real_path; // consumer managed real path - uint32_t prod_managed_len; // length of producer path managed by DYAD - uint32_t cons_managed_len; // length of consumer path managed by DYAD - uint32_t prod_real_len; // length of producer managed real path - uint32_t cons_real_len; // length of consumer managed real path - uint32_t prod_managed_hash; // hash of producer path managed by DYAD - uint32_t cons_managed_hash; // hash of consumer path managed by DYAD - uint32_t prod_real_hash; // hash of producer managed real path - uint32_t cons_real_hash; // hash of consumer managed real path - uint32_t delim_len; // length of path delimiter - // User Facing - bool debug; // if true, perform debug logging - bool check; // if true, perform some check logging - bool reenter; // if false, do not recursively enter DYAD - bool initialized; // if true, DYAD is initialized - bool shared_storage; // if true, the managed path is shared - bool async_publish; // Enable asynchronous publish by producer - bool fsync_write; // Apply fsync after write by producer - unsigned int key_depth; // Depth of bins for the Flux KVS - unsigned int key_bins; // Number of bins for the Flux KVS - uint32_t rank; // Flux rank for DYAD - uint32_t service_mux; // Number of Flux brokers sharing node-local storage - uint32_t node_idx; // Index of the node hosting broker(s) - int pid; // unix process id, obtained by getpid() - char* kvs_namespace; // Flux KVS namespace for DYAD - char* prod_managed_path; // producer path managed by DYAD - char* cons_managed_path; // consumer path managed by DYAD - bool relative_to_managed_path; // relative path is relative to the managed path -}; -typedef struct dyad_ctx dyad_ctx_t; -typedef void* ucx_ep_cache_h; - - -#ifdef __cplusplus -} -#endif - -#endif // DYAD_COMMON_STRUCTURES_H diff --git a/src/dyad/common/dyad_structures_int.h b/src/dyad/common/dyad_structures_int.h new file mode 100644 index 00000000..d6eb8f73 --- /dev/null +++ b/src/dyad/common/dyad_structures_int.h @@ -0,0 +1,69 @@ +#ifndef DYAD_COMMON_STRUCTURES_INT_H +#define DYAD_COMMON_STRUCTURES_INT_H + +#if defined(DYAD_HAS_CONFIG) +#include +#else +#error "no config" +#endif + +#include + +#ifdef __cplusplus +#include +#else +#include +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @struct dyad_ctx + */ +struct dyad_ctx { + // Internal + void *h; // the Flux handle for DYAD + struct dyad_dtl *dtl_handle; // Opaque handle to DTL info + const char *fname; // Used to track which file is getting processed. + bool use_fs_locks; // Used to track if fs locks should be used. + char *prod_real_path; // producer managed real path + char *cons_real_path; // consumer managed real path + uint32_t prod_managed_len; // length of producer path managed by DYAD + uint32_t cons_managed_len; // length of consumer path managed by DYAD + uint32_t prod_real_len; // length of producer managed real path + uint32_t cons_real_len; // length of consumer managed real path + uint32_t prod_managed_hash; // hash of producer path managed by DYAD + uint32_t cons_managed_hash; // hash of consumer path managed by DYAD + uint32_t prod_real_hash; // hash of producer managed real path + uint32_t cons_real_hash; // hash of consumer managed real path + uint32_t delim_len; // length of path delimiter + // User Facing + bool debug; // if true, perform debug logging + bool check; // if true, perform some check logging + bool reenter; // if false, do not recursively enter DYAD + bool initialized; // if true, DYAD is initialized + bool shared_storage; // if true, the managed path is shared + bool async_publish; // Enable asynchronous publish by producer + bool fsync_write; // Apply fsync after write by producer + unsigned int key_depth; // Depth of bins for the Flux KVS + unsigned int key_bins; // Number of bins for the Flux KVS + uint32_t rank; // Flux rank for DYAD + uint32_t service_mux; // Number of Flux brokers sharing node-local storage + uint32_t node_idx; // Index of the node hosting broker(s) + int pid; // unix process id, obtained by getpid() + char *kvs_namespace; // Flux KVS namespace for DYAD + char *prod_managed_path; // producer path managed by DYAD + char *cons_managed_path; // consumer path managed by DYAD + bool + relative_to_managed_path; // relative path is relative to the managed path +}; +typedef void *ucx_ep_cache_h; + +#ifdef __cplusplus +} +#endif + +#endif // DYAD_COMMON_STRUCTURES_INT_H diff --git a/src/dyad/core/CMakeLists.txt b/src/dyad/core/CMakeLists.txt index 5c6e6338..9ff43aa8 100644 --- a/src/dyad/core/CMakeLists.txt +++ b/src/dyad/core/CMakeLists.txt @@ -1,68 +1,35 @@ -set(DYAD_CORE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/dyad_core.c) -set(DYAD_CORE_PRIVATE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_rc.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_dtl.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_envs.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_logging.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_profiler.h - ${CMAKE_CURRENT_SOURCE_DIR}/../dtl/dyad_dtl_api.h - ${CMAKE_CURRENT_SOURCE_DIR}/../utils/utils.h - ${CMAKE_CURRENT_SOURCE_DIR}/../utils/murmur3.h - ${CMAKE_CURRENT_SOURCE_DIR}/dyad_ctx.h - ${CMAKE_CURRENT_SOURCE_DIR}/dyad_core.h) -set(DYAD_CORE_PUBLIC_HEADERS) - set(DYAD_CTX_SRC ${CMAKE_CURRENT_SOURCE_DIR}/dyad_ctx.c) -set(DYAD_CTX_PRIVATE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/dyad_ctx.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_rc.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_dtl.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_envs.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_logging.h +set(DYAD_CTX_PRIVATE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_logging.h ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_profiler.h) - -add_library(${PROJECT_NAME}_core SHARED ${DYAD_CORE_SRC} - ${DYAD_CORE_PUBLIC_HEADERS} ${DYAD_CORE_PRIVATE_HEADERS}) -set_target_properties(${PROJECT_NAME}_core PROPERTIES CMAKE_INSTALL_RPATH - "${CMAKE_INSTALL_PREFIX}/${DYAD_LIBDIR}") -target_link_libraries(${PROJECT_NAME}_core PRIVATE Jansson::Jansson flux::core) -target_link_libraries(${PROJECT_NAME}_core PRIVATE ${PROJECT_NAME}_ctx ${PROJECT_NAME}_utils - ${PROJECT_NAME}_murmur3 ${PROJECT_NAME}_dtl) - -target_compile_definitions(${PROJECT_NAME}_core PUBLIC BUILDING_DYAD=1) -target_compile_definitions(${PROJECT_NAME}_core PUBLIC DYAD_HAS_CONFIG) -target_include_directories(${PROJECT_NAME}_core PUBLIC - $ # will be used for generated code - $) # wil be used for sub projects -target_include_directories(${PROJECT_NAME}_core SYSTEM PRIVATE ${JANSSON_INCLUDE_DIRS}) -target_include_directories(${PROJECT_NAME}_core SYSTEM PRIVATE ${FluxCore_INCLUDE_DIRS}) +set(DYAD_CTX_PUBLIC_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/core/dyad_ctx.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/common/dyad_rc.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/common/dyad_dtl.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/common/dyad_envs.h) add_library(${PROJECT_NAME}_ctx SHARED ${DYAD_CTX_SRC} - ${DYAD_CTX_PRIVATE_HEADERS}) + ${DYAD_CTX_PUBLIC_HEADERS} ${DYAD_CTX_PRIVATE_HEADERS}) target_compile_definitions(${PROJECT_NAME}_ctx PUBLIC BUILDING_DYAD=1) target_compile_definitions(${PROJECT_NAME}_ctx PUBLIC DYAD_HAS_CONFIG) +target_include_directories(${PROJECT_NAME}_ctx PUBLIC + $ + $ + $) target_include_directories(${PROJECT_NAME}_ctx SYSTEM PRIVATE ${JANSSON_INCLUDE_DIRS}) target_include_directories(${PROJECT_NAME}_ctx SYSTEM PRIVATE ${FluxCore_INCLUDE_DIRS}) - target_link_libraries(${PROJECT_NAME}_ctx PRIVATE ${PROJECT_NAME}_dtl ${PROJECT_NAME}_utils) +target_link_libraries(${PROJECT_NAME}_ctx PRIVATE ${PROJECT_NAME}_dtl ${PROJECT_NAME}_utils) if (TARGET DYAD_C_FLAGS_werror) - target_link_libraries(${PROJECT_NAME}_core PRIVATE DYAD_C_FLAGS_werror) target_link_libraries(${PROJECT_NAME}_ctx PRIVATE DYAD_C_FLAGS_werror) endif () -if(DYAD_PROFILER STREQUAL "PERFFLOW_ASPECT") - target_link_libraries(${PROJECT_NAME}_core PRIVATE perfflowaspect::perfflowaspect) - target_include_directories(${PROJECT_NAME} SYSTEM PRIVATE ${perfflowaspect_INCLUDE_DIRS}) - target_compile_definitions(${PROJECT_NAME}_core PRIVATE DYAD_PERFFLOW=1) -endif() -if(DYAD_PROFILER STREQUAL "DFTRACER") - target_link_libraries(${PROJECT_NAME}_core PRIVATE ${DFTRACER_LIBRARIES}) -endif() install( - TARGETS ${PROJECT_NAME}_core ${PROJECT_NAME}_ctx + TARGETS ${PROJECT_NAME}_ctx EXPORT ${DYAD_EXPORTED_TARGETS} - LIBRARY DESTINATION ${DYAD_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${DYAD_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${DYAD_INSTALL_BIN_DIR} + LIBRARY DESTINATION ${DYAD_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${DYAD_INSTALL_LIBDIR} + RUNTIME DESTINATION ${DYAD_INSTALL_BINDIR} ) -if(NOT "${DYAD_CORE_PUBLIC_HEADERS}" STREQUAL "") - dyad_install_headers("${DYAD_CORE_PUBLIC_HEADERS}" ${CMAKE_CURRENT_SOURCE_DIR}) -endif() + +if(NOT "${DYAD_CTX_PUBLIC_HEADERS}" STREQUAL "") + dyad_install_headers("${DYAD_CTX_PUBLIC_HEADERS}" ${CMAKE_CURRENT_SOURCE_DIR}) +endif() \ No newline at end of file diff --git a/src/dyad/core/dyad_core.c b/src/dyad/core/dyad_core.c deleted file mode 100644 index e51c153a..00000000 --- a/src/dyad/core/dyad_core.c +++ /dev/null @@ -1,955 +0,0 @@ -#if defined(DYAD_HAS_CONFIG) -#include -#else -#error "no config" -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __cplusplus -#include -#include -#else -#include -#include -#include -#endif - - - - -DYAD_DLL_EXPORTED int gen_path_key (const char* restrict str, - char* restrict path_key, - const size_t len, - const uint32_t depth, - const uint32_t width) -{ - DYAD_C_FUNCTION_START(); - static const uint32_t seeds[10] = - {104677u, 104681u, 104683u, 104693u, 104701u, 104707u, 104711u, 104717u, 104723u, 104729u}; - - uint32_t seed = 57u; - uint32_t hash[4] = {0u}; // Output for the hash - size_t cx = 0ul; - int n = 0; - if (str == NULL || path_key == NULL || len == 0ul) { - DYAD_C_FUNCTION_END(); - return -1; - } - size_t str_len = strlen (str); - if (str_len == 0ul) { - DYAD_C_FUNCTION_END(); - return -1; - } - const char* str_long = str; - - path_key[0] = '\0'; - - // Just append the string so that it can be as large as 128 bytes. - if (str_len < 128ul) { - char buf[256] = {'\0'}; - memcpy (buf, str, str_len); - memset (buf + str_len, '@', 128ul - str_len); - buf[128u] = '\0'; - str_len = 128ul; - str_long = buf; - } - - for (uint32_t d = 0u; d < depth; d++) { - seed += seeds[d % 10]; - MurmurHash3_x64_128 (str_long, str_len, seed, hash); - uint32_t bin = (hash[0] ^ hash[1] ^ hash[2] ^ hash[3]) % width; - n = snprintf (path_key + cx, len - cx, "%x.", bin); - cx += n; - if (cx >= len || n < 0) { - DYAD_C_FUNCTION_END(); - return -1; - } - } - n = snprintf (path_key + cx, len - cx, "%s", str); - // FIXME: cx + n >= len fails for str_len > 256 - if (n < 0) { - DYAD_C_FUNCTION_END(); - return -1; - } - DYAD_C_FUNCTION_UPDATE_STR ("path_key", path_key); - DYAD_C_FUNCTION_END(); - return 0; -} - -static void future_cleanup_cb (flux_future_t *f, void *arg) -{ - if (flux_future_get (f, NULL) < 0) { - DYAD_LOG_STDERR ("future_cleanup: future error detected with.%s", ""); - } - flux_future_destroy (f); -} - -DYAD_CORE_FUNC_MODS dyad_rc_t dyad_kvs_commit (const dyad_ctx_t* restrict ctx, - flux_kvs_txn_t* restrict txn) -{ - DYAD_C_FUNCTION_START(); - flux_future_t* f = NULL; - dyad_rc_t rc = DYAD_RC_OK; - DYAD_LOG_INFO (ctx, "Committing transaction to KVS"); - // Commit the transaction to the Flux KVS - f = flux_kvs_commit ((flux_t*) ctx->h, ctx->kvs_namespace, 0, txn); - // If the commit failed, log an error and return DYAD_BADCOMMIT - if (f == NULL) { - DYAD_LOG_ERROR (ctx, "Could not commit transaction to Flux KVS"); - rc = DYAD_RC_BADCOMMIT; - goto kvs_commit_region_finish; - } - if (ctx->async_publish) { - if (flux_future_then (f, -1, future_cleanup_cb, NULL) < 0) { - DYAD_LOG_ERROR (ctx, "Error with flux_future_then"); - } - } else { - // If the commit is pending, wait for it to complete - flux_future_wait_for (f, -1.0); - // Once the commit is complete, destroy the future and transaction - flux_future_destroy (f); - f = NULL; - } - rc = DYAD_RC_OK; -kvs_commit_region_finish:; - DYAD_C_FUNCTION_END(); - return rc; -} - -DYAD_CORE_FUNC_MODS dyad_rc_t publish_via_flux (const dyad_ctx_t* restrict ctx, - const char* restrict upath) -{ - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_STR ("fname", ctx->fname); - DYAD_C_FUNCTION_UPDATE_STR ("upath", upath); - dyad_rc_t rc = DYAD_RC_OK; - flux_kvs_txn_t* txn = NULL; - const size_t topic_len = PATH_MAX; - char topic[PATH_MAX + 1] = {'\0'}; - memset (topic, 0, topic_len + 1); - memset (topic, '\0', topic_len + 1); - // Generate the KVS key from the file path relative to - // the producer-managed directory - DYAD_LOG_INFO (ctx, "Generating KVS key from path (%s)", upath); - gen_path_key (upath, topic, topic_len, ctx->key_depth, ctx->key_bins); - // Crete and pack a Flux KVS transaction. - // The transaction will contain a single key-value pair - // with the previously generated key as the key and the - // producer's rank as the value - DYAD_LOG_INFO (ctx, "Creating KVS transaction under the key %s", topic); - txn = flux_kvs_txn_create (); - if (txn == NULL) { - DYAD_LOG_ERROR (ctx, "Could not create Flux KVS transaction"); - rc = DYAD_RC_FLUXFAIL; - goto publish_done; - } - if (flux_kvs_txn_pack (txn, 0, topic, "i", ctx->rank) < 0) { - DYAD_LOG_ERROR (ctx, "Could not pack Flux KVS transaction"); - rc = DYAD_RC_FLUXFAIL; - goto publish_done; - } - // Call dyad_kvs_commit to commit the transaction into the Flux KVS - rc = dyad_kvs_commit (ctx, txn); - // If dyad_kvs_commit failed, log an error and forward the return code - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, "dyad_kvs_commit failed!"); - goto publish_done; - } - rc = DYAD_RC_OK; -publish_done:; - if (txn != NULL) { - flux_kvs_txn_destroy (txn); - } - DYAD_C_FUNCTION_END(); - return rc; -} - -DYAD_DLL_EXPORTED dyad_rc_t dyad_commit (dyad_ctx_t* restrict ctx, const char* restrict fname) -{ - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_STR ("fname", ctx->fname); - dyad_rc_t rc = DYAD_RC_OK; - char upath[PATH_MAX+1] = {'\0'}; -#if 0 - if (fname == NULL || strlen (fname) > PATH_MAX) { - rc = DYAD_RC_SYSFAIL; - goto get_metadata_done; - } -#endif - // As this is a function called for DYAD producer, ctx->prod_managed_path - // must be a valid string (!NULL). ctx->delim_len is verified to be greater - // than 0 during initialization. - if (ctx->relative_to_managed_path && - //(strlen (fname) > 0ul) && // checked where get_path() was - (strncmp (fname, DYAD_PATH_DELIM, ctx->delim_len) != 0)) - { // fname is a relative path that is relative to the prod_managed_path - memcpy (upath, fname, strlen (fname)); - } else if (!cmp_canonical_path_prefix (ctx, true, fname, upath, PATH_MAX)) { - // Extract the path to the file specified by fname relative to the - // producer-managed path - // This relative path will be stored in upath - DYAD_LOG_INFO (ctx, "%s is not in the Producer's managed path", fname); - rc = DYAD_RC_OK; - goto commit_done; - } - DYAD_C_FUNCTION_UPDATE_STR ("upath", upath); - DYAD_LOG_INFO (ctx, "Obtained file path relative to producer directory: %s", upath); - // Call publish_via_flux to actually store information about the file into - // the Flux KVS - // Fence this call with reassignments of reenter so that, if intercepting - // file I/O API calls, we will not get stuck in infinite recursion - ctx->reenter = false; - rc = publish_via_flux (ctx, upath); - ctx->reenter = true; - -commit_done:; - // If "check" is set and the operation was successful, set the - // DYAD_CHECK_ENV environment variable to "ok" - if (rc == DYAD_RC_OK && (ctx && ctx->check)) { - setenv (DYAD_CHECK_ENV, "ok", 1); - } - DYAD_C_FUNCTION_END(); - return rc; -} - -static void print_mdata (const dyad_ctx_t* restrict ctx, - const dyad_metadata_t* restrict mdata) -{ - if (mdata == NULL) { - DYAD_LOG_INFO (ctx, "Cannot print a NULL metadata object!"); - } else { - DYAD_LOG_INFO (ctx, "Printing contents of DYAD Metadata object"); - DYAD_LOG_INFO (ctx, "fpath = %s", mdata->fpath); - DYAD_LOG_INFO (ctx, "owner_rank = %u", mdata->owner_rank); - } -} - -DYAD_DLL_EXPORTED dyad_rc_t dyad_kvs_read (const dyad_ctx_t* restrict ctx, - const char* restrict topic, - const char* restrict upath, - bool should_wait, - dyad_metadata_t** restrict mdata) -{ - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_STR ("upath", upath); - dyad_rc_t rc = DYAD_RC_OK; - int kvs_lookup_flags = 0; - flux_future_t* f = NULL; - if (mdata == NULL) { - DYAD_LOG_ERROR (ctx, "Metadata double pointer is NULL. " \ - "Cannot correctly create metadata object"); - rc = DYAD_RC_NOTFOUND; - goto kvs_read_end; - } - // Lookup information about the desired file (represented by kvs_topic) - // from the Flux KVS. If there is no information, wait for it to be - // made available - if (should_wait) - kvs_lookup_flags = FLUX_KVS_WAITCREATE; - DYAD_LOG_INFO (ctx, "Retrieving information from KVS under the key %s", topic); - f = flux_kvs_lookup ((flux_t*) ctx->h, ctx->kvs_namespace, kvs_lookup_flags, topic); - // If the KVS lookup failed, log an error and return DYAD_BADLOOKUP - if (f == NULL) { - DYAD_LOG_ERROR (ctx, "KVS lookup failed!\n"); - rc = DYAD_RC_NOTFOUND; - goto kvs_read_end; - } - // Extract the rank of the producer from the KVS response - DYAD_LOG_INFO (ctx, "Building metadata object from KVS entry\n"); - if (*mdata != NULL) { - DYAD_LOG_INFO (ctx, "Metadata object is already allocated. Skipping allocation"); - } else { - *mdata = (dyad_metadata_t*)malloc (sizeof (struct dyad_metadata)); - if (*mdata == NULL) { - DYAD_LOG_ERROR (ctx, "Cannot allocate memory for metadata object"); - rc = DYAD_RC_SYSFAIL; - goto kvs_read_end; - } - } - size_t upath_len = strlen (upath); - (*mdata)->fpath = (char*)malloc (upath_len + 1); - if ((*mdata)->fpath == NULL) { - DYAD_LOG_ERROR (ctx, "Cannot allocate memory for fpath in metadata object"); - rc = DYAD_RC_SYSFAIL; - goto kvs_read_end; - } - memset ((*mdata)->fpath, '\0', upath_len + 1); - memcpy ((*mdata)->fpath, upath, upath_len); - rc = flux_kvs_lookup_get_unpack (f, "i", &((*mdata)->owner_rank)); - // If the extraction did not work, log an error and return DYAD_BADFETCH - if (rc < 0) { - DYAD_LOG_ERROR (ctx, "Could not unpack owner's rank from KVS response\n"); - rc = DYAD_RC_BADMETADATA; - goto kvs_read_end; - } - DYAD_LOG_INFO (ctx, "Successfully created DYAD Metadata object"); - print_mdata (ctx, *mdata); - DYAD_C_FUNCTION_UPDATE_STR ("fpath", (*mdata)->fpath); - DYAD_C_FUNCTION_UPDATE_INT ("owner_rank", (*mdata)->owner_rank); - rc = DYAD_RC_OK; - -kvs_read_end:; - if (DYAD_IS_ERROR (rc) && mdata != NULL && *mdata != NULL) { - dyad_free_metadata (mdata); - } - if (f != NULL) { - flux_future_destroy (f); - f = NULL; - } - DYAD_C_FUNCTION_END(); - return rc; -} - - - -DYAD_CORE_FUNC_MODS dyad_rc_t dyad_fetch_metadata (const dyad_ctx_t* restrict ctx, - const char* restrict fname, - const char* restrict upath, - dyad_metadata_t** restrict mdata) -{ - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_STR ("fname", fname); - dyad_rc_t rc = DYAD_RC_OK; - const size_t topic_len = PATH_MAX; - char topic[PATH_MAX+1] = {'\0'}; - *mdata = NULL; -#if 0 - if (fname == NULL || upath == NULL || strlen (fname) == 0ul || strlen (upath) == 0ul) { - rc = DYAD_RC_BADFIO; - goto get_metadata_done; - } -#endif - // Set reenter to false to avoid recursively performing DYAD operations - DYAD_LOG_INFO (ctx, "Obtained file path relative to consumer directory: %s\n", upath); - DYAD_C_FUNCTION_UPDATE_STR ("upath", upath); - // Generate the KVS key from the file path relative to - // the consumer-managed directory - gen_path_key (upath, topic, topic_len, ctx->key_depth, ctx->key_bins); - DYAD_LOG_INFO (ctx, "Generated KVS key for consumer: %s\n", topic); - // Call dyad_kvs_read to retrieve infromation about the file - // from the Flux KVS - rc = dyad_kvs_read (ctx, topic, upath, true, mdata); - // If an error occured in dyad_kvs_read, log it and propagate the return - // code - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, "dyad_kvs_read failed!\n"); - goto fetch_done; - } - // There are two cases where we do not want to perform file transfer: - // 1. if the shared storage feature is enabled - // 2. if the producer and consumer share the same rank - // In either of these cases, skip the creation of the dyad_kvs_response_t - // object, and return DYAD_OK. This will cause the file transfer step to be - // skipped - DYAD_C_FUNCTION_UPDATE_INT ("owner_rank", (*mdata)->owner_rank); - DYAD_C_FUNCTION_UPDATE_INT ("node_idx", ctx->node_idx); - if (((*mdata)->owner_rank / ctx->service_mux) == ctx->node_idx) { - DYAD_LOG_INFO (ctx, \ - "Either shared-storage is indicated or the producer rank (%u) is the" \ - " same as the consumer rank (%u)", (*mdata)->owner_rank, ctx->rank); - if (mdata != NULL && *mdata != NULL) { - dyad_free_metadata (mdata); - } - rc = DYAD_RC_OK; - DYAD_C_FUNCTION_UPDATE_INT ("is_local", 1); - goto fetch_done; - } - DYAD_C_FUNCTION_UPDATE_INT ("is_local", 0); - rc = DYAD_RC_OK; - -fetch_done:; - DYAD_C_FUNCTION_END(); - return rc; -} - -DYAD_DLL_EXPORTED dyad_rc_t dyad_get_data (const dyad_ctx_t* restrict ctx, - const dyad_metadata_t* restrict mdata, - char** restrict file_data, - size_t* restrict file_len) -{ - DYAD_C_FUNCTION_START(); - dyad_rc_t rc = DYAD_RC_OK; - flux_future_t* f = NULL; - json_t* rpc_payload = NULL; - DYAD_LOG_INFO (ctx, "Packing payload for RPC to DYAD module"); - DYAD_C_FUNCTION_UPDATE_INT ("owner_rank", mdata->owner_rank); - DYAD_C_FUNCTION_UPDATE_STR ("fpath", mdata->fpath); - rc = ctx->dtl_handle->rpc_pack (ctx, mdata->fpath, mdata->owner_rank, &rpc_payload); - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, "Cannot create JSON payload for Flux RPC to " \ - "DYAD module\n"); - goto get_done; - } - DYAD_LOG_INFO (ctx, "Sending payload for RPC to DYAD module"); - f = flux_rpc_pack ((flux_t*) ctx->h, - DYAD_DTL_RPC_NAME, - mdata->owner_rank, - FLUX_RPC_STREAMING, - "o", - rpc_payload); - if (f == NULL) { - DYAD_LOG_ERROR (ctx, "Cannot send RPC to producer module\n"); - rc = DYAD_RC_BADRPC; - goto get_done; - } - DYAD_LOG_INFO (ctx, "Receive RPC response from DYAD module"); - rc = ctx->dtl_handle->rpc_recv_response (ctx, f); - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, "Cannot receive and/or parse the RPC response\n"); - goto get_done; - } - DYAD_LOG_INFO (ctx, "Establish DTL connection with DYAD module"); - rc = ctx->dtl_handle->establish_connection (ctx); - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, \ - "Cannot establish connection with DYAD module on broker " \ - "%u\n", \ - mdata->owner_rank); - goto get_done; - } - DYAD_LOG_INFO (ctx, "Receive file data via DTL"); - rc = ctx->dtl_handle->recv (ctx, (void**)file_data, file_len); - DYAD_LOG_INFO (ctx, "Close DTL connection with DYAD module"); - ctx->dtl_handle->close_connection (ctx); - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, "Cannot receive data from producer module\n"); - goto get_done; - } - DYAD_C_FUNCTION_UPDATE_INT ("file_len", *file_len); - - rc = DYAD_RC_OK; - -get_done:; - // There are two return codes that have special meaning when coming from the - // DTL: - // * DYAD_RC_RPC_FINISHED: occurs when an ENODATA error occurs - // * DYAD_RC_BADRPC: occurs when a previous RPC operation fails - // In either of these cases, we do not need to wait for the end of stream - // because the RPC is already completely messed up. If we do not have either - // of these cases, we will wait for one more RPC message. If everything went - // well in the module, this last message will set errno to ENODATA (i.e., - // end of stream). Otherwise, something went wrong, so we'll return - // DYAD_RC_BADRPC. - DYAD_LOG_INFO (ctx, "Wait for end-of-stream message from module (current RC = %d)\n", rc); - if (rc != DYAD_RC_RPC_FINISHED && rc != DYAD_RC_BADRPC) { - if (!(flux_rpc_get (f, NULL) < 0 && errno == ENODATA)) { - DYAD_LOG_ERROR (ctx, \ - "An error occured at end of getting data! Either the " \ - "module sent too many responses, or the module " \ - "failed with a bad error (errno = %d)\n", \ - errno); - rc = DYAD_RC_BADRPC; - } - } -#ifdef DYAD_ENABLE_UCX_RMA - ctx->dtl_handle->get_buffer(ctx, 0, (void**)file_data); - ssize_t read_len = 0l; - memcpy (&read_len, *file_data, sizeof (read_len)); - if (read_len < 0l) { - *file_len = 0ul; - DYAD_LOG_DEBUG (ctx, "Not able to read from %s file", mdata->fpath); - rc = DYAD_RC_BADFIO; - } else { - *file_len = (size_t) read_len; - } - *file_data = ((char*)*file_data) + sizeof (read_len); - DYAD_LOG_INFO (ctx, "Read %zd bytes from %s file", *file_len, mdata->fpath); -#endif - DYAD_LOG_INFO (ctx, "Destroy the Flux future for the RPC\n"); - flux_future_destroy (f); - DYAD_C_FUNCTION_END(); - return rc; -} - -DYAD_CORE_FUNC_MODS dyad_rc_t dyad_cons_store (const dyad_ctx_t* restrict ctx, - const dyad_metadata_t* restrict mdata, - int fd, const size_t data_len, - char* restrict file_data) -{ - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_INT ("fd", fd); - dyad_rc_t rc = DYAD_RC_OK; - const char* odir = NULL; - char file_path[PATH_MAX + 1] = {'\0'}; - char file_path_copy[PATH_MAX + 1] = {'\0'}; - mode_t m = (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH | S_ISGID); - size_t written_len = 0; - memset (file_path, 0, PATH_MAX + 1); - memset (file_path_copy, 0, PATH_MAX + 1); - - // Build the full path to the file being consumed - strncpy (file_path, ctx->cons_managed_path, PATH_MAX - 1); - concat_str (file_path, mdata->fpath, "/", PATH_MAX); - memcpy (file_path_copy, file_path, PATH_MAX); // dirname modifies the arg - DYAD_C_FUNCTION_UPDATE_STR ("cons_managed_path", ctx->cons_managed_path); - DYAD_C_FUNCTION_UPDATE_STR ("fpath", mdata->fpath); - DYAD_C_FUNCTION_UPDATE_STR ("file_path_copy", file_path_copy); - - DYAD_LOG_INFO (ctx, "Saving retrieved data to %s\n", file_path); - // Create the directory as needed - // TODO: Need to be consistent with the mode at the source - odir = dirname (file_path_copy); - if ((strncmp (odir, ".", strlen (".")) != 0) && (mkdir_as_needed (odir, m) < 0)) { - DYAD_LOG_ERROR (ctx, "Cannot create needed directories for pulled file\n"); - rc = DYAD_RC_BADFIO; - goto pull_done; - } - - // Write the file contents to the location specified by the user - written_len = write (fd, file_data, data_len); - if (written_len != data_len) { - DYAD_LOG_ERROR (ctx, "cons store write of pulled file failed!\n"); - rc = DYAD_RC_BADFIO; - goto pull_done; - } - DYAD_C_FUNCTION_UPDATE_INT ("data_len", data_len); - rc = DYAD_RC_OK; - -pull_done:; - // If "check" is set and the operation was successful, set the - // DYAD_CHECK_ENV environment variable to "ok" - if (rc == DYAD_RC_OK && (ctx && ctx->check)) - setenv(DYAD_CHECK_ENV, "ok", 1); - DYAD_C_FUNCTION_END(); - return rc; -} - -dyad_rc_t dyad_produce (dyad_ctx_t* restrict ctx, const char* restrict fname) -{ - DYAD_C_FUNCTION_START(); - ctx->fname = fname; - DYAD_C_FUNCTION_UPDATE_STR ("fname", ctx->fname); - DYAD_LOG_DEBUG (ctx, "Executing dyad_produce"); - dyad_rc_t rc = DYAD_RC_OK; - // If the context is not defined, then it is not valid. - // So, return DYAD_NOCTX - if (!ctx || !ctx->h) { - DYAD_LOG_ERROR(ctx, "No CTX found in dyad_produce"); - rc = DYAD_RC_NOCTX; - goto produce_done; - } - // If the producer-managed path is NULL or empty, then the context is not - // valid for a producer operation. So, return DYAD_BADMANAGEDPATH - if (ctx->prod_managed_path == NULL) { - DYAD_LOG_ERROR(ctx, "No or empty producer managed path was found %s", \ - ctx->prod_managed_path); - rc = DYAD_RC_BADMANAGEDPATH; - goto produce_done; - } - // If the context is valid, call dyad_commit to perform - // the producer operation - rc = dyad_commit (ctx, fname); -produce_done:; - DYAD_C_FUNCTION_END(); - return rc; -} - -/** This function is coupled with Python API. This populates `mdata' which - * is used by `dyad_consume_w_metadata ()' - */ -dyad_rc_t dyad_get_metadata (dyad_ctx_t* restrict ctx, - const char* restrict fname, - bool should_wait, - dyad_metadata_t** restrict mdata) -{ - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_STR ("fname", fname); - DYAD_C_FUNCTION_UPDATE_INT ("should_wait", should_wait); - dyad_rc_t rc = DYAD_RC_OK; - -#if 0 - if (fname == NULL || strlen (fname) > PATH_MAX) { - rc = DYAD_RC_SYSFAIL; - goto get_metadata_done; - } -#endif - const size_t fname_len = strlen (fname); - char upath[PATH_MAX+1] = {'\0'}; - - DYAD_LOG_INFO (ctx, "Obtaining file path relative to consumer directory: %s", upath); - - if (fname_len == 0ul) { - rc = DYAD_RC_BADFIO; - goto get_metadata_done; - } - if (ctx->relative_to_managed_path && - (strncmp (fname, DYAD_PATH_DELIM, ctx->delim_len) != 0)) - { // fname is a relative path that is relative to the cons_managed_path - memcpy (upath, fname, fname_len); - } else if (!cmp_canonical_path_prefix (ctx, false, fname, upath, PATH_MAX)) { - // Extract the path to the file specified by fname relative to the - // producer-managed path - // This relative path will be stored in upath - //DYAD_LOG_TRACE (ctx, "%s is not in the Consumer's managed path\n", fname); - // NOTE: This is different from what dyad_fetch/commit returns, - // which is DYAD_RC_OK such that dyad does not interfere accesses on - // non-managed directories. - rc = DYAD_RC_UNTRACKED; - goto get_metadata_done; - } - ctx->reenter = false; - DYAD_C_FUNCTION_UPDATE_STR ("upath", upath); - - // check if file exist locally, if so skip kvs - int fd = open (fname, O_RDONLY); - if (fd != -1) { - close (fd); - if (mdata == NULL) { - DYAD_LOG_ERROR (ctx, "Metadata double pointer is NULL. " \ - "Cannot correctly create metadata object"); - rc = DYAD_RC_NOTFOUND; - goto get_metadata_done; - } - if (*mdata != NULL) { - DYAD_LOG_INFO (ctx, "Metadata object is already allocated. Skipping allocation"); - } else { - *mdata = (dyad_metadata_t*)malloc (sizeof (struct dyad_metadata)); - if (*mdata == NULL) { - DYAD_LOG_ERROR (ctx, "Cannot allocate memory for metadata object"); - rc = DYAD_RC_SYSFAIL; - goto get_metadata_done; - } - } - (*mdata)->fpath = (char*)malloc (fname_len + 1); - if ((*mdata)->fpath == NULL) { - DYAD_LOG_ERROR (ctx, "Cannot allocate memory for fpath in metadata object"); - rc = DYAD_RC_SYSFAIL; - goto get_metadata_done; - } - memset ((*mdata)->fpath, '\0', fname_len + 1); - memcpy ((*mdata)->fpath, fname, fname_len); - (*mdata)->owner_rank = ctx->rank; - rc = DYAD_RC_OK; - goto get_metadata_done; - } - - const size_t topic_len = PATH_MAX; - char topic[PATH_MAX+1] = {'\0'}; - DYAD_LOG_INFO (ctx, "Generating KVS key: %s", topic); - gen_path_key (upath, topic, topic_len, ctx->key_depth, ctx->key_bins); - rc = dyad_kvs_read (ctx, topic, upath, should_wait, mdata); - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, "Could not read data from the KVS"); - goto get_metadata_done; - } - rc = DYAD_RC_OK; - -get_metadata_done:; - if (DYAD_IS_ERROR (rc) && mdata != NULL && *mdata != NULL) { - dyad_free_metadata (mdata); - } - ctx->reenter = true; - DYAD_C_FUNCTION_END(); - return rc; -} - -dyad_rc_t dyad_free_metadata (dyad_metadata_t** mdata) -{ - DYAD_C_FUNCTION_START(); - if (mdata == NULL || *mdata == NULL) { - return DYAD_RC_OK; - } - if ((*mdata)->fpath != NULL) - free ((*mdata)->fpath); - free (*mdata); - *mdata = NULL; - DYAD_C_FUNCTION_END(); - return DYAD_RC_OK; -} - -dyad_rc_t dyad_consume (dyad_ctx_t* restrict ctx, const char* restrict fname) -{ - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_STR ("fname", fname); - dyad_rc_t rc = DYAD_RC_OK; - int lock_fd = -1, io_fd = -1; - ssize_t file_size = -1; - char* file_data = NULL; - size_t data_len = 0ul; - dyad_metadata_t* mdata = NULL; - struct flock exclusive_lock; - char upath[PATH_MAX+1] = {'\0'}; - - // If the context is not defined, then it is not valid. - // So, return DYAD_NOCTX - if (!ctx || !ctx->h) { - rc = DYAD_RC_NOCTX; - goto consume_close; - } - // If the consumer-managed path is NULL or empty, then the context is not - // valid for a consumer operation. So, return DYAD_BADMANAGEDPATH - if (ctx->cons_managed_path == NULL) { - rc = DYAD_RC_BADMANAGEDPATH; - goto consume_close; - } - - if (ctx->relative_to_managed_path && (strlen (fname) > 0ul) && - (strncmp (fname, DYAD_PATH_DELIM, ctx->delim_len) != 0)) - { // fname is a relative path that is relative to the cons_managed_path - memcpy (upath, fname, strlen (fname)); - } else if (!cmp_canonical_path_prefix (ctx, false, fname, upath, PATH_MAX)) { - // Extract the path to the file specified by fname relative to the - // consumer-managed path - // This relative path will be stored in upath - //DYAD_LOG_TRACE (ctx, "%s is not in the Consumer's managed path\n", fname); - rc = DYAD_RC_OK; - goto consume_close; - } - ctx->reenter = false; - - lock_fd = open (fname, O_RDWR | O_CREAT, 0666); - if (lock_fd == -1) { - // This could be a system file on which users have no write permission - DYAD_LOG_ERROR (ctx, "Cannot create file (%s) for dyad_consume!\n", fname); - rc = DYAD_RC_BADFIO; - goto consume_close; - } - rc = dyad_excl_flock (ctx, lock_fd, &exclusive_lock); - if (DYAD_IS_ERROR (rc)) { - dyad_release_flock (ctx, lock_fd, &exclusive_lock); - goto consume_done; - } - file_size = get_file_size (lock_fd); - if (ctx->shared_storage) { - dyad_release_flock (ctx, lock_fd, &exclusive_lock); - if (!ctx->use_fs_locks || file_size <= 0) { - // as file size was zero that means consumer won the lock first so has to wait for kvs. - // or we cannot use file lock based synchronization as it does not work with the - // files managed by c++ fstream. - rc = dyad_fetch_metadata (ctx, fname, upath, &mdata); - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, "dyad_fetch_metadata failed fore shared storage!\n"); - goto consume_done; - } - } - } else { - if (file_size <= 0) { - DYAD_LOG_INFO (ctx, "[node %u rank %u pid %d] File (%s with lock_fd %d) is not fetched yet", \ - ctx->node_idx, ctx->rank, ctx->pid, fname, lock_fd); - // Call dyad_fetch to get (and possibly wait on) - // data from the Flux KVS - rc = dyad_fetch_metadata (ctx, fname, upath, &mdata); - // If an error occured in dyad_fetch_metadata, log an error - // and return the corresponding DYAD return code - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, "dyad_fetch_metadata failed!\n"); - dyad_release_flock (ctx, lock_fd, &exclusive_lock); - goto consume_done; - } - // If dyad_fetch_metadata was successful, but mdata is still NULL, - // then we need to skip data transfer. - // This is either because producer and consumer share storage - // or because the file is not on the managed directory. - if (mdata == NULL) { - DYAD_LOG_INFO (ctx, "File '%s' is local!\n", fname); - rc = DYAD_RC_OK; - dyad_release_flock (ctx, lock_fd, &exclusive_lock); - goto consume_done; - } - - // Call dyad_get_data to dispatch a RPC to the producer's Flux broker - // and retrieve the data associated with the file - rc = dyad_get_data (ctx, mdata, &file_data, &data_len); - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, "dyad_get_data failed!\n"); - dyad_release_flock (ctx, lock_fd, &exclusive_lock); - goto consume_done; - } - DYAD_C_FUNCTION_UPDATE_INT ("data_len", data_len); - io_fd = open (fname, O_WRONLY); - DYAD_C_FUNCTION_UPDATE_INT ("io_fd", io_fd); - if (io_fd == -1) { - DYAD_LOG_ERROR (ctx, "Cannot open file (%s) in write mode for dyad_consume!\n", fname); - rc = DYAD_RC_BADFIO; - goto consume_close; - } - // Call dyad_pull to fetch the data from the producer's - // Flux broker - rc = dyad_cons_store (ctx, mdata, io_fd, data_len, file_data); - // Regardless if there was an error in dyad_pull, - // free the KVS response object - if (mdata != NULL) { - dyad_free_metadata (&mdata); - } - if (close (io_fd) != 0) { - rc = DYAD_RC_BADFIO; - dyad_release_flock (ctx, lock_fd, &exclusive_lock); - goto consume_done; - } - // If an error occured in dyad_pull, log it - // and return the corresponding DYAD return code - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, "dyad_cons_store failed!\n"); - dyad_release_flock (ctx, lock_fd, &exclusive_lock); - goto consume_done; - }; - } - dyad_release_flock (ctx, lock_fd, &exclusive_lock); - } - DYAD_C_FUNCTION_UPDATE_INT ("file_size", file_size); -consume_done:; - if (close (lock_fd) != 0) { - rc = DYAD_RC_BADFIO; - } - if (file_data != NULL) { - ctx->dtl_handle->return_buffer (ctx, (void**)&file_data); - } - // Set reenter to true to allow additional intercepting -consume_close:; - ctx->reenter = true; - DYAD_C_FUNCTION_END(); - return rc; -} - -dyad_rc_t dyad_consume_w_metadata (dyad_ctx_t* restrict ctx, const char* fname, - const dyad_metadata_t* restrict mdata) -{ - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_STR ("fname", fname); - dyad_rc_t rc = DYAD_RC_OK; - int lock_fd = -1, io_fd = -1; - ssize_t file_size = -1; - char* file_data = NULL; - size_t data_len = 0ul; - struct flock exclusive_lock; - // If the context is not defined, then it is not valid. - // So, return DYAD_NOCTX - if (!ctx || !ctx->h) { - rc = DYAD_RC_NOCTX; - goto consume_close; - } - // If dyad_get_metadata was successful, but mdata is still NULL, - // then we need to skip data transfer. - if (mdata == NULL) { - DYAD_LOG_INFO (ctx, "File '%s' is local!\n", fname); - rc = DYAD_RC_OK; - goto consume_close; - } - // If the consumer-managed path is NULL or empty, then the context is not - // valid for a consumer operation. So, return DYAD_BADMANAGEDPATH - if (ctx->cons_managed_path == NULL) { - rc = DYAD_RC_BADMANAGEDPATH; - goto consume_close; - } - // Set reenter to false to avoid recursively performing DYAD operations - ctx->reenter = false; - lock_fd = open (fname, O_RDWR | O_CREAT, 0666); - DYAD_C_FUNCTION_UPDATE_INT ("lock_fd", lock_fd); - if (lock_fd == -1) { - DYAD_LOG_ERROR (ctx, "Cannot create file (%s) for dyad_consume_w_metadata!\n", fname); - rc = DYAD_RC_BADFIO; - goto consume_close; - } - rc = dyad_excl_flock (ctx, lock_fd, &exclusive_lock); - if (DYAD_IS_ERROR (rc)) { - dyad_release_flock (ctx, lock_fd, &exclusive_lock); - goto consume_close; - } - if ((file_size = get_file_size (lock_fd)) <= 0) { - DYAD_LOG_INFO (ctx, "[node %u rank %u pid %d] File (%s with fd %d) is not fetched yet", \ - ctx->node_idx, ctx->rank, ctx->pid, fname, lock_fd); - - // Call dyad_get_data to dispatch a RPC to the producer's Flux broker - // and retrieve the data associated with the file - rc = dyad_get_data (ctx, mdata, &file_data, &data_len); - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, "dyad_get_data failed!\n"); - dyad_release_flock (ctx, lock_fd, &exclusive_lock); - goto consume_done; - } - DYAD_C_FUNCTION_UPDATE_INT ("data_len", data_len); - io_fd = open (fname, O_WRONLY); - DYAD_C_FUNCTION_UPDATE_INT ("io_fd", io_fd); - if (io_fd == -1) { - DYAD_LOG_ERROR (ctx, "Cannot open file (%s) in write mode for dyad_consume!\n", fname); - rc = DYAD_RC_BADFIO; - goto consume_close; - } - // Call dyad_pull to fetch the data from the producer's - // Flux broker - rc = dyad_cons_store (ctx, mdata, io_fd, data_len, file_data); - - if (close (io_fd) != 0) { - rc = DYAD_RC_BADFIO; - dyad_release_flock (ctx, lock_fd, &exclusive_lock); - goto consume_done; - } - // If an error occured in dyad_pull, log it - // and return the corresponding DYAD return code - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, "dyad_cons_store failed!\n"); - dyad_release_flock (ctx, io_fd, &exclusive_lock); - goto consume_done; - }; - } - dyad_release_flock (ctx, lock_fd, &exclusive_lock); - DYAD_C_FUNCTION_UPDATE_INT ("file_size", file_size); - - if (close (lock_fd) != 0) { - rc = DYAD_RC_BADFIO; - goto consume_done; - } - rc = DYAD_RC_OK; -consume_done:; - if (file_data != NULL) { - ctx->dtl_handle->return_buffer (ctx, (void**)&file_data); - } -consume_close:; - // Set reenter to true to allow additional intercepting - ctx->reenter = true; - DYAD_C_FUNCTION_END(); - return rc; -} - -#if DYAD_SYNC_DIR -int dyad_sync_directory (dyad_ctx_t* restrict ctx, const char* restrict path) -{ - DYAD_C_FUNCTION_START(); - DYAD_C_FUNCTION_UPDATE_STR ("path", path); - // Flush new directory entry https://lwn.net/Articles/457671/ - char path_copy[PATH_MAX + 1] = {'\0'}; - int odir_fd = -1; - char* odir = NULL; - bool reenter = false; - int rc = 0; - memset (path_copy, 0, PATH_MAX + 1); - - strncpy (path_copy, path, PATH_MAX); - odir = dirname (path_copy); - - reenter = ctx->reenter; // backup ctx->reenter - if (ctx != NULL) - ctx->reenter = false; - - if ((odir_fd = open (odir, O_RDONLY)) < 0) { - IPRINTF (ctx, "Cannot open the directory \"%s\"\n", odir); - rc = -1; - } else { - if (fsync (odir_fd) < 0) { - IPRINTF (ctx, "Cannot flush the directory \"%s\"\n", odir); - rc = -1; - } - if (close (odir_fd) < 0) { - IPRINTF (ctx, "Cannot close the directory \"%s\"\n", odir); - rc = -1; - } - } - if (ctx != NULL) - ctx->reenter = reenter; - DYAD_C_FUNCTION_END(); - return rc; -} -#endif diff --git a/src/dyad/core/dyad_core.h b/src/dyad/core/dyad_core.h deleted file mode 100644 index b58577ca..00000000 --- a/src/dyad/core/dyad_core.h +++ /dev/null @@ -1,148 +0,0 @@ -#ifndef DYAD_CORE_DYAD_CORE_H -#define DYAD_CORE_DYAD_CORE_H - -#if defined(DYAD_HAS_CONFIG) -#include -#else -#error "no config" -#endif - -#include -#include -#include -#include - -#ifdef __cplusplus -#include -#include -#else -#include -#include -#include -#endif - - -#ifdef __cplusplus -extern "C" { -#endif -#if DYAD_PERFFLOW -#define DYAD_CORE_FUNC_MODS __attribute__ ((annotate ("@critical_path()"))) static -#else -#define DYAD_CORE_FUNC_MODS static inline -#endif -DYAD_DLL_EXPORTED extern const struct dyad_ctx dyad_ctx_default; - -struct dyad_metadata { - char* fpath; - uint32_t owner_rank; -}; -typedef struct dyad_metadata dyad_metadata_t; - -// Debug message -#ifndef DPRINTF -#if VA_OPT_SUPPORTED -#define DPRINTF(curr_dyad_ctx, fmt, ...) \ - do { \ - if ((curr_dyad_ctx) && (curr_dyad_ctx)->debug) \ - fprintf (stderr, (fmt) __VA_OPT__(,) __VA_ARGS__); \ - } while (0) -#else -#define DPRINTF(curr_dyad_ctx, fmt, ...) \ - do { \ - if ((curr_dyad_ctx) && (curr_dyad_ctx)->debug) \ - fprintf (stderr, (fmt), ##__VA_ARGS__); \ - } while (0) -#endif -#endif // DPRINTF - -#define TIME_DIFF(Tstart, Tend) \ - ((double)(1000000000L * ((Tend).tv_sec - (Tstart).tv_sec) + (Tend).tv_nsec - (Tstart).tv_nsec) \ - / 1000000000L) - -// Detailed information message that can be omitted -#if DYAD_FULL_DEBUG -#define IPRINTF DPRINTF -#define IPRINTF_DEFINED -#else -#define IPRINTF(curr_dyad_ctx, fmt, ...) -#endif // DYAD_FULL_DEBUG - - -/** - * @brief Wrapper function that performs all the common tasks needed - * of a producer - * @param[in] ctx the DYAD context for the operation - * @param[in] fname the name of the file being "produced" - * - * @return An error code from dyad_rc.h - */ -DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_produce (dyad_ctx_t* ctx, const char* fname); - -/** - * @brief Obtain DYAD metadata for a file in the consumer-managed directory - * @param[in] ctx the DYAD context for the operation - * @param[in] fname the name of the file for which metadata is obtained - * @param[in] should_wait if true, wait for the file to be produced before returning - * @param[out] mdata a dyad_metadata_t object containing the metadata for the file - * - * @return An error code from dyad_rc.h - */ -DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_get_metadata (dyad_ctx_t* ctx, - const char* fname, - bool should_wait, - dyad_metadata_t** mdata); - -DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_free_metadata (dyad_metadata_t** mdata); - -/** - * @brief Wrapper function that performs all the common tasks needed - * of a consumer - * @param[in] ctx the DYAD context for the operation - * @param[in] fname the name of the file being "consumed" - * - * @return An error code from dyad_rc.h - */ -DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_consume (dyad_ctx_t* ctx, const char* fname); - -/** - * @brief Wrapper function that performs all the common tasks needed - * of a consumer - * @param[in] ctx the DYAD context for the operation - * @param[in] fname the name of the file being "consumed" - * @param[in] mdata a dyad_metadata_t object containing the metadata for the file - * User is responsible for deallocating this object - * - * @return An error code from dyad_rc.h - */ -DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_consume_w_metadata (dyad_ctx_t* ctx, const char* fname, - const dyad_metadata_t* mdata); - - -/** - * Private Function definitions - */ -DYAD_DLL_EXPORTED dyad_rc_t dyad_get_data (const dyad_ctx_t* ctx, const dyad_metadata_t* mdata, - char** file_data, - size_t* file_len); -DYAD_DLL_EXPORTED dyad_rc_t dyad_commit (dyad_ctx_t* ctx, const char* fname); - -DYAD_DLL_EXPORTED int gen_path_key (const char* str, char* path_key, - const size_t len, - const uint32_t depth, - const uint32_t width); - -DYAD_DLL_EXPORTED dyad_rc_t dyad_kvs_read (const dyad_ctx_t* ctx, - const char* topic, - const char* upath, - bool should_wait, - dyad_metadata_t** mdata); - -#if DYAD_SYNC_DIR -DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED int dyad_sync_directory (dyad_ctx_t* ctx, const char* path); -#endif - -#ifdef __cplusplus -} -#endif - -#endif /* DYAD_CORE_DYAD_CORE */ diff --git a/src/dyad/core/dyad_ctx.c b/src/dyad/core/dyad_ctx.c index 376d22aa..2b5af118 100644 --- a/src/dyad/core/dyad_ctx.c +++ b/src/dyad/core/dyad_ctx.c @@ -7,7 +7,7 @@ #include #include #include -#include +// #include #include #include #include @@ -32,790 +32,785 @@ // To ensure we don't have multiple initialization, we need the following: // 1) The DYAD context (ctx below) must be static // 2) The DYAD context should be on the heap (done w/ malloc in dyad_init) -static __thread dyad_ctx_t* ctx = NULL; +static __thread dyad_ctx_t *ctx = NULL; const struct dyad_ctx dyad_ctx_default = { // Internal - NULL, // h - NULL, // dtl_handle - NULL, // fname - false, // use_fs_locks - NULL, // prod_real_path - NULL, // cons_real_path - 0u, // prod_managed_len - 0u, // cons_managed_len - 0u, // prod_real_len - 0u, // cons_real_len - 0u, // prod_managed_hash - 0u, // cons_managed_hash - 0u, // prod_real_hash - 0u, // cons_real_hash - 0u, // delim_len + NULL, // h + NULL, // dtl_handle + NULL, // fname + false, // use_fs_locks + NULL, // prod_real_path + NULL, // cons_real_path + 0u, // prod_managed_len + 0u, // cons_managed_len + 0u, // prod_real_len + 0u, // cons_real_len + 0u, // prod_managed_hash + 0u, // cons_managed_hash + 0u, // prod_real_hash + 0u, // cons_real_hash + 0u, // delim_len // User facing - false, // debug - false, // check - false, // reenter - true, // initialized - false, // shared_storage - false, // async_publish - false, // fsync_write - 3u, // key_depth - 1024u, // key_bins - 0u, // rank - 1u, // service_mux - 0u, // node_idx - -1, // pid - NULL, // kvs_namespace - NULL, // prod_managed_path - NULL, // cons_managed_path - false // relative_to_managed_path + false, // debug + false, // check + false, // reenter + true, // initialized + false, // shared_storage + false, // async_publish + false, // fsync_write + 3u, // key_depth + 1024u, // key_bins + 0u, // rank + 1u, // service_mux + 0u, // node_idx + -1, // pid + NULL, // kvs_namespace + NULL, // prod_managed_path + NULL, // cons_managed_path + false // relative_to_managed_path }; -DYAD_DLL_EXPORTED dyad_ctx_t* dyad_ctx_get () -{ - return ctx; -} +DYAD_DLL_EXPORTED dyad_ctx_t *dyad_ctx_get() { return ctx; } -DYAD_DLL_EXPORTED void dyad_ctx_init (const dyad_dtl_comm_mode_t dtl_comm_mode, void* flux_handle) -{ +DYAD_DLL_EXPORTED void dyad_ctx_init(const dyad_dtl_comm_mode_t dtl_comm_mode, + void *flux_handle) { #if DYAD_PROFILER == 3 - DFTRACER_C_FINI (); + DFTRACER_C_FINI(); #endif - DYAD_C_FUNCTION_START (); - dyad_rc_t rc = DYAD_RC_OK; - - rc = dyad_init_env (dtl_comm_mode, flux_handle); - - if (DYAD_IS_ERROR (rc)) { - fprintf (stderr, "Failed to initialize DYAD (code = %d)", rc); - if (ctx != NULL) { - ctx->initialized = false; - ctx->reenter = false; - } - DYAD_C_FUNCTION_END (); - return; - } else { - DYAD_LOG_INFO (ctx, "DYAD Initialized on loading using env variables"); - } - DYAD_C_FUNCTION_END (); + DYAD_C_FUNCTION_START(); + dyad_rc_t rc = DYAD_RC_OK; + + rc = dyad_init_env(dtl_comm_mode, flux_handle); + + if (DYAD_IS_ERROR(rc)) { + fprintf(stderr, "Failed to initialize DYAD (code = %d)", rc); + if (ctx != NULL) { + ctx->initialized = false; + ctx->reenter = false; + } + DYAD_C_FUNCTION_END(); + return; + } else { + DYAD_LOG_INFO(ctx, "DYAD Initialized on loading using env variables"); + } + DYAD_C_FUNCTION_END(); } -DYAD_DLL_EXPORTED void dyad_ctx_fini () -{ - DYAD_C_FUNCTION_START (); - if (ctx == NULL) { - DYAD_C_FUNCTION_END (); - goto dyad_wrapper_fini_done; - } - dyad_finalize (); +DYAD_DLL_EXPORTED void dyad_ctx_fini() { + DYAD_C_FUNCTION_START(); + if (ctx == NULL) { + DYAD_C_FUNCTION_END(); + goto dyad_wrapper_fini_done; + } + dyad_finalize(); dyad_wrapper_fini_done:; - DYAD_C_FUNCTION_END (); + DYAD_C_FUNCTION_END(); #if DYAD_PROFILER == 3 - DFTRACER_C_FINI (); + DFTRACER_C_FINI(); #endif } -dyad_rc_t dyad_clear (); +dyad_rc_t dyad_clear(); DYAD_DLL_EXPORTED -dyad_rc_t dyad_init (bool debug, - bool check, - bool shared_storage, - bool reinit, - bool async_publish, - bool fsync_write, - unsigned int key_depth, - unsigned int key_bins, - unsigned int service_mux, - const char* kvs_namespace, - const char* prod_managed_path, - const char* cons_managed_path, - bool relative_to_managed_path, - const char* dtl_mode_str, - const dyad_dtl_comm_mode_t dtl_comm_mode, - void* flux_handle) -{ - DYAD_LOGGER_INIT (); - unsigned my_rank = 0u; - size_t namespace_len = 0ul; +dyad_rc_t dyad_init(bool debug, bool check, bool shared_storage, bool reinit, + bool async_publish, bool fsync_write, + unsigned int key_depth, unsigned int key_bins, + unsigned int service_mux, const char *kvs_namespace, + const char *prod_managed_path, + const char *cons_managed_path, + bool relative_to_managed_path, const char *dtl_mode_str, + const dyad_dtl_comm_mode_t dtl_comm_mode, + void *flux_handle) { + DYAD_LOGGER_INIT(); + unsigned my_rank = 0u; + size_t namespace_len = 0ul; #ifdef DYAD_PROFILER_DFTRACER - const char* file_prefix = getenv (DFTRACER_LOG_FILE); - if (file_prefix == NULL) - file_prefix = "./dyad_"; - char log_file[4096] = {'\0'}; - sprintf (log_file, "%s_core-%d.pfw", file_prefix, getpid ()); - DFTRACER_C_INIT_NO_BIND (log_file, NULL, NULL); + const char *file_prefix = getenv(DFTRACER_LOG_FILE); + if (file_prefix == NULL) + file_prefix = "./dyad_"; + char log_file[4096] = {'\0'}; + sprintf(log_file, "%s_core-%d.pfw", file_prefix, getpid()); + DFTRACER_C_INIT_NO_BIND(log_file, NULL, NULL); #endif - DYAD_C_FUNCTION_START (); - dyad_rc_t rc = DYAD_RC_OK; - // Check if the actual dyad_ctx_t object is not NULL. - // If it is not NULL, that means the dyad_ctx_t object - // has either been allocated or fully initialized. - // If it's initialized, simply print a message and - // return DYAD_OK. - if (!reinit && (ctx != NULL)) { - if ((ctx)->initialized) { - // TODO Indicate already initialized - DYAD_LOG_DEBUG ((ctx), "DYAD context already initialized\n"); - rc = DYAD_RC_OK; - goto init_region_finish; - } - } else { - if (reinit) - dyad_finalize (); - // Allocate the dyad_ctx_t object and make sure the allocation - // worked successfully - ctx = (dyad_ctx_t*)malloc (sizeof (struct dyad_ctx)); - if (ctx == NULL) { - fprintf (stderr, "Could not allocate DYAD context!\n"); - rc = DYAD_RC_NOCTX; - goto init_region_finish; - } - } - // Set the initial contents of the dyad_ctx_t object to dyad_ctx_default. - *ctx = dyad_ctx_default; - // If neither managed path is provided, DYAD will not do anything. - // So, simply print a warning and return DYAD_OK. - if (prod_managed_path == NULL && cons_managed_path == NULL) { - fprintf (stderr, - "Warning: no managed path provided! DYAD will not do " - "anything!\n"); - rc = DYAD_RC_OK; - goto init_region_finish; - } + DYAD_C_FUNCTION_START(); + dyad_rc_t rc = DYAD_RC_OK; + // Check if the actual dyad_ctx_t object is not NULL. + // If it is not NULL, that means the dyad_ctx_t object + // has either been allocated or fully initialized. + // If it's initialized, simply print a message and + // return DYAD_OK. + if (!reinit && (ctx != NULL)) { + if ((ctx)->initialized) { + DYAD_LOG_DEBUG((ctx), "DYAD context already initialized\n"); + rc = DYAD_RC_OK; + goto init_region_finish; + } + } else { + if (reinit) + dyad_finalize(); + // Allocate the dyad_ctx_t object and make sure the allocation + // worked successfully + ctx = (dyad_ctx_t *)malloc(sizeof(struct dyad_ctx)); + if (ctx == NULL) { + fprintf(stderr, "Could not allocate DYAD context!\n"); + rc = DYAD_RC_NOCTX; + goto init_region_finish; + } + } + // Set the initial contents of the dyad_ctx_t object to dyad_ctx_default. + *ctx = dyad_ctx_default; + // If neither managed path is provided, DYAD will not do anything. + // So, simply print a warning and return DYAD_OK. + if (prod_managed_path == NULL && cons_managed_path == NULL) { + fprintf(stderr, "Warning: no managed path provided! DYAD will not do " + "anything!\n"); + rc = DYAD_RC_OK; + goto init_region_finish; + } #ifdef DYAD_PATH_DELIM - if (DYAD_PATH_DELIM == NULL || strlen (DYAD_PATH_DELIM) == 0ul) { - fprintf (stderr, "Invalid DYAD_PATH_DELIM defined in 'dyad_config.h'!\n"); - rc = DYAD_RC_NOCTX; - goto init_region_failed; - } - ctx->delim_len = strlen (DYAD_PATH_DELIM); -#else - fprintf (stderr, "DYAD_PATH_DELIM is not defined in 'dyad_config.h'!\n"); + if (DYAD_PATH_DELIM == NULL || strlen(DYAD_PATH_DELIM) == 0ul) { + fprintf(stderr, "Invalid DYAD_PATH_DELIM defined in 'dyad_config.h'!\n"); rc = DYAD_RC_NOCTX; goto init_region_failed; + } + ctx->delim_len = strlen(DYAD_PATH_DELIM); +#else + fprintf(stderr, "DYAD_PATH_DELIM is not defined in 'dyad_config.h'!\n"); + rc = DYAD_RC_NOCTX; + goto init_region_failed; #endif - // Set the values in dyad_ctx_t that don't need allocation - ctx->debug = debug; - ctx->check = check; - ctx->shared_storage = shared_storage; - ctx->async_publish = async_publish; - ctx->fsync_write = fsync_write; - ctx->key_depth = key_depth; - ctx->key_bins = key_bins; - - // Open a Flux handle and store it in the dyad_ctx_t - // object. If the open operation failed, return DYAD_FLUXFAIL - // In case of module, we use the flux handle passed to mod_main() - // instead of getting it by flux_open() - - if (flux_handle != NULL) { - ctx->h = (flux_t*)flux_handle; - } else { - ctx->h = flux_open (NULL, 0); - } - if (ctx->h == NULL) { - fprintf (stderr, "Could not open Flux handle!\n"); - rc = DYAD_RC_FLUXFAIL; - goto init_region_finish; - } - - // Get the rank of the Flux broker corresponding - // to the handle. If this fails, return DYAD_FLUXFAIL - DYAD_LOG_INFO (ctx, "DYAD_CORE: getting Flux rank"); - if (flux_get_rank (ctx->h, &(ctx->rank)) < 0) { - DYAD_LOG_INFO (ctx, "Could not get Flux rank!\n"); - rc = DYAD_RC_FLUXFAIL; - goto init_region_finish; - } - my_rank = ctx->rank; - ctx->service_mux = (service_mux < 1u) ? 1u : service_mux; - ctx->node_idx = ctx->rank / ctx->service_mux; - ctx->pid = getpid (); - if (my_rank == 0) { - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: debug %s", ctx->debug ? "true" : "false"); - DYAD_LOG_INFO (ctx, - "DYAD_CORE INIT: shared_storage %s", - ctx->shared_storage ? "true" : "false"); - DYAD_LOG_INFO (ctx, - "DYAD_CORE INIT: async_publish %s", - ctx->async_publish ? "true" : "false"); - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: fsync_write %s", ctx->fsync_write ? "true" : "false"); - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: kvs key depth %u", ctx->key_depth); - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: kvs key bins %u", ctx->key_bins); - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: broker rank %u", my_rank); - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: pid %u", ctx->pid); - } - - // If the namespace is provided, copy it into the dyad_ctx_t object - DYAD_LOG_INFO (ctx, "DYAD_CORE: saving KVS namespace"); - if (kvs_namespace == NULL) { - DYAD_LOG_ERROR (ctx, "No KVS namespace provided!\n"); - // TODO see if we want a different return val - goto init_region_failed; - } - namespace_len = strlen (kvs_namespace); - ctx->kvs_namespace = (char*)calloc (namespace_len + 1, sizeof (char)); - if (ctx->kvs_namespace == NULL) { - DYAD_LOG_ERROR (ctx, "Could not allocate buffer for KVS namespace!\n"); - goto init_region_failed; - } - strncpy (ctx->kvs_namespace, kvs_namespace, namespace_len + 1); - if (my_rank == 0) { - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: kvs_namespace %s", ctx->kvs_namespace); - } - - // Initialize the DTL based on the value of dtl_mode - // If an error occurs, log it and return an error - DYAD_LOG_INFO (ctx, "DYAD_CORE: inintializing DYAD DTL %s", dtl_mode_str); - rc = dyad_set_and_init_dtl_mode (dtl_mode_str, dtl_comm_mode); - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, "Cannot initialize the DTL %s\n", dtl_mode_str); - goto init_region_failed; - } - if (my_rank == 0) { - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: dtl_mode %s", dtl_mode_str); - } - - // If the producer-managed path is provided, copy it into the dyad_ctx_t object - if (dyad_set_prod_path (prod_managed_path) != DYAD_RC_OK) { - goto init_region_failed; - } - - // If the consumer-managed path is provided, copy it into the dyad_ctx_t object - if (dyad_set_cons_path (cons_managed_path) != DYAD_RC_OK) { - goto init_region_failed; - } - - ctx->relative_to_managed_path = relative_to_managed_path; - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: relative_to_managed_path %s", ctx->relative_to_managed_path ? "true" : "false"); + // Set the values in dyad_ctx_t that don't need allocation + ctx->debug = debug; + ctx->check = check; + ctx->shared_storage = shared_storage; + ctx->async_publish = async_publish; + ctx->fsync_write = fsync_write; + ctx->key_depth = key_depth; + ctx->key_bins = key_bins; + + // Open a Flux handle and store it in the dyad_ctx_t + // object. If the open operation failed, return DYAD_FLUXFAIL + // In case of module, we use the flux handle passed to mod_main() + // instead of getting it by flux_open() + + if (flux_handle != NULL) { + ctx->h = (flux_t *)flux_handle; + } else { + ctx->h = flux_open(NULL, 0); + } + if (ctx->h == NULL) { + fprintf(stderr, "Could not open Flux handle!\n"); + rc = DYAD_RC_FLUXFAIL; + goto init_region_finish; + } + + // Get the rank of the Flux broker corresponding + // to the handle. If this fails, return DYAD_FLUXFAIL + DYAD_LOG_INFO(ctx, "DYAD_CORE: getting Flux rank"); + if (flux_get_rank(ctx->h, &(ctx->rank)) < 0) { + DYAD_LOG_INFO(ctx, "Could not get Flux rank!\n"); + rc = DYAD_RC_FLUXFAIL; + goto init_region_finish; + } + my_rank = ctx->rank; + ctx->service_mux = (service_mux < 1u) ? 1u : service_mux; + ctx->node_idx = ctx->rank / ctx->service_mux; + ctx->pid = getpid(); + if (my_rank == 0) { + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: debug %s", + ctx->debug ? "true" : "false"); + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: shared_storage %s", + ctx->shared_storage ? "true" : "false"); + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: async_publish %s", + ctx->async_publish ? "true" : "false"); + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: fsync_write %s", + ctx->fsync_write ? "true" : "false"); + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: kvs key depth %u", ctx->key_depth); + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: kvs key bins %u", ctx->key_bins); + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: broker rank %u", my_rank); + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: pid %u", ctx->pid); + } + + // If the namespace is provided, copy it into the dyad_ctx_t object + DYAD_LOG_INFO(ctx, "DYAD_CORE: saving KVS namespace"); + if (kvs_namespace == NULL) { + DYAD_LOG_ERROR(ctx, "No KVS namespace provided!\n"); + // TODO see if we want a different return val + goto init_region_failed; + } + namespace_len = strlen(kvs_namespace); + ctx->kvs_namespace = (char *)calloc(namespace_len + 1, sizeof(char)); + if (ctx->kvs_namespace == NULL) { + DYAD_LOG_ERROR(ctx, "Could not allocate buffer for KVS namespace!\n"); + goto init_region_failed; + } + strncpy(ctx->kvs_namespace, kvs_namespace, namespace_len + 1); + if (my_rank == 0) { + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: kvs_namespace %s", ctx->kvs_namespace); + } + + // Initialize the DTL based on the value of dtl_mode + // If an error occurs, log it and return an error + DYAD_LOG_INFO(ctx, "DYAD_CORE: inintializing DYAD DTL %s", dtl_mode_str); + rc = dyad_set_and_init_dtl_mode(dtl_mode_str, dtl_comm_mode); + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(ctx, "Cannot initialize the DTL %s\n", dtl_mode_str); + goto init_region_failed; + } + if (my_rank == 0) { + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: dtl_mode %s", dtl_mode_str); + } + + // If the producer-managed path is provided, copy it into the dyad_ctx_t + // object + if (dyad_set_prod_path(prod_managed_path) != DYAD_RC_OK) { + goto init_region_failed; + } - DYAD_C_FUNCTION_UPDATE_STR ("prod_managed_path", ctx->prod_managed_path); - DYAD_C_FUNCTION_UPDATE_STR ("cons_managed_path", ctx->cons_managed_path); - ctx->use_fs_locks = true; // This is default value except for streams which dont have a fp. - // TODO Print logging info - rc = DYAD_RC_OK; - // TODO: Add folder option here. + // If the consumer-managed path is provided, copy it into the dyad_ctx_t + // object + if (dyad_set_cons_path(cons_managed_path) != DYAD_RC_OK) { + goto init_region_failed; + } + + ctx->relative_to_managed_path = relative_to_managed_path; + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: relative_to_managed_path %s", + ctx->relative_to_managed_path ? "true" : "false"); + + DYAD_C_FUNCTION_UPDATE_STR("prod_managed_path", ctx->prod_managed_path); + DYAD_C_FUNCTION_UPDATE_STR("cons_managed_path", ctx->cons_managed_path); + ctx->use_fs_locks = + true; // This is default value except for streams which dont have a fp. + // TODO Print logging info + rc = DYAD_RC_OK; + // TODO: Add folder option here. #ifndef DYAD_LOGGER_NO_LOG - char log_file_name[4096] = {'\0'}; - char err_file_name[4096] = {'\0'}; - mkdir_as_needed ("logs", (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH | S_ISGID)); - sprintf (log_file_name, "logs/dyad_core_%d.out", ctx->pid); - sprintf (err_file_name, "logs/dyad_core_%d.err", ctx->pid); - DYAD_LOG_STDERR_REDIRECT (err_file_name); - DYAD_LOG_STDERR_REDIRECT (log_file_name); -#endif // DYAD_LOGGER_NO_LOG - // Initialization is now complete! - ctx->initialized = true; - goto init_region_finish; + char log_file_name[4096] = {'\0'}; + char err_file_name[4096] = {'\0'}; + mkdir_as_needed("logs", (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH | S_ISGID)); + sprintf(log_file_name, "logs/dyad_core_%d.out", ctx->pid); + sprintf(err_file_name, "logs/dyad_core_%d.err", ctx->pid); + DYAD_LOG_STDERR_REDIRECT(err_file_name); + DYAD_LOG_STDERR_REDIRECT(log_file_name); +#endif // DYAD_LOGGER_NO_LOG + // Initialization is now complete! + ctx->initialized = true; + goto init_region_finish; init_region_failed:; - dyad_clear (); - // Set the initial contents of the dyad_ctx_t object to dyad_ctx_default. - *ctx = dyad_ctx_default; - if (rc == DYAD_RC_NOCTX) { - ctx->initialized = false; - ctx->reenter = false; - } - return DYAD_RC_NOCTX; + dyad_clear(); + // Set the initial contents of the dyad_ctx_t object to dyad_ctx_default. + *ctx = dyad_ctx_default; + if (rc == DYAD_RC_NOCTX) { + ctx->initialized = false; + ctx->reenter = false; + } + return DYAD_RC_NOCTX; init_region_finish:; - ctx->reenter = true; - DYAD_C_FUNCTION_END (); - return rc; + ctx->reenter = true; + DYAD_C_FUNCTION_END(); + return rc; } -DYAD_DLL_EXPORTED dyad_rc_t dyad_init_env (const dyad_dtl_comm_mode_t dtl_comm_mode, - void* flux_handle) -{ - DYAD_C_FUNCTION_START (); - const char* e = NULL; - bool debug = false; - bool check = false; - bool shared_storage = false; - bool reinit = false; - bool async_publish = false; - bool fsync_write = false; - bool relative_to_managed_path = false; - unsigned int key_depth = 0u; - unsigned int key_bins = 0u; - unsigned int service_mux = 1u; - const char* kvs_namespace = NULL; - const char* prod_managed_path = NULL; - const char* cons_managed_path = NULL; - const char* dtl_mode = NULL; - - if ((e = getenv (DYAD_SYNC_DEBUG_ENV))) { - debug = true; - } else { - debug = false; - } - - if (debug) { - DYAD_LOG_STDERR ( - "DYAD_CORE: Initializing with environment " - "variables\n"); - } - - if ((e = getenv (DYAD_SYNC_CHECK_ENV))) { - check = true; - } else { - check = false; - } - - if ((e = getenv (DYAD_SHARED_STORAGE_ENV))) { - shared_storage = true; - } else { - shared_storage = false; - } - - if ((e = getenv (DYAD_REINIT_ENV))) { - reinit = true; - } else { - reinit = false; - } - - if ((e = getenv (DYAD_ASYNC_PUBLISH_ENV))) { - async_publish = true; - } else { - async_publish = false; - } - - if ((e = getenv (DYAD_FSYNC_WRITE_ENV))) { - fsync_write = true; - } else { - fsync_write = false; - } - - if ((e = getenv (DYAD_KEY_DEPTH_ENV))) { - key_depth = atoi (e); - } else { - key_depth = 3; - } - - if ((e = getenv (DYAD_KEY_BINS_ENV))) { - key_bins = atoi (e); - } else { - key_bins = 1024; - } - - if ((e = getenv (DYAD_SERVICE_MUX_ENV))) { - service_mux = atoi (e); - } else { - service_mux = 1u; - } - - if ((e = getenv (DYAD_KVS_NAMESPACE_ENV))) { - kvs_namespace = e; - } else { - kvs_namespace = NULL; - } - - if ((e = getenv (DYAD_PATH_CONSUMER_ENV))) { - cons_managed_path = e; - } else { - cons_managed_path = NULL; - } - - if ((e = getenv (DYAD_PATH_PRODUCER_ENV))) { - prod_managed_path = e; - } else { - prod_managed_path = NULL; - } - - if ((e = getenv (DYAD_PATH_RELATIVE_ENV))) { - relative_to_managed_path = true; - } else { - relative_to_managed_path = false; - } - - if ((e = getenv (DYAD_DTL_MODE_ENV))) { - dtl_mode = e; - } else { - dtl_mode = dyad_dtl_mode_name[DYAD_DTL_DEFAULT]; - DYAD_LOG_STDERR ("%s is not set. Defaulting to %s\n", DYAD_DTL_MODE_ENV, dtl_mode); - } - if (debug) { - DYAD_LOG_STDERR ( - "DYAD_CORE: retrieved configuration from environment. " - "Now initializing DYAD\n"); - } - dyad_rc_t rc = dyad_init (debug, - check, - shared_storage, - reinit, - async_publish, - fsync_write, - key_depth, - key_bins, - service_mux, - kvs_namespace, - prod_managed_path, - cons_managed_path, - relative_to_managed_path, - dtl_mode, - dtl_comm_mode, - flux_handle); - DYAD_C_FUNCTION_END (); - return rc; +DYAD_DLL_EXPORTED dyad_rc_t +dyad_init_env(const dyad_dtl_comm_mode_t dtl_comm_mode, void *flux_handle) { + DYAD_C_FUNCTION_START(); + const char *e = NULL; + bool debug = false; + bool check = false; + bool shared_storage = false; + bool reinit = false; + bool async_publish = false; + bool fsync_write = false; + bool relative_to_managed_path = false; + unsigned int key_depth = 0u; + unsigned int key_bins = 0u; + unsigned int service_mux = 1u; + const char *kvs_namespace = NULL; + const char *prod_managed_path = NULL; + const char *cons_managed_path = NULL; + const char *dtl_mode = NULL; + + if ((e = getenv(DYAD_SYNC_DEBUG_ENV))) { + debug = true; + } else { + debug = false; + } + + if (debug) { + DYAD_LOG_STDERR("DYAD_CORE: Initializing with environment " + "variables\n"); + } + + if ((e = getenv(DYAD_SYNC_CHECK_ENV))) { + check = true; + } else { + check = false; + } + + if ((e = getenv(DYAD_SHARED_STORAGE_ENV))) { + shared_storage = true; + } else { + shared_storage = false; + } + + if ((e = getenv(DYAD_REINIT_ENV))) { + reinit = true; + } else { + reinit = false; + } + + if ((e = getenv(DYAD_ASYNC_PUBLISH_ENV))) { + async_publish = true; + } else { + async_publish = false; + } + + if ((e = getenv(DYAD_FSYNC_WRITE_ENV))) { + fsync_write = true; + } else { + fsync_write = false; + } + + if ((e = getenv(DYAD_KEY_DEPTH_ENV))) { + key_depth = atoi(e); + } else { + key_depth = 3; + } + + if ((e = getenv(DYAD_KEY_BINS_ENV))) { + key_bins = atoi(e); + } else { + key_bins = 1024; + } + + if ((e = getenv(DYAD_SERVICE_MUX_ENV))) { + service_mux = atoi(e); + } else { + service_mux = 1u; + } + + if ((e = getenv(DYAD_KVS_NAMESPACE_ENV))) { + kvs_namespace = e; + } else { + kvs_namespace = NULL; + } + + if ((e = getenv(DYAD_PATH_CONSUMER_ENV))) { + cons_managed_path = e; + } else { + cons_managed_path = NULL; + } + + if ((e = getenv(DYAD_PATH_PRODUCER_ENV))) { + prod_managed_path = e; + } else { + prod_managed_path = NULL; + } + + if ((e = getenv(DYAD_PATH_RELATIVE_ENV))) { + relative_to_managed_path = true; + } else { + relative_to_managed_path = false; + } + + if ((e = getenv(DYAD_DTL_MODE_ENV))) { + dtl_mode = e; + } else { + dtl_mode = dyad_dtl_mode_name[DYAD_DTL_DEFAULT]; + DYAD_LOG_STDERR("%s is not set. Defaulting to %s\n", DYAD_DTL_MODE_ENV, + dtl_mode); + } + if (debug) { + DYAD_LOG_STDERR("DYAD_CORE: retrieved configuration from environment. " + "Now initializing DYAD\n"); + } + dyad_rc_t rc = + dyad_init(debug, check, shared_storage, reinit, async_publish, + fsync_write, key_depth, key_bins, service_mux, kvs_namespace, + prod_managed_path, cons_managed_path, relative_to_managed_path, + dtl_mode, dtl_comm_mode, flux_handle); + DYAD_C_FUNCTION_END(); + return rc; } -DYAD_DLL_EXPORTED dyad_rc_t dyad_set_and_init_dtl_mode (const char* dtl_name, - dyad_dtl_comm_mode_t dtl_comm_mode) -{ - int rc = DYAD_RC_OK; - size_t dtl_name_len = 0ul; - dyad_dtl_mode_t dtl_mode = DYAD_DTL_DEFAULT; - - DYAD_C_FUNCTION_START (); - - if (!dtl_name) { - return DYAD_RC_BADDTLMODE; - } - - dtl_name_len = strlen (dtl_name); - if (strncmp (dtl_name, dyad_dtl_mode_name[DYAD_DTL_DEFAULT], dtl_name_len) == 0) { - dtl_mode = DYAD_DTL_DEFAULT; - } else if (strncmp (dtl_name, dyad_dtl_mode_name[DYAD_DTL_UCX], dtl_name_len) == 0) { - dtl_mode = DYAD_DTL_UCX; - } else if (strncmp (dtl_name, dyad_dtl_mode_name[DYAD_DTL_FLUX_RPC], dtl_name_len) == 0) { - dtl_mode = DYAD_DTL_FLUX_RPC; - } else { - DYAD_LOG_STDERR ("Invalid env %s = %s.\n", DYAD_DTL_MODE_ENV, dtl_name); - return DYAD_RC_BADDTLMODE; - } - - DYAD_LOG_INFO (ctx, "DYAD_CORE: DYAD DTL mode to initialize: %s", dyad_dtl_mode_name[dtl_mode]); - - // If the ctx->dtl_handle is already null, it will just return success - rc = dyad_dtl_finalize (ctx); - if (DYAD_IS_ERROR (rc)) { - goto set_and_init_dtl_mode_region_finish; - } - - rc = dyad_dtl_init (ctx, dtl_mode, dtl_comm_mode, ctx->debug); - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (ctx, "Cannot initialize the DTL %s\n", dyad_dtl_mode_name[dtl_mode]); - } +DYAD_DLL_EXPORTED dyad_rc_t dyad_set_and_init_dtl_mode( + const char *dtl_name, dyad_dtl_comm_mode_t dtl_comm_mode) { + int rc = DYAD_RC_OK; + size_t dtl_name_len = 0ul; + dyad_dtl_mode_t dtl_mode = DYAD_DTL_DEFAULT; + + DYAD_C_FUNCTION_START(); + + if (!dtl_name) { + return DYAD_RC_BADDTLMODE; + } + + dtl_name_len = strlen(dtl_name); + if (strncmp(dtl_name, dyad_dtl_mode_name[DYAD_DTL_DEFAULT], dtl_name_len) == + 0) { + dtl_mode = DYAD_DTL_DEFAULT; + } else if (strncmp(dtl_name, dyad_dtl_mode_name[DYAD_DTL_UCX], + dtl_name_len) == 0) { + dtl_mode = DYAD_DTL_UCX; + } else if (strncmp(dtl_name, dyad_dtl_mode_name[DYAD_DTL_FLUX_RPC], + dtl_name_len) == 0) { + dtl_mode = DYAD_DTL_FLUX_RPC; + } else { + DYAD_LOG_STDERR("Invalid env %s = %s.\n", DYAD_DTL_MODE_ENV, dtl_name); + return DYAD_RC_BADDTLMODE; + } + + DYAD_LOG_INFO(ctx, "DYAD_CORE: DYAD DTL mode to initialize: %s", + dyad_dtl_mode_name[dtl_mode]); + + // If the ctx->dtl_handle is already null, it will just return success + rc = dyad_dtl_finalize(ctx); + if (DYAD_IS_ERROR(rc)) { + goto set_and_init_dtl_mode_region_finish; + } + + rc = dyad_dtl_init(ctx, dtl_mode, dtl_comm_mode, ctx->debug); + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(ctx, "Cannot initialize the DTL %s\n", + dyad_dtl_mode_name[dtl_mode]); + } set_and_init_dtl_mode_region_finish:; - DYAD_C_FUNCTION_END (); - return rc; + DYAD_C_FUNCTION_END(); + return rc; } -DYAD_DLL_EXPORTED dyad_rc_t dyad_set_prod_path (const char* prod_managed_path) -{ - dyad_rc_t rc = DYAD_RC_OK; - bool reenter_backup = false; - size_t prod_path_len = 0ul; - size_t prod_realpath_len = 0ul; - char prod_real_path[PATH_MAX + 1] = {'\0'}; - - if (!ctx) { - DYAD_LOG_STDERR ("Empty Producer managed path to allocate!\n"); - rc = DYAD_RC_NOCTX; - goto set_prod_path_region_finish; - } else { - reenter_backup = ctx->reenter; - ctx->reenter = false; - } - DYAD_C_FUNCTION_START (); - - if (ctx->prod_managed_path) { - free (ctx->prod_managed_path); - ctx->prod_managed_path = NULL; - ctx->prod_managed_len = 0u; - ctx->prod_managed_hash = 0u; - } +DYAD_DLL_EXPORTED dyad_rc_t dyad_set_prod_path(const char *prod_managed_path) { + dyad_rc_t rc = DYAD_RC_OK; + bool reenter_backup = false; + size_t prod_path_len = 0ul; + size_t prod_realpath_len = 0ul; + char prod_real_path[PATH_MAX + 1] = {'\0'}; - if (ctx->prod_real_path) { - free (ctx->prod_real_path); - ctx->prod_real_path = NULL; - ctx->prod_real_len = 0u; - ctx->prod_real_hash = 0u; + if (!ctx) { + DYAD_LOG_STDERR("Empty Producer managed path to allocate!\n"); + rc = DYAD_RC_NOCTX; + goto set_prod_path_region_finish; + } else { + reenter_backup = ctx->reenter; + ctx->reenter = false; + } + DYAD_C_FUNCTION_START(); + + if (ctx->prod_managed_path) { + free(ctx->prod_managed_path); + ctx->prod_managed_path = NULL; + ctx->prod_managed_len = 0u; + ctx->prod_managed_hash = 0u; + } + + if (ctx->prod_real_path) { + free(ctx->prod_real_path); + ctx->prod_real_path = NULL; + ctx->prod_real_len = 0u; + ctx->prod_real_hash = 0u; + } + + DYAD_LOG_INFO(ctx, "DYAD_CORE: saving producer path"); + if (prod_managed_path == NULL) { + DYAD_LOG_INFO(ctx, "DYAD_CORE: producer path is not provided"); + ctx->prod_managed_path = NULL; + ctx->prod_managed_len = 0u; + ctx->prod_managed_hash = 0u; + ctx->prod_real_path = NULL; + ctx->prod_real_len = 0u; + ctx->prod_real_hash = 0u; + } else { + if ((prod_path_len = strlen(prod_managed_path)) == 0ul) { + DYAD_LOG_ERROR(ctx, "Empty Producer managed path to allocate!\n"); + rc = DYAD_RC_BADMANAGEDPATH; + goto set_prod_path_region_failed; + } + ctx->prod_managed_path = (char *)calloc(prod_path_len + 1, sizeof(char)); + if (ctx->prod_managed_path == NULL) { + DYAD_LOG_ERROR(ctx, + "Could not allocate buffer for Producer managed path!\n"); + rc = DYAD_RC_SYSFAIL; + goto set_prod_path_region_failed; + } + if (!memcpy(ctx->prod_managed_path, prod_managed_path, prod_path_len + 1)) { + DYAD_LOG_ERROR(ctx, "Could not copy Producer managed path!\n"); + rc = DYAD_RC_SYSFAIL; + goto set_prod_path_region_failed; + } + ctx->prod_managed_len = prod_path_len; + ctx->prod_managed_hash = hash_str(ctx->prod_managed_path, DYAD_SEED); + if (ctx->prod_managed_hash == 0u) { + DYAD_LOG_ERROR(ctx, "Could not hash Producer managed path!\n"); + rc = DYAD_RC_BADMANAGEDPATH; + goto set_prod_path_region_failed; + } + + if (!realpath(prod_managed_path, prod_real_path)) { + DYAD_LOG_DEBUG(ctx, "Could not get Producer real path!\n"); + // perror ("Could not get Producer real path!\n"); + // This is not necessarily an error. This can happen when the environment + // variables for both managed paths are set but consumer does not create + // the managed path of producer or vice versa. `realpath()' fails if a + // path does not exist. + } else { + prod_realpath_len = strlen(prod_real_path); } - DYAD_LOG_INFO (ctx, "DYAD_CORE: saving producer path"); - if (prod_managed_path == NULL) { - DYAD_LOG_INFO (ctx, "DYAD_CORE: producer path is not provided"); - ctx->prod_managed_path = NULL; - ctx->prod_managed_len = 0u; - ctx->prod_managed_hash = 0u; - ctx->prod_real_path = NULL; - ctx->prod_real_len = 0u; - ctx->prod_real_hash = 0u; + // If the realpath is the same, we turn off checking against it as it is + // redundant + if (prod_realpath_len == 0u || + strncmp(prod_managed_path, prod_real_path, prod_path_len) == 0) { + DYAD_LOG_INFO(ctx, "DYAD_CORE: producer real path is redundant"); + ctx->prod_real_path = NULL; + ctx->prod_real_len = 0u; + ctx->prod_real_hash = 0u; } else { - if ((prod_path_len = strlen (prod_managed_path)) == 0ul) { - DYAD_LOG_ERROR (ctx, "Empty Producer managed path to allocate!\n"); - rc = DYAD_RC_BADMANAGEDPATH; - goto set_prod_path_region_failed; - } - ctx->prod_managed_path = (char*)calloc (prod_path_len + 1, sizeof (char)); - if (ctx->prod_managed_path == NULL) { - DYAD_LOG_ERROR (ctx, "Could not allocate buffer for Producer managed path!\n"); - rc = DYAD_RC_SYSFAIL; - goto set_prod_path_region_failed; - } - if (!memcpy (ctx->prod_managed_path, prod_managed_path, prod_path_len + 1)) { - DYAD_LOG_ERROR (ctx, "Could not copy Producer managed path!\n"); - rc = DYAD_RC_SYSFAIL; - goto set_prod_path_region_failed; - } - ctx->prod_managed_len = prod_path_len; - ctx->prod_managed_hash = hash_str (ctx->prod_managed_path, DYAD_SEED); - if (ctx->prod_managed_hash == 0u) { - DYAD_LOG_ERROR (ctx, "Could not hash Producer managed path!\n"); - rc = DYAD_RC_BADMANAGEDPATH; - goto set_prod_path_region_failed; - } - - if (!realpath (prod_managed_path, prod_real_path)) { - DYAD_LOG_DEBUG (ctx, "Could not get Producer real path!\n"); - // perror ("Could not get Producer real path!\n"); - // This is not necessarily an error. This can happen when the environment - // variables for both managed paths are set but consumer does not create - // the managed path of producer or vice versa. `realpath()' fails if a - // path does not exist. - } else { - prod_realpath_len = strlen (prod_real_path); - } - - // If the realpath is the same, we turn off checking against it as it is redundant - if (prod_realpath_len == 0u - || strncmp (prod_managed_path, prod_real_path, prod_path_len) == 0) { - DYAD_LOG_INFO (ctx, "DYAD_CORE: producer real path is redundant"); - ctx->prod_real_path = NULL; - ctx->prod_real_len = 0u; - ctx->prod_real_hash = 0u; - } else { - ctx->prod_real_path = (char*)calloc (prod_realpath_len + 1, sizeof (char)); - if (ctx->prod_real_path == NULL || prod_realpath_len == 0ul) { - DYAD_LOG_ERROR (ctx, "Could not allocate buffer for Producer managed path!\n"); - rc = DYAD_RC_SYSFAIL; - goto set_prod_path_region_failed; - } - - if (!memcpy (ctx->prod_real_path, prod_real_path, prod_realpath_len)) { - DYAD_LOG_ERROR (ctx, "Could not copy Producer real path!\n"); - rc = DYAD_RC_SYSFAIL; - goto set_prod_path_region_failed; - } - ctx->prod_real_path[prod_realpath_len] = '\0'; - ctx->prod_real_len = prod_realpath_len; - ctx->prod_real_hash = hash_str (ctx->prod_real_path, DYAD_SEED); - if (ctx->prod_real_hash == 0u) { - DYAD_LOG_ERROR (ctx, "Could not hash Producer real path!\n"); - rc = DYAD_RC_BADMANAGEDPATH; - goto set_prod_path_region_failed; - } - } - - if (ctx->rank == 0) { - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: prod_managed_path %s", ctx->prod_managed_path); - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: prod_managed_len %u", ctx->prod_managed_len); - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: prod_managed_hash %u", ctx->prod_managed_hash); - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: prod_real_path %s", ctx->prod_real_path); - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: prod_real_len %u", ctx->prod_real_len); - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: prod_real_hash %u", ctx->prod_real_hash); - } - } + ctx->prod_real_path = (char *)calloc(prod_realpath_len + 1, sizeof(char)); + if (ctx->prod_real_path == NULL || prod_realpath_len == 0ul) { + DYAD_LOG_ERROR( + ctx, "Could not allocate buffer for Producer managed path!\n"); + rc = DYAD_RC_SYSFAIL; + goto set_prod_path_region_failed; + } + + if (!memcpy(ctx->prod_real_path, prod_real_path, prod_realpath_len)) { + DYAD_LOG_ERROR(ctx, "Could not copy Producer real path!\n"); + rc = DYAD_RC_SYSFAIL; + goto set_prod_path_region_failed; + } + ctx->prod_real_path[prod_realpath_len] = '\0'; + ctx->prod_real_len = prod_realpath_len; + ctx->prod_real_hash = hash_str(ctx->prod_real_path, DYAD_SEED); + if (ctx->prod_real_hash == 0u) { + DYAD_LOG_ERROR(ctx, "Could not hash Producer real path!\n"); + rc = DYAD_RC_BADMANAGEDPATH; + goto set_prod_path_region_failed; + } + } + + if (ctx->rank == 0) { + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: prod_managed_path %s", + ctx->prod_managed_path); + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: prod_managed_len %u", + ctx->prod_managed_len); + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: prod_managed_hash %u", + ctx->prod_managed_hash); + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: prod_real_path %s", + ctx->prod_real_path); + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: prod_real_len %u", + ctx->prod_real_len); + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: prod_real_hash %u", + ctx->prod_real_hash); + } + } set_prod_path_region_failed:; set_prod_path_region_finish:; - if (ctx) { - ctx->reenter = reenter_backup; - } - DYAD_C_FUNCTION_END (); - return rc; + if (ctx) { + ctx->reenter = reenter_backup; + } + DYAD_C_FUNCTION_END(); + return rc; } -DYAD_DLL_EXPORTED dyad_rc_t dyad_set_cons_path (const char* cons_managed_path) -{ - dyad_rc_t rc = DYAD_RC_OK; - bool reenter_backup = false; - size_t cons_path_len = 0ul; - size_t cons_realpath_len = 0ul; - char cons_real_path[PATH_MAX + 1] = {'\0'}; - - if (!ctx) { - DYAD_LOG_STDERR ("Empty Consumer managed path to allocate!\n"); - rc = DYAD_RC_NOCTX; - goto set_cons_path_region_finish; - } else { - reenter_backup = ctx->reenter; - ctx->reenter = false; - } - DYAD_C_FUNCTION_START (); - - if (ctx->cons_managed_path) { - free (ctx->cons_managed_path); - ctx->cons_managed_path = NULL; - ctx->cons_managed_len = 0u; - ctx->cons_managed_hash = 0u; - } +DYAD_DLL_EXPORTED dyad_rc_t dyad_set_cons_path(const char *cons_managed_path) { + dyad_rc_t rc = DYAD_RC_OK; + bool reenter_backup = false; + size_t cons_path_len = 0ul; + size_t cons_realpath_len = 0ul; + char cons_real_path[PATH_MAX + 1] = {'\0'}; - if (ctx->cons_real_path) { - free (ctx->cons_real_path); - ctx->cons_real_path = NULL; - ctx->cons_real_len = 0u; - ctx->cons_real_hash = 0u; + if (!ctx) { + DYAD_LOG_STDERR("Empty Consumer managed path to allocate!\n"); + rc = DYAD_RC_NOCTX; + goto set_cons_path_region_finish; + } else { + reenter_backup = ctx->reenter; + ctx->reenter = false; + } + DYAD_C_FUNCTION_START(); + + if (ctx->cons_managed_path) { + free(ctx->cons_managed_path); + ctx->cons_managed_path = NULL; + ctx->cons_managed_len = 0u; + ctx->cons_managed_hash = 0u; + } + + if (ctx->cons_real_path) { + free(ctx->cons_real_path); + ctx->cons_real_path = NULL; + ctx->cons_real_len = 0u; + ctx->cons_real_hash = 0u; + } + + DYAD_LOG_INFO(ctx, "DYAD_CORE: saving consucer path"); + if (cons_managed_path == NULL) { + DYAD_LOG_INFO(ctx, "DYAD_CORE: consucer path is not provided"); + ctx->cons_managed_path = NULL; + ctx->cons_managed_len = 0u; + ctx->cons_managed_hash = 0u; + ctx->cons_real_path = NULL; + ctx->cons_real_len = 0u; + ctx->cons_real_hash = 0u; + } else { + if ((cons_path_len = strlen(cons_managed_path)) == 0ul) { + DYAD_LOG_ERROR(ctx, "Empty Consumer managed path to allocate!\n"); + rc = DYAD_RC_BADMANAGEDPATH; + goto set_cons_path_region_failed; + } + ctx->cons_managed_path = (char *)calloc(cons_path_len + 1, sizeof(char)); + if (ctx->cons_managed_path == NULL) { + DYAD_LOG_ERROR(ctx, + "Could not allocate buffer for Consumer managed path!\n"); + rc = DYAD_RC_SYSFAIL; + goto set_cons_path_region_failed; + } + if (!memcpy(ctx->cons_managed_path, cons_managed_path, cons_path_len + 1)) { + DYAD_LOG_ERROR(ctx, "Could not copy Consumer managed path!\n"); + rc = DYAD_RC_SYSFAIL; + goto set_cons_path_region_failed; + } + ctx->cons_managed_len = cons_path_len; + ctx->cons_managed_hash = hash_str(ctx->cons_managed_path, DYAD_SEED); + if (ctx->cons_managed_hash == 0u) { + DYAD_LOG_ERROR(ctx, "Could not hash Consumer managed path!\n"); + rc = DYAD_RC_BADMANAGEDPATH; + goto set_cons_path_region_failed; + } + + if (!realpath(cons_managed_path, cons_real_path)) { + DYAD_LOG_DEBUG(ctx, "Could not get Consumer real path!\n"); + // perror ("Could not get Consumer real path!\n"); + // This is not necessarily an error. This can happen when the environment + // variables for both managed paths are set but consumer does not create + // the managed path of consucer or vice versa. `realpath()' fails if a + // path does not exist. + } else { + cons_realpath_len = strlen(cons_real_path); } - DYAD_LOG_INFO (ctx, "DYAD_CORE: saving consucer path"); - if (cons_managed_path == NULL) { - DYAD_LOG_INFO (ctx, "DYAD_CORE: consucer path is not provided"); - ctx->cons_managed_path = NULL; - ctx->cons_managed_len = 0u; - ctx->cons_managed_hash = 0u; - ctx->cons_real_path = NULL; - ctx->cons_real_len = 0u; - ctx->cons_real_hash = 0u; + // If the realpath is the same, we turn off checking against it as it is + // redundant + if (cons_realpath_len == 0u || + strncmp(cons_managed_path, cons_real_path, cons_path_len) == 0) { + DYAD_LOG_INFO(ctx, "DYAD_CORE: consucer real path is redundant"); + ctx->cons_real_path = NULL; + ctx->cons_real_len = 0u; + ctx->cons_real_hash = 0u; } else { - if ((cons_path_len = strlen (cons_managed_path)) == 0ul) { - DYAD_LOG_ERROR (ctx, "Empty Consumer managed path to allocate!\n"); - rc = DYAD_RC_BADMANAGEDPATH; - goto set_cons_path_region_failed; - } - ctx->cons_managed_path = (char*)calloc (cons_path_len + 1, sizeof (char)); - if (ctx->cons_managed_path == NULL) { - DYAD_LOG_ERROR (ctx, "Could not allocate buffer for Consumer managed path!\n"); - rc = DYAD_RC_SYSFAIL; - goto set_cons_path_region_failed; - } - if (!memcpy (ctx->cons_managed_path, cons_managed_path, cons_path_len + 1)) { - DYAD_LOG_ERROR (ctx, "Could not copy Consumer managed path!\n"); - rc = DYAD_RC_SYSFAIL; - goto set_cons_path_region_failed; - } - ctx->cons_managed_len = cons_path_len; - ctx->cons_managed_hash = hash_str (ctx->cons_managed_path, DYAD_SEED); - if (ctx->cons_managed_hash == 0u) { - DYAD_LOG_ERROR (ctx, "Could not hash Consumer managed path!\n"); - rc = DYAD_RC_BADMANAGEDPATH; - goto set_cons_path_region_failed; - } - - if (!realpath (cons_managed_path, cons_real_path)) { - DYAD_LOG_DEBUG (ctx, "Could not get Consumer real path!\n"); - // perror ("Could not get Consumer real path!\n"); - // This is not necessarily an error. This can happen when the environment - // variables for both managed paths are set but consumer does not create - // the managed path of consucer or vice versa. `realpath()' fails if a - // path does not exist. - } else { - cons_realpath_len = strlen (cons_real_path); - } - - // If the realpath is the same, we turn off checking against it as it is redundant - if (cons_realpath_len == 0u - || strncmp (cons_managed_path, cons_real_path, cons_path_len) == 0) { - DYAD_LOG_INFO (ctx, "DYAD_CORE: consucer real path is redundant"); - ctx->cons_real_path = NULL; - ctx->cons_real_len = 0u; - ctx->cons_real_hash = 0u; - } else { - ctx->cons_real_path = (char*)calloc (cons_realpath_len + 1, sizeof (char)); - if (ctx->cons_real_path == NULL || cons_realpath_len == 0ul) { - DYAD_LOG_ERROR (ctx, "Could not allocate buffer for Consumer managed path!\n"); - rc = DYAD_RC_SYSFAIL; - goto set_cons_path_region_failed; - } - - if (!memcpy (ctx->cons_real_path, cons_real_path, cons_realpath_len)) { - DYAD_LOG_ERROR (ctx, "Could not copy Consumer real path!\n"); - rc = DYAD_RC_SYSFAIL; - goto set_cons_path_region_failed; - } - ctx->cons_real_path[cons_realpath_len] = '\0'; - ctx->cons_real_len = cons_realpath_len; - ctx->cons_real_hash = hash_str (ctx->cons_real_path, DYAD_SEED); - if (ctx->cons_real_hash == 0u) { - DYAD_LOG_ERROR (ctx, "Could not hash Consumer real path!\n"); - rc = DYAD_RC_BADMANAGEDPATH; - goto set_cons_path_region_failed; - } - } - - if (ctx->rank == 0) { - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: cons_managed_path %s", ctx->cons_managed_path); - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: cons_managed_len %u", ctx->cons_managed_len); - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: cons_managed_hash %u", ctx->cons_managed_hash); - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: cons_real_path %s", ctx->cons_real_path); - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: cons_real_len %u", ctx->cons_real_len); - DYAD_LOG_INFO (ctx, "DYAD_CORE INIT: cons_real_hash %u", ctx->cons_real_hash); - } - } + ctx->cons_real_path = (char *)calloc(cons_realpath_len + 1, sizeof(char)); + if (ctx->cons_real_path == NULL || cons_realpath_len == 0ul) { + DYAD_LOG_ERROR( + ctx, "Could not allocate buffer for Consumer managed path!\n"); + rc = DYAD_RC_SYSFAIL; + goto set_cons_path_region_failed; + } + + if (!memcpy(ctx->cons_real_path, cons_real_path, cons_realpath_len)) { + DYAD_LOG_ERROR(ctx, "Could not copy Consumer real path!\n"); + rc = DYAD_RC_SYSFAIL; + goto set_cons_path_region_failed; + } + ctx->cons_real_path[cons_realpath_len] = '\0'; + ctx->cons_real_len = cons_realpath_len; + ctx->cons_real_hash = hash_str(ctx->cons_real_path, DYAD_SEED); + if (ctx->cons_real_hash == 0u) { + DYAD_LOG_ERROR(ctx, "Could not hash Consumer real path!\n"); + rc = DYAD_RC_BADMANAGEDPATH; + goto set_cons_path_region_failed; + } + } + + if (ctx->rank == 0) { + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: cons_managed_path %s", + ctx->cons_managed_path); + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: cons_managed_len %u", + ctx->cons_managed_len); + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: cons_managed_hash %u", + ctx->cons_managed_hash); + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: cons_real_path %s", + ctx->cons_real_path); + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: cons_real_len %u", + ctx->cons_real_len); + DYAD_LOG_INFO(ctx, "DYAD_CORE INIT: cons_real_hash %u", + ctx->cons_real_hash); + } + } set_cons_path_region_failed:; set_cons_path_region_finish:; - if (ctx) { - ctx->reenter = reenter_backup; - } - DYAD_C_FUNCTION_END (); - return rc; + if (ctx) { + ctx->reenter = reenter_backup; + } + DYAD_C_FUNCTION_END(); + return rc; } -DYAD_DLL_EXPORTED dyad_rc_t dyad_clear () -{ - DYAD_C_FUNCTION_START (); - dyad_rc_t rc = DYAD_RC_OK; - DYAD_LOG_STDERR ("DYAD context is being cleared!\n"); - if (ctx == NULL) { - rc = DYAD_RC_OK; - goto clear_region_finish; - } - dyad_dtl_finalize (ctx); - if (ctx->h != NULL) { - flux_close (ctx->h); - ctx->h = NULL; - } - if (ctx->kvs_namespace != NULL) { - free (ctx->kvs_namespace); - ctx->kvs_namespace = NULL; - } - if (ctx->prod_managed_path != NULL) { - free (ctx->prod_managed_path); - ctx->prod_managed_path = NULL; - } - if (ctx->prod_real_path != NULL) { - free (ctx->prod_real_path); - ctx->prod_real_path = NULL; - } - if (ctx->cons_managed_path != NULL) { - free (ctx->cons_managed_path); - ctx->cons_managed_path = NULL; - } - if (ctx->cons_real_path != NULL) { - free (ctx->cons_real_path); - ctx->cons_real_path = NULL; - } +DYAD_DLL_EXPORTED dyad_rc_t dyad_clear() { + DYAD_C_FUNCTION_START(); + dyad_rc_t rc = DYAD_RC_OK; + DYAD_LOG_STDERR("DYAD context is being cleared!\n"); + if (ctx == NULL) { rc = DYAD_RC_OK; + goto clear_region_finish; + } + dyad_dtl_finalize(ctx); + if (ctx->h != NULL) { + flux_close(ctx->h); + ctx->h = NULL; + } + if (ctx->kvs_namespace != NULL) { + free(ctx->kvs_namespace); + ctx->kvs_namespace = NULL; + } + if (ctx->prod_managed_path != NULL) { + free(ctx->prod_managed_path); + ctx->prod_managed_path = NULL; + } + if (ctx->prod_real_path != NULL) { + free(ctx->prod_real_path); + ctx->prod_real_path = NULL; + } + if (ctx->cons_managed_path != NULL) { + free(ctx->cons_managed_path); + ctx->cons_managed_path = NULL; + } + if (ctx->cons_real_path != NULL) { + free(ctx->cons_real_path); + ctx->cons_real_path = NULL; + } + rc = DYAD_RC_OK; clear_region_finish:; - DYAD_C_FUNCTION_END (); + DYAD_C_FUNCTION_END(); #ifdef DYAD_PROFILER_DFTRACER - DFTRACER_C_FINI (); + DFTRACER_C_FINI(); #endif - return rc; + return rc; } -DYAD_DLL_EXPORTED dyad_rc_t dyad_finalize () -{ - DYAD_C_FUNCTION_START (); - dyad_rc_t rc = DYAD_RC_OK; - //DYAD_LOG_STDERR ("DYAD context is being destroyed!\n"); - if (ctx == NULL) { - rc = DYAD_RC_OK; - goto finalize_region_finish; - } - dyad_clear (); - free (ctx); - ctx = NULL; +DYAD_DLL_EXPORTED dyad_rc_t dyad_finalize() { + DYAD_C_FUNCTION_START(); + dyad_rc_t rc = DYAD_RC_OK; + // DYAD_LOG_STDERR ("DYAD context is being destroyed!\n"); + if (ctx == NULL) { rc = DYAD_RC_OK; + goto finalize_region_finish; + } + dyad_clear(); + free(ctx); + ctx = NULL; + rc = DYAD_RC_OK; finalize_region_finish:; - DYAD_C_FUNCTION_END (); + DYAD_C_FUNCTION_END(); #ifdef DYAD_PROFILER_DFTRACER - DFTRACER_C_FINI (); + DFTRACER_C_FINI(); #endif - return rc; + return rc; } diff --git a/src/dyad/dtl/CMakeLists.txt b/src/dyad/dtl/CMakeLists.txt index 528ece78..e02eab19 100644 --- a/src/dyad/dtl/CMakeLists.txt +++ b/src/dyad/dtl/CMakeLists.txt @@ -1,9 +1,9 @@ # DTL Interface set(DTL_SRC ${CMAKE_CURRENT_SOURCE_DIR}/dyad_dtl_api.c) set(DTL_PRIVATE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/dyad_dtl_api.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_structures.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_dtl.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_rc.h) + ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_structures_int.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/common/dyad_dtl.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/common/dyad_rc.h) set(DTL_PUBLIC_HEADERS) # Flux implementation for DTL @@ -29,26 +29,21 @@ endif() add_library(${PROJECT_NAME}_dtl SHARED ${DTL_SRC} ${DTL_PUBLIC_HEADERS} ${DTL_PRIVATE_HEADERS}) target_link_libraries(${PROJECT_NAME}_dtl PRIVATE ${PROJECT_NAME}_utils Jansson::Jansson flux::core flux::optparse) set_target_properties(${PROJECT_NAME}_dtl PROPERTIES CMAKE_INSTALL_RPATH - "${CMAKE_INSTALL_PREFIX}/${DYAD_LIBDIR}") + "${DYAD_INSTALL_LIBDIR}") if(DYAD_ENABLE_UCX_DTL OR DYAD_ENABLE_UCX_DATA_RMA) target_link_libraries(${PROJECT_NAME}_dtl PRIVATE ucx::ucp ucx::ucs) target_include_directories(${PROJECT_NAME}_dtl SYSTEM PUBLIC ${ucx_INCLUDE_DIRS}) - if (TARGET DYAD_CXX_FLAGS_werror) - target_link_libraries(${PROJECT_NAME}_dtl PRIVATE DYAD_CXX_FLAGS_werror) - endif () endif() target_compile_definitions(${PROJECT_NAME}_dtl PUBLIC DYAD_HAS_CONFIG) target_include_directories(${PROJECT_NAME}_dtl PUBLIC $ - $) + $) target_include_directories(${PROJECT_NAME}_dtl SYSTEM PUBLIC ${JANSSON_INCLUDE_DIRS}) target_include_directories(${PROJECT_NAME}_dtl SYSTEM PUBLIC ${FluxCore_INCLUDE_DIRS}) -if (TARGET DYAD_C_FLAGS_werror) - target_link_libraries(${PROJECT_NAME}_dtl PRIVATE DYAD_C_FLAGS_werror) -endif () +dyad_add_werror_if_needed(${PROJECT_NAME}_dtl) if(DYAD_LOGGER STREQUAL "CPP_LOGGER") target_link_libraries(${PROJECT_NAME}_dtl PRIVATE ${CPP_LOGGER_LIBRARIES}) endif() @@ -59,9 +54,9 @@ endif() install( TARGETS ${PROJECT_NAME}_dtl EXPORT ${DYAD_EXPORTED_TARGETS} - LIBRARY DESTINATION ${DYAD_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${DYAD_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${DYAD_INSTALL_BIN_DIR} + LIBRARY DESTINATION ${DYAD_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${DYAD_INSTALL_LIBDIR} + RUNTIME DESTINATION ${DYAD_INSTALL_BINDIR} ) if(NOT ${DTL_PUBLIC_HEADERS} STREQUAL "") dyad_install_headers("${DTL_PUBLIC_HEADERS}" ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/src/dyad/dtl/dyad_dtl_api.h b/src/dyad/dtl/dyad_dtl_api.h index 92081bcf..62c6ca5d 100644 --- a/src/dyad/dtl/dyad_dtl_api.h +++ b/src/dyad/dtl/dyad_dtl_api.h @@ -7,12 +7,15 @@ #error "no config" #endif +// clang-format off #include #include -#include +#include #include #include #include +// clang-format on + #ifdef __cplusplus #include @@ -27,36 +30,34 @@ struct dyad_dtl_flux; // Union type to store the underlying DTL contexts union dyad_dtl_private { - struct dyad_dtl_ucx* ucx_dtl_handle; - struct dyad_dtl_flux* flux_dtl_handle; + struct dyad_dtl_ucx *ucx_dtl_handle; + struct dyad_dtl_flux *flux_dtl_handle; } __attribute__((aligned(16))); typedef union dyad_dtl_private dyad_dtl_private_t; struct dyad_dtl { - dyad_dtl_private_t private_dtl; - dyad_dtl_mode_t mode; - dyad_rc_t (*rpc_pack) (const dyad_ctx_t* ctx, - const char* upath, - uint32_t producer_rank, - json_t** packed_obj); - dyad_rc_t (*rpc_unpack) (const dyad_ctx_t* ctx, const flux_msg_t* packed_obj, char** upath); - dyad_rc_t (*rpc_respond) (const dyad_ctx_t* ctx, const flux_msg_t* orig_msg); - dyad_rc_t (*rpc_recv_response) (const dyad_ctx_t* ctx, flux_future_t* f); - dyad_rc_t (*get_buffer) (const dyad_ctx_t* ctx, size_t data_size, void** data_buf); - dyad_rc_t (*return_buffer) (const dyad_ctx_t* ctx, void** data_buf); - dyad_rc_t (*establish_connection) (const dyad_ctx_t* ctx); - dyad_rc_t (*send) (const dyad_ctx_t* ctx, void* buf, size_t buflen); - dyad_rc_t (*recv) (const dyad_ctx_t* ctx, void** buf, size_t* buflen); - dyad_rc_t (*close_connection) (const dyad_ctx_t* ctx); + dyad_dtl_private_t private_dtl; + dyad_dtl_mode_t mode; + dyad_rc_t (*rpc_pack)(const dyad_ctx_t *ctx, const char *upath, + uint32_t producer_rank, json_t **packed_obj); + dyad_rc_t (*rpc_unpack)(const dyad_ctx_t *ctx, const flux_msg_t *packed_obj, + char **upath); + dyad_rc_t (*rpc_respond)(const dyad_ctx_t *ctx, const flux_msg_t *orig_msg); + dyad_rc_t (*rpc_recv_response)(const dyad_ctx_t *ctx, flux_future_t *f); + dyad_rc_t (*get_buffer)(const dyad_ctx_t *ctx, size_t data_size, + void **data_buf); + dyad_rc_t (*return_buffer)(const dyad_ctx_t *ctx, void **data_buf); + dyad_rc_t (*establish_connection)(const dyad_ctx_t *ctx); + dyad_rc_t (*send)(const dyad_ctx_t *ctx, void *buf, size_t buflen); + dyad_rc_t (*recv)(const dyad_ctx_t *ctx, void **buf, size_t *buflen); + dyad_rc_t (*close_connection)(const dyad_ctx_t *ctx); } __attribute__((aligned(256))); typedef struct dyad_dtl dyad_dtl_t; -dyad_rc_t dyad_dtl_init (dyad_ctx_t* ctx, - dyad_dtl_mode_t mode, - dyad_dtl_comm_mode_t comm_mode, - bool debug); +dyad_rc_t dyad_dtl_init(dyad_ctx_t *ctx, dyad_dtl_mode_t mode, + dyad_dtl_comm_mode_t comm_mode, bool debug); -dyad_rc_t dyad_dtl_finalize (dyad_ctx_t* ctx); +dyad_rc_t dyad_dtl_finalize(dyad_ctx_t *ctx); #ifdef __cplusplus } diff --git a/src/dyad/dtl/ucx_ep_cache.cpp b/src/dyad/dtl/ucx_ep_cache.cpp index 7d33c619..8493793b 100644 --- a/src/dyad/dtl/ucx_ep_cache.cpp +++ b/src/dyad/dtl/ucx_ep_cache.cpp @@ -4,10 +4,12 @@ #error "no config" #endif +// clang-format off #include #include #include -#include +#include +// clang-format on #include #include @@ -18,224 +20,220 @@ using key_type = uint64_t; using cache_type = std::unordered_map; static void __attribute__((unused)) -dyad_ucx_ep_err_handler (void* arg, ucp_ep_h ep, ucs_status_t status) -{ - DYAD_C_FUNCTION_START(); - dyad_ctx_t __attribute__((unused)) *ctx = (dyad_ctx_t*)arg; - DYAD_LOG_ERROR (ctx, "An error occured on the UCP endpoint (status = %d)", status); - DYAD_C_FUNCTION_END(); +dyad_ucx_ep_err_handler(void *arg, ucp_ep_h ep, ucs_status_t status) { + DYAD_C_FUNCTION_START(); + dyad_ctx_t __attribute__((unused)) *ctx = (dyad_ctx_t *)arg; + DYAD_LOG_ERROR(ctx, "An error occured on the UCP endpoint (status = %d)", + status); + DYAD_C_FUNCTION_END(); } -dyad_rc_t ucx_connect (const dyad_ctx_t *ctx, - ucp_worker_h worker, - const ucp_address_t* addr, - ucp_ep_h* ep) -{ - DYAD_C_FUNCTION_START(); - dyad_rc_t rc = DYAD_RC_OK; - ucp_ep_params_t params; - ucs_status_t status = UCS_OK; - params.field_mask = UCP_EP_PARAM_FIELD_REMOTE_ADDRESS; - params.address = addr; - status = ucp_ep_create (worker, ¶ms, ep); - if (UCX_STATUS_FAIL (status)) { - DYAD_LOG_ERROR (ctx, "ucp_ep_create failed with status %d", (int)status); - rc = DYAD_RC_UCXCOMM_FAIL; - goto ucx_connect_done; - } - if (*ep == NULL) { - DYAD_LOG_ERROR (ctx, "ucp_ep_create succeeded, but returned a NULL endpoint"); - rc = DYAD_RC_UCXCOMM_FAIL; - goto ucx_connect_done; - } +dyad_rc_t ucx_connect(const dyad_ctx_t *ctx, ucp_worker_h worker, + const ucp_address_t *addr, ucp_ep_h *ep) { + DYAD_C_FUNCTION_START(); + dyad_rc_t rc = DYAD_RC_OK; + ucp_ep_params_t params; + ucs_status_t status = UCS_OK; + params.field_mask = UCP_EP_PARAM_FIELD_REMOTE_ADDRESS; + params.address = addr; + status = ucp_ep_create(worker, ¶ms, ep); + if (UCX_STATUS_FAIL(status)) { + DYAD_LOG_ERROR(ctx, "ucp_ep_create failed with status %d", (int)status); + rc = DYAD_RC_UCXCOMM_FAIL; + goto ucx_connect_done; + } + if (*ep == NULL) { + DYAD_LOG_ERROR(ctx, + "ucp_ep_create succeeded, but returned a NULL endpoint"); + rc = DYAD_RC_UCXCOMM_FAIL; + goto ucx_connect_done; + } ucx_connect_done:; - DYAD_C_FUNCTION_END(); - return rc; + DYAD_C_FUNCTION_END(); + return rc; } -dyad_rc_t ucx_disconnect (const dyad_ctx_t *ctx, ucp_worker_h worker, ucp_ep_h ep) -{ - DYAD_C_FUNCTION_START(); - dyad_rc_t rc = DYAD_RC_OK; - ucs_status_t status = UCS_OK; - ucs_status_ptr_t stat_ptr; - if (ep != NULL) { - // ucp_tag_send_sync_nbx is the prefered version of this send - // since UCX 1.9 However, some systems (e.g., Lassen) may have - // an older verison This conditional compilation will use - // ucp_tag_send_sync_nbx if using UCX 1.9+, and it will use the - // deprecated ucp_tag_send_sync_nb if using UCX < 1.9. +dyad_rc_t ucx_disconnect(const dyad_ctx_t *ctx, ucp_worker_h worker, + ucp_ep_h ep) { + DYAD_C_FUNCTION_START(); + dyad_rc_t rc = DYAD_RC_OK; + ucs_status_t status = UCS_OK; + ucs_status_ptr_t stat_ptr; + if (ep != NULL) { + // ucp_tag_send_sync_nbx is the prefered version of this send + // since UCX 1.9 However, some systems (e.g., Lassen) may have + // an older verison This conditional compilation will use + // ucp_tag_send_sync_nbx if using UCX 1.9+, and it will use the + // deprecated ucp_tag_send_sync_nb if using UCX < 1.9. #if UCP_API_VERSION >= UCP_VERSION(1, 10) - ucp_request_param_t close_params; - close_params.op_attr_mask = UCP_OP_ATTR_FIELD_FLAGS; - close_params.flags = UCP_EP_CLOSE_FLAG_FORCE; - stat_ptr = ucp_ep_close_nbx (ep, &close_params); + ucp_request_param_t close_params; + close_params.op_attr_mask = UCP_OP_ATTR_FIELD_FLAGS; + close_params.flags = UCP_EP_CLOSE_FLAG_FORCE; + stat_ptr = ucp_ep_close_nbx(ep, &close_params); #else - // TODO change to FORCE if we decide to enable err handleing - // mode - stat_ptr = ucp_ep_close_nb (ep, UCP_EP_CLOSE_MODE_FORCE); + // TODO change to FORCE if we decide to enable err handleing + // mode + stat_ptr = ucp_ep_close_nb(ep, UCP_EP_CLOSE_MODE_FORCE); #endif - // Don't use dyad_ucx_request_wait here because ep_close behaves - // differently than other UCX calls - if (stat_ptr != NULL) { - // Endpoint close is in-progress. - // Wait until finished - if (UCS_PTR_IS_PTR (stat_ptr)) { - do { - ucp_worker_progress (worker); - status = ucp_request_check_status (stat_ptr); - } while (status == UCS_INPROGRESS); - ucp_request_free (stat_ptr); - } - // An error occurred during endpoint closure - // However, the endpoint can no longer be used - // Get the status code for reporting - else { - status = UCS_PTR_STATUS (stat_ptr); - } - if (UCX_STATUS_FAIL (status)) { - rc = DYAD_RC_UCXEP_FAIL; - goto ucx_disconnect_region_finish; - } - } + // Don't use dyad_ucx_request_wait here because ep_close behaves + // differently than other UCX calls + if (stat_ptr != NULL) { + // Endpoint close is in-progress. + // Wait until finished + if (UCS_PTR_IS_PTR(stat_ptr)) { + do { + ucp_worker_progress(worker); + status = ucp_request_check_status(stat_ptr); + } while (status == UCS_INPROGRESS); + ucp_request_free(stat_ptr); + } + // An error occurred during endpoint closure + // However, the endpoint can no longer be used + // Get the status code for reporting + else { + status = UCS_PTR_STATUS(stat_ptr); + } + if (UCX_STATUS_FAIL(status)) { + rc = DYAD_RC_UCXEP_FAIL; + goto ucx_disconnect_region_finish; + } } + } ucx_disconnect_region_finish: - DYAD_C_FUNCTION_END(); - return rc; + DYAD_C_FUNCTION_END(); + return rc; } -dyad_rc_t dyad_ucx_ep_cache_init (const dyad_ctx_t *ctx, ucx_ep_cache_h* cache) -{ - DYAD_C_FUNCTION_START(); - dyad_rc_t rc = DYAD_RC_OK; - if (cache == nullptr || *cache != nullptr) { - rc = DYAD_RC_BADBUF; - goto ucx_ep_cache_init_done; - } - *cache = reinterpret_cast (new (std::nothrow) cache_type ()); - if (*cache == nullptr) { - rc = DYAD_RC_SYSFAIL; - goto ucx_ep_cache_init_done; - } +dyad_rc_t dyad_ucx_ep_cache_init(const dyad_ctx_t *ctx, ucx_ep_cache_h *cache) { + DYAD_C_FUNCTION_START(); + dyad_rc_t rc = DYAD_RC_OK; + if (cache == nullptr || *cache != nullptr) { + rc = DYAD_RC_BADBUF; + goto ucx_ep_cache_init_done; + } + *cache = reinterpret_cast(new (std::nothrow) cache_type()); + if (*cache == nullptr) { + rc = DYAD_RC_SYSFAIL; + goto ucx_ep_cache_init_done; + } ucx_ep_cache_init_done:; - DYAD_C_FUNCTION_END(); - return rc; + DYAD_C_FUNCTION_END(); + return rc; } -dyad_rc_t dyad_ucx_ep_cache_find (const dyad_ctx_t *ctx, - const ucx_ep_cache_h cache, - const ucp_address_t* addr, - const size_t addr_size, - ucp_ep_h* ep) -{ - DYAD_C_FUNCTION_START(); - dyad_rc_t rc = DYAD_RC_OK; - if (ep == nullptr || *ep != nullptr) { - rc = DYAD_RC_BADBUF; - goto ucx_ep_cache_find_done; - } - try { - const auto* cpp_cache = reinterpret_cast (cache); - auto key = ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key; - auto cache_it = cpp_cache->find (key); - if (cache_it == cpp_cache->cend ()) { - *ep = nullptr; - rc = DYAD_RC_NOTFOUND; - } else { - *ep = cache_it->second; - rc = DYAD_RC_OK; - } - } catch (...) { - *ep = nullptr; - rc = DYAD_RC_SYSFAIL; +dyad_rc_t dyad_ucx_ep_cache_find(const dyad_ctx_t *ctx, + const ucx_ep_cache_h cache, + const ucp_address_t *addr, + const size_t addr_size, ucp_ep_h *ep) { + DYAD_C_FUNCTION_START(); + dyad_rc_t rc = DYAD_RC_OK; + if (ep == nullptr || *ep != nullptr) { + rc = DYAD_RC_BADBUF; + goto ucx_ep_cache_find_done; + } + try { + const auto *cpp_cache = reinterpret_cast(cache); + auto key = ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key; + auto cache_it = cpp_cache->find(key); + if (cache_it == cpp_cache->cend()) { + *ep = nullptr; + rc = DYAD_RC_NOTFOUND; + } else { + *ep = cache_it->second; + rc = DYAD_RC_OK; } + } catch (...) { + *ep = nullptr; + rc = DYAD_RC_SYSFAIL; + } ucx_ep_cache_find_done:; - DYAD_C_FUNCTION_END(); - return rc; + DYAD_C_FUNCTION_END(); + return rc; } -dyad_rc_t dyad_ucx_ep_cache_insert (const dyad_ctx_t *ctx, - ucx_ep_cache_h cache, - const ucp_address_t* addr, - const size_t addr_size, - ucp_worker_h worker) -{ - DYAD_C_FUNCTION_START(); - dyad_rc_t rc = DYAD_RC_OK; - try { - cache_type* cpp_cache = reinterpret_cast (cache); - uint64_t key = ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key; - DYAD_C_FUNCTION_UPDATE_INT("cons_key", ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key) - auto cache_it = cpp_cache->find (key); - if (cache_it != cpp_cache->end ()) { - rc = DYAD_RC_OK; - } else { - DYAD_LOG_INFO (ctx, "No cache entry found. Creating new connection"); - rc = ucx_connect (ctx, worker, addr, &ctx->dtl_handle->private_dtl.ucx_dtl_handle->ep); - if (!DYAD_IS_ERROR (rc)) { - cpp_cache->insert_or_assign(key, ctx->dtl_handle->private_dtl.ucx_dtl_handle->ep); - rc = DYAD_RC_OK; - } - } - } catch (...) { - rc = DYAD_RC_SYSFAIL; +dyad_rc_t dyad_ucx_ep_cache_insert(const dyad_ctx_t *ctx, ucx_ep_cache_h cache, + const ucp_address_t *addr, + const size_t addr_size, + ucp_worker_h worker) { + DYAD_C_FUNCTION_START(); + dyad_rc_t rc = DYAD_RC_OK; + try { + cache_type *cpp_cache = reinterpret_cast(cache); + uint64_t key = + ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key; + DYAD_C_FUNCTION_UPDATE_INT( + "cons_key", + ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key) + auto cache_it = cpp_cache->find(key); + if (cache_it != cpp_cache->end()) { + rc = DYAD_RC_OK; + } else { + DYAD_LOG_INFO(ctx, "No cache entry found. Creating new connection"); + rc = ucx_connect(ctx, worker, addr, + &ctx->dtl_handle->private_dtl.ucx_dtl_handle->ep); + if (!DYAD_IS_ERROR(rc)) { + cpp_cache->insert_or_assign( + key, ctx->dtl_handle->private_dtl.ucx_dtl_handle->ep); + rc = DYAD_RC_OK; + } } - DYAD_C_FUNCTION_END(); - return rc; + } catch (...) { + rc = DYAD_RC_SYSFAIL; + } + DYAD_C_FUNCTION_END(); + return rc; } -static inline cache_type::iterator cache_remove_impl (const dyad_ctx_t *ctx, - cache_type* cache, - cache_type::iterator it, - ucp_worker_h worker) -{ - DYAD_C_FUNCTION_START(); - if (it != cache->end ()) { - ucx_disconnect (ctx, worker, it->second); - // The UCP address was allocated with 'malloc' while unpacking - // the RPC message. So, we extract it from the key and free - // it after erasing the iterator - auto next_it = cache->erase (it); - DYAD_C_FUNCTION_END(); - return next_it; - } +static inline cache_type::iterator cache_remove_impl(const dyad_ctx_t *ctx, + cache_type *cache, + cache_type::iterator it, + ucp_worker_h worker) { + DYAD_C_FUNCTION_START(); + if (it != cache->end()) { + ucx_disconnect(ctx, worker, it->second); + // The UCP address was allocated with 'malloc' while unpacking + // the RPC message. So, we extract it from the key and free + // it after erasing the iterator + auto next_it = cache->erase(it); DYAD_C_FUNCTION_END(); - return cache->end (); + return next_it; + } + DYAD_C_FUNCTION_END(); + return cache->end(); } -dyad_rc_t dyad_ucx_ep_cache_remove (const dyad_ctx_t *ctx, - ucx_ep_cache_h cache, - const ucp_address_t* addr, - const size_t addr_size, - ucp_worker_h worker) -{ - DYAD_C_FUNCTION_START(); - dyad_rc_t rc = DYAD_RC_OK; - try { - cache_type* cpp_cache = reinterpret_cast (cache); - auto key = ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key; - cache_type::iterator cache_it = cpp_cache->find (key); - cache_remove_impl (ctx, cpp_cache, cache_it, worker); - rc = DYAD_RC_OK; - } catch (...) { - rc = DYAD_RC_SYSFAIL; - } - DYAD_C_FUNCTION_END(); - return rc; +dyad_rc_t dyad_ucx_ep_cache_remove(const dyad_ctx_t *ctx, ucx_ep_cache_h cache, + const ucp_address_t *addr, + const size_t addr_size, + ucp_worker_h worker) { + DYAD_C_FUNCTION_START(); + dyad_rc_t rc = DYAD_RC_OK; + try { + cache_type *cpp_cache = reinterpret_cast(cache); + auto key = ctx->dtl_handle->private_dtl.ucx_dtl_handle->consumer_conn_key; + cache_type::iterator cache_it = cpp_cache->find(key); + cache_remove_impl(ctx, cpp_cache, cache_it, worker); + rc = DYAD_RC_OK; + } catch (...) { + rc = DYAD_RC_SYSFAIL; + } + DYAD_C_FUNCTION_END(); + return rc; } -dyad_rc_t dyad_ucx_ep_cache_finalize (const dyad_ctx_t *ctx, ucx_ep_cache_h* cache, ucp_worker_h worker) -{ - DYAD_C_FUNCTION_START(); - if (cache == nullptr || *cache == nullptr) { - return DYAD_RC_OK; - } - cache_type* cpp_cache = reinterpret_cast (*cache); - for (cache_type::iterator it = cpp_cache->begin (); it != cpp_cache->end ();) { - it = cache_remove_impl (ctx, cpp_cache, it, worker); - } - delete cpp_cache; - *cache = nullptr; - DYAD_C_FUNCTION_END(); +dyad_rc_t dyad_ucx_ep_cache_finalize(const dyad_ctx_t *ctx, + ucx_ep_cache_h *cache, + ucp_worker_h worker) { + DYAD_C_FUNCTION_START(); + if (cache == nullptr || *cache == nullptr) { return DYAD_RC_OK; + } + cache_type *cpp_cache = reinterpret_cast(*cache); + for (cache_type::iterator it = cpp_cache->begin(); it != cpp_cache->end();) { + it = cache_remove_impl(ctx, cpp_cache, it, worker); + } + delete cpp_cache; + *cache = nullptr; + DYAD_C_FUNCTION_END(); + return DYAD_RC_OK; } diff --git a/src/dyad/dtl/ucx_ep_cache.h b/src/dyad/dtl/ucx_ep_cache.h index aaa88c54..5ada63b8 100644 --- a/src/dyad/dtl/ucx_ep_cache.h +++ b/src/dyad/dtl/ucx_ep_cache.h @@ -7,11 +7,13 @@ #error "no config" #endif +// clang-format off #include #include -#include +#include #include #include +// clang-format on #ifdef __cplusplus extern "C" { @@ -21,40 +23,32 @@ extern "C" { // of UCX operations #define UCX_STATUS_FAIL(status) (status != UCS_OK) +dyad_rc_t ucx_connect(const dyad_ctx_t *ctx, ucp_worker_h worker, + const ucp_address_t *addr, ucp_ep_h *ep); - -dyad_rc_t ucx_connect (const dyad_ctx_t *ctx, - ucp_worker_h worker, - const ucp_address_t* addr, - ucp_ep_h* ep); - -dyad_rc_t ucx_disconnect (const dyad_ctx_t *ctx, ucp_worker_h worker, ucp_ep_h ep); +dyad_rc_t ucx_disconnect(const dyad_ctx_t *ctx, ucp_worker_h worker, + ucp_ep_h ep); // NOTE: in future, add option to configure replacement strategy -dyad_rc_t dyad_ucx_ep_cache_init (const dyad_ctx_t *ctx, ucx_ep_cache_h* cache); +dyad_rc_t dyad_ucx_ep_cache_init(const dyad_ctx_t *ctx, ucx_ep_cache_h *cache); // NOTE: not positive if UCP addresses are 100% unique by worker -dyad_rc_t dyad_ucx_ep_cache_find (const dyad_ctx_t *ctx, - const ucx_ep_cache_h cache, - const ucp_address_t* addr, - const size_t addr_size, - ucp_ep_h* ep); - -dyad_rc_t dyad_ucx_ep_cache_insert (const dyad_ctx_t *ctx, - ucx_ep_cache_h cache, - const ucp_address_t* addr, - const size_t addr_size, - ucp_worker_h worker); - -dyad_rc_t dyad_ucx_ep_cache_remove (const dyad_ctx_t *ctx, - ucx_ep_cache_h cache, - const ucp_address_t* addr, - const size_t addr_size, - ucp_worker_h worker); - -dyad_rc_t dyad_ucx_ep_cache_finalize (const dyad_ctx_t *ctx, - ucx_ep_cache_h* cache, - ucp_worker_h worker); +dyad_rc_t dyad_ucx_ep_cache_find(const dyad_ctx_t *ctx, + const ucx_ep_cache_h cache, + const ucp_address_t *addr, + const size_t addr_size, ucp_ep_h *ep); + +dyad_rc_t dyad_ucx_ep_cache_insert(const dyad_ctx_t *ctx, ucx_ep_cache_h cache, + const ucp_address_t *addr, + const size_t addr_size, ucp_worker_h worker); + +dyad_rc_t dyad_ucx_ep_cache_remove(const dyad_ctx_t *ctx, ucx_ep_cache_h cache, + const ucp_address_t *addr, + const size_t addr_size, ucp_worker_h worker); + +dyad_rc_t dyad_ucx_ep_cache_finalize(const dyad_ctx_t *ctx, + ucx_ep_cache_h *cache, + ucp_worker_h worker); #ifdef __cplusplus } diff --git a/src/dyad/modules/CMakeLists.txt b/src/dyad/modules/CMakeLists.txt index 2a5105aa..42d30233 100644 --- a/src/dyad/modules/CMakeLists.txt +++ b/src/dyad/modules/CMakeLists.txt @@ -1,21 +1,21 @@ set(DYAD_MODULE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/dyad.c) -set(DYAD_MODULE_PRIVATE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_envs.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_dtl.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_rc.h +set(DYAD_MODULE_PRIVATE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/common/dyad_envs.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/common/dyad_dtl.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/common/dyad_rc.h ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_logging.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_structures.h + ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_structures_int.h ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_profiler.h ${CMAKE_CURRENT_SOURCE_DIR}/../dtl/dyad_dtl_api.h ${CMAKE_CURRENT_SOURCE_DIR}/../utils/read_all.h ${CMAKE_CURRENT_SOURCE_DIR}/../utils/utils.h - ${CMAKE_CURRENT_SOURCE_DIR}/../core/dyad_ctx.h) + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/core/dyad_ctx.h) set(DYAD_MODULE_PUBLIC_HEADERS) add_library(${PROJECT_NAME} SHARED ${DYAD_MODULE_SRC} ${DYAD_MODULE_PRIVATE_HEADERS} ${DYAD_MODULE_PUBLIC_HEADERS}) set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "") set_target_properties(${PROJECT_NAME} PROPERTIES CMAKE_INSTALL_RPATH - "${CMAKE_INSTALL_PREFIX}/${DYAD_LIBDIR}") + "${DYAD_INSTALL_LIBDIR}") target_link_libraries(${PROJECT_NAME} PRIVATE Jansson::Jansson) target_link_libraries(${PROJECT_NAME} PRIVATE ${PROJECT_NAME}_dtl) target_link_libraries(${PROJECT_NAME} PRIVATE ${PROJECT_NAME}_ctx) @@ -24,25 +24,23 @@ target_compile_definitions(${PROJECT_NAME} PUBLIC BUILDING_DYAD=1) target_compile_definitions(${PROJECT_NAME} PUBLIC DYAD_HAS_CONFIG) target_include_directories(${PROJECT_NAME} PUBLIC $ - $) + $) target_include_directories(${PROJECT_NAME} SYSTEM PRIVATE ${JANSSON_INCLUDE_DIRS}) target_include_directories(${PROJECT_NAME} SYSTEM PRIVATE ${FluxCore_INCLUDE_DIRS}) add_executable(test_opt_parse test_opt_parse.c - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_envs.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_dtl.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_rc.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/common/dyad_envs.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/common/dyad_dtl.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/common/dyad_rc.h ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_logging.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_structures.h) + ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_structures_int.h) target_compile_definitions(test_opt_parse PUBLIC DYAD_HAS_CONFIG) target_include_directories(test_opt_parse PUBLIC $ - $) + $) -if (TARGET DYAD_C_FLAGS_werror) - target_link_libraries(${PROJECT_NAME} PRIVATE DYAD_C_FLAGS_werror) - target_link_libraries(test_opt_parse PRIVATE DYAD_C_FLAGS_werror) -endif () +dyad_add_werror_if_needed(${PROJECT_NAME}) +dyad_add_werror_if_needed(test_opt_parse) if(DYAD_PROFILER STREQUAL "PERFFLOW_ASPECT") target_link_libraries(${PROJECT_NAME} PRIVATE perfflowaspect::perfflowaspect) @@ -56,9 +54,9 @@ endif() install( TARGETS ${PROJECT_NAME} EXPORT ${DYAD_EXPORTED_TARGETS} - LIBRARY DESTINATION ${DYAD_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${DYAD_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${DYAD_INSTALL_BIN_DIR} + LIBRARY DESTINATION ${DYAD_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${DYAD_INSTALL_LIBDIR} + RUNTIME DESTINATION ${DYAD_INSTALL_BINDIR} ) if(NOT "${DYAD_MODULE_PUBLIC_HEADERS}" STREQUAL "") dyad_install_headers("${DYAD_MODULE_PUBLIC_HEADERS}" ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/src/dyad/modules/dyad.c b/src/dyad/modules/dyad.c index 31dce08a..1e922cab 100644 --- a/src/dyad/modules/dyad.c +++ b/src/dyad/modules/dyad.c @@ -14,17 +14,19 @@ #error "no config" #endif -// #include +// clang-format off +// #include #include #include #include #include #include -#include +#include #include #include #include #include +// clang-format on #if defined(__cplusplus) #include @@ -41,7 +43,7 @@ #include #include #include -#endif // defined(__cplusplus) +#endif // defined(__cplusplus) #include #include @@ -49,461 +51,482 @@ #include #include -#define TIME_DIFF(Tstart, Tend) \ - ((double)(1000000000L * ((Tend).tv_sec - (Tstart).tv_sec) + (Tend).tv_nsec - (Tstart).tv_nsec) \ - / 1000000000L) +#define TIME_DIFF(Tstart, Tend) \ + ((double)(1000000000L * ((Tend).tv_sec - (Tstart).tv_sec) + (Tend).tv_nsec - \ + (Tstart).tv_nsec) / \ + 1000000000L) struct dyad_mod_ctx { - flux_msg_handler_t **handlers; - dyad_ctx_t *ctx; + flux_msg_handler_t **handlers; + dyad_ctx_t *ctx; }; const struct dyad_mod_ctx dyad_mod_ctx_default = {NULL, NULL}; typedef struct dyad_mod_ctx dyad_mod_ctx_t; -static void dyad_mod_fini (void) __attribute__ ((destructor)); +static void dyad_mod_fini(void) __attribute__((destructor)); -void dyad_mod_fini (void) -{ - flux_t *h = flux_open (NULL, 0); +void dyad_mod_fini(void) { + flux_t *h = flux_open(NULL, 0); - if (h != NULL) { - } + if (h != NULL) { + } #ifdef DYAD_PROFILER_DFTRACER - DFTRACER_C_FINI (); + DFTRACER_C_FINI(); #endif } -static void freectx (void *arg) -{ - dyad_mod_ctx_t *mod_ctx = (dyad_mod_ctx_t *)arg; - flux_msg_handler_delvec (mod_ctx->handlers); - if (mod_ctx->ctx) { - dyad_ctx_fini (); - mod_ctx->ctx = NULL; - } - free (mod_ctx); +static void freectx(void *arg) { + dyad_mod_ctx_t *mod_ctx = (dyad_mod_ctx_t *)arg; + flux_msg_handler_delvec(mod_ctx->handlers); + if (mod_ctx->ctx) { + dyad_ctx_fini(); + mod_ctx->ctx = NULL; + } + free(mod_ctx); } -static dyad_mod_ctx_t *get_mod_ctx (flux_t *h) -{ - dyad_mod_ctx_t *mod_ctx = (dyad_mod_ctx_t *)flux_aux_get (h, "dyad"); - - if (!mod_ctx) { - mod_ctx = (dyad_mod_ctx_t *)malloc (sizeof (*mod_ctx)); - if (mod_ctx == NULL) { - DYAD_LOG_STDERR ("DYAD_MOD: could not allocate memory for context"); - goto getctx_error; - } - mod_ctx->handlers = NULL; - mod_ctx->ctx = NULL; - - if (flux_aux_set (h, "dyad", mod_ctx, freectx) < 0) { - DYAD_LOG_STDERR ("DYAD_MOD: flux_aux_set() failed!"); - goto getctx_error; - } +static dyad_mod_ctx_t *get_mod_ctx(flux_t *h) { + dyad_mod_ctx_t *mod_ctx = (dyad_mod_ctx_t *)flux_aux_get(h, "dyad"); + + if (!mod_ctx) { + mod_ctx = (dyad_mod_ctx_t *)malloc(sizeof(*mod_ctx)); + if (mod_ctx == NULL) { + DYAD_LOG_STDERR("DYAD_MOD: could not allocate memory for context"); + goto getctx_error; } + mod_ctx->handlers = NULL; + mod_ctx->ctx = NULL; - goto getctx_done; + if (flux_aux_set(h, "dyad", mod_ctx, freectx) < 0) { + DYAD_LOG_STDERR("DYAD_MOD: flux_aux_set() failed!"); + goto getctx_error; + } + } + + goto getctx_done; getctx_error:; - return NULL; + return NULL; getctx_done: - return mod_ctx; + return mod_ctx; } /* request callback called when dyad.fetch request is invoked */ #if DYAD_PERFFLOW -__attribute__ ((annotate ("@critical_path()"))) +__attribute__((annotate("@critical_path()"))) #endif static void -dyad_fetch_request_cb (flux_t *h, flux_msg_handler_t *w, const flux_msg_t *msg, void *arg) -{ - DYAD_C_FUNCTION_START (); - dyad_mod_ctx_t *mod_ctx = get_mod_ctx (h); - DYAD_LOG_INFO (mod_ctx->ctx, "Launched callback for %s", DYAD_DTL_RPC_NAME); - ssize_t inlen = 0l; - char *inbuf = NULL; - int fd = -1; - uint32_t userid = 0u; - char *upath = NULL; - char fullpath[PATH_MAX + 1] = {'\0'}; - int saved_errno = errno; - ssize_t file_size = 0l; - dyad_rc_t rc = 0; - struct flock shared_lock; - if (!flux_msg_is_streaming (msg)) { - errno = EPROTO; - goto fetch_error_wo_flock; - } - - if (flux_msg_get_userid (msg, &userid) < 0) - goto fetch_error_wo_flock; - - DYAD_LOG_INFO (mod_ctx->ctx, "DYAD_MOD: unpacking RPC message"); - - rc = mod_ctx->ctx->dtl_handle->rpc_unpack (mod_ctx->ctx, msg, &upath); - - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (mod_ctx->ctx, "Could not unpack message from client"); - errno = EPROTO; - goto fetch_error_wo_flock; - } - DYAD_C_FUNCTION_UPDATE_STR ("upath", upath); - DYAD_LOG_DEBUG (mod_ctx->ctx, "DYAD_MOD: requested user_path: %s", upath); - DYAD_LOG_DEBUG (mod_ctx->ctx, "DYAD_MOD: sending initial response to consumer"); - - rc = mod_ctx->ctx->dtl_handle->rpc_respond (mod_ctx->ctx, msg); - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (mod_ctx->ctx, "Could not send primary RPC response to client"); - goto fetch_error_wo_flock; - } - - strncpy (fullpath, mod_ctx->ctx->prod_managed_path, PATH_MAX - 1); - concat_str (fullpath, upath, "/", PATH_MAX); - DYAD_C_FUNCTION_UPDATE_STR ("fullpath", fullpath); +dyad_fetch_request_cb(flux_t *h, flux_msg_handler_t *w, const flux_msg_t *msg, + void *arg) { + DYAD_C_FUNCTION_START(); + dyad_mod_ctx_t *mod_ctx = get_mod_ctx(h); + DYAD_LOG_INFO(mod_ctx->ctx, "Launched callback for %s", DYAD_DTL_RPC_NAME); + ssize_t inlen = 0l; + char *inbuf = NULL; + int fd = -1; + uint32_t userid = 0u; + char *upath = NULL; + char fullpath[PATH_MAX + 1] = {'\0'}; + int saved_errno = errno; + ssize_t file_size = 0l; + dyad_rc_t rc = 0; + struct flock shared_lock; + if (!flux_msg_is_streaming(msg)) { + errno = EPROTO; + goto fetch_error_wo_flock; + } + + if (flux_msg_get_userid(msg, &userid) < 0) + goto fetch_error_wo_flock; + + DYAD_LOG_INFO(mod_ctx->ctx, "DYAD_MOD: unpacking RPC message"); + + rc = mod_ctx->ctx->dtl_handle->rpc_unpack(mod_ctx->ctx, msg, &upath); + + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(mod_ctx->ctx, "Could not unpack message from client"); + errno = EPROTO; + goto fetch_error_wo_flock; + } + DYAD_C_FUNCTION_UPDATE_STR("upath", upath); + DYAD_LOG_DEBUG(mod_ctx->ctx, "DYAD_MOD: requested user_path: %s", upath); + DYAD_LOG_DEBUG(mod_ctx->ctx, + "DYAD_MOD: sending initial response to consumer"); + + rc = mod_ctx->ctx->dtl_handle->rpc_respond(mod_ctx->ctx, msg); + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(mod_ctx->ctx, + "Could not send primary RPC response to client"); + goto fetch_error_wo_flock; + } + + strncpy(fullpath, mod_ctx->ctx->prod_managed_path, PATH_MAX - 1); + concat_str(fullpath, upath, "/", PATH_MAX); + DYAD_C_FUNCTION_UPDATE_STR("fullpath", fullpath); #if DYAD_SPIN_WAIT - if (!get_stat (fullpath, 1000U, 1000L)) { - DYAD_LOG_ERR (mod_ctx->ctx, "DYAD_MOD: Failed to access info on \"%s\".", fullpath); - // goto error; - } -#endif // DYAD_SPIN_WAIT - - DYAD_LOG_INFO (mod_ctx->ctx, "DYAD_MOD: Reading file %s for transfer", fullpath); - fd = open (fullpath, O_RDONLY); - - if (fd < 0) { - DYAD_LOG_ERROR (mod_ctx->ctx, "DYAD_MOD: Failed to open file \"%s\".", fullpath); - goto fetch_error_wo_flock; - } - rc = dyad_shared_flock (mod_ctx->ctx, fd, &shared_lock); - if (DYAD_IS_ERROR (rc)) { - goto fetch_error; - } - file_size = get_file_size (fd); - DYAD_LOG_DEBUG (mod_ctx->ctx, "DYAD_MOD: file %s has size %zd", fullpath, file_size); - rc = mod_ctx->ctx->dtl_handle->get_buffer (mod_ctx->ctx, file_size, (void **)&inbuf); + if (!get_stat(fullpath, 1000U, 1000L)) { + DYAD_LOG_ERR(mod_ctx->ctx, "DYAD_MOD: Failed to access info on \"%s\".", + fullpath); + // goto error; + } +#endif // DYAD_SPIN_WAIT + + DYAD_LOG_INFO(mod_ctx->ctx, "DYAD_MOD: Reading file %s for transfer", + fullpath); + fd = open(fullpath, O_RDONLY); + + if (fd < 0) { + DYAD_LOG_ERROR(mod_ctx->ctx, "DYAD_MOD: Failed to open file \"%s\".", + fullpath); + goto fetch_error_wo_flock; + } + rc = dyad_shared_flock(mod_ctx->ctx, fd, &shared_lock); + if (DYAD_IS_ERROR(rc)) { + goto fetch_error; + } + file_size = get_file_size(fd); + DYAD_LOG_DEBUG(mod_ctx->ctx, "DYAD_MOD: file %s has size %zd", fullpath, + file_size); + rc = mod_ctx->ctx->dtl_handle->get_buffer(mod_ctx->ctx, file_size, + (void **)&inbuf); #ifdef DYAD_ENABLE_UCX_RMA - // To reduce the number of RMA calls, we are encoding file size at the start of the buffer - memcpy (inbuf, &file_size, sizeof (file_size)); + // To reduce the number of RMA calls, we are encoding file size at the start + // of the buffer + memcpy(inbuf, &file_size, sizeof(file_size)); #endif - if (file_size > 0l) { + if (file_size > 0l) { #ifdef DYAD_ENABLE_UCX_RMA - inlen = read (fd, inbuf + sizeof (file_size), file_size); + inlen = read(fd, inbuf + sizeof(file_size), file_size); #else - inlen = read (fd, inbuf, file_size); + inlen = read(fd, inbuf, file_size); #endif - if (inlen != file_size) { - DYAD_LOG_ERROR (mod_ctx->ctx, "DYAD_MOD: Failed to load file \"%s\" only read %zd of %zd.", fullpath, inlen, file_size); - goto fetch_error; - } + if (inlen != file_size) { + DYAD_LOG_ERROR( + mod_ctx->ctx, + "DYAD_MOD: Failed to load file \"%s\" only read %zd of %zd.", + fullpath, inlen, file_size); + goto fetch_error; + } #ifdef DYAD_ENABLE_UCX_RMA - inlen = file_size + sizeof (file_size); + inlen = file_size + sizeof(file_size); #endif - DYAD_C_FUNCTION_UPDATE_INT ("file_size", file_size); - DYAD_LOG_DEBUG (mod_ctx->ctx, "Closing file pointer"); - dyad_release_flock (mod_ctx->ctx, fd, &shared_lock); - close (fd); - DYAD_LOG_DEBUG (mod_ctx->ctx, "Is inbuf NULL? -> %i", (int)(inbuf == NULL)); - DYAD_LOG_DEBUG (mod_ctx->ctx, "Establish DTL connection with consumer"); - rc = mod_ctx->ctx->dtl_handle->establish_connection (mod_ctx->ctx); - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (mod_ctx->ctx, "Could not establish DTL connection with client"); - errno = ECONNREFUSED; - goto fetch_error_wo_flock; - } - DYAD_LOG_DEBUG (mod_ctx->ctx, "Send file to consumer with DTL"); - rc = mod_ctx->ctx->dtl_handle->send (mod_ctx->ctx, inbuf, inlen); - DYAD_LOG_DEBUG (mod_ctx->ctx, "Close DTL connection with consumer"); - mod_ctx->ctx->dtl_handle->close_connection (mod_ctx->ctx); - mod_ctx->ctx->dtl_handle->return_buffer (mod_ctx->ctx, (void **)&inbuf); - if (DYAD_IS_ERROR (rc)) { - DYAD_LOG_ERROR (mod_ctx->ctx, "Could not send data to client via DTL\n"); - errno = ECOMM; - goto fetch_error_wo_flock; - } - } else { - goto fetch_error; + DYAD_C_FUNCTION_UPDATE_INT("file_size", file_size); + DYAD_LOG_DEBUG(mod_ctx->ctx, "Closing file pointer"); + dyad_release_flock(mod_ctx->ctx, fd, &shared_lock); + close(fd); + DYAD_LOG_DEBUG(mod_ctx->ctx, "Is inbuf NULL? -> %i", (int)(inbuf == NULL)); + DYAD_LOG_DEBUG(mod_ctx->ctx, "Establish DTL connection with consumer"); + rc = mod_ctx->ctx->dtl_handle->establish_connection(mod_ctx->ctx); + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(mod_ctx->ctx, + "Could not establish DTL connection with client"); + errno = ECONNREFUSED; + goto fetch_error_wo_flock; } - DYAD_LOG_DEBUG (mod_ctx->ctx, "Close RPC message stream with an ENODATA (%d) message", ENODATA); - if (flux_respond_error (h, msg, ENODATA, NULL) < 0) { - DYAD_LOG_ERROR (mod_ctx->ctx, "DYAD_MOD: %s: flux_respond_error with ENODATA failed\n", __func__); + DYAD_LOG_DEBUG(mod_ctx->ctx, "Send file to consumer with DTL"); + rc = mod_ctx->ctx->dtl_handle->send(mod_ctx->ctx, inbuf, inlen); + DYAD_LOG_DEBUG(mod_ctx->ctx, "Close DTL connection with consumer"); + mod_ctx->ctx->dtl_handle->close_connection(mod_ctx->ctx); + mod_ctx->ctx->dtl_handle->return_buffer(mod_ctx->ctx, (void **)&inbuf); + if (DYAD_IS_ERROR(rc)) { + DYAD_LOG_ERROR(mod_ctx->ctx, "Could not send data to client via DTL\n"); + errno = ECOMM; + goto fetch_error_wo_flock; } - DYAD_LOG_INFO (mod_ctx->ctx, "Finished %s module invocation\n", DYAD_DTL_RPC_NAME); - goto end_fetch_cb; + } else { + goto fetch_error; + } + DYAD_LOG_DEBUG(mod_ctx->ctx, + "Close RPC message stream with an ENODATA (%d) message", + ENODATA); + if (flux_respond_error(h, msg, ENODATA, NULL) < 0) { + DYAD_LOG_ERROR(mod_ctx->ctx, + "DYAD_MOD: %s: flux_respond_error with ENODATA failed\n", + __func__); + } + DYAD_LOG_INFO(mod_ctx->ctx, "Finished %s module invocation\n", + DYAD_DTL_RPC_NAME); + goto end_fetch_cb; fetch_error:; - dyad_release_flock (mod_ctx->ctx, fd, &shared_lock); - close (fd); + dyad_release_flock(mod_ctx->ctx, fd, &shared_lock); + close(fd); fetch_error_wo_flock:; - DYAD_LOG_ERROR (mod_ctx->ctx, "Close RPC message stream with an error (errno = %d)\n", errno); - if (flux_respond_error (h, msg, errno, NULL) < 0) { - DYAD_LOG_ERROR (mod_ctx->ctx, "DYAD_MOD: %s: flux_respond_error", __func__); - } - errno = saved_errno; - DYAD_C_FUNCTION_END (); - return; + DYAD_LOG_ERROR(mod_ctx->ctx, + "Close RPC message stream with an error (errno = %d)\n", + errno); + if (flux_respond_error(h, msg, errno, NULL) < 0) { + DYAD_LOG_ERROR(mod_ctx->ctx, "DYAD_MOD: %s: flux_respond_error", __func__); + } + errno = saved_errno; + DYAD_C_FUNCTION_END(); + return; end_fetch_cb:; - errno = saved_errno; - DYAD_C_FUNCTION_END (); - return; + errno = saved_errno; + DYAD_C_FUNCTION_END(); + return; } -static const struct flux_msg_handler_spec htab[] = - {{FLUX_MSGTYPE_REQUEST, DYAD_DTL_RPC_NAME, dyad_fetch_request_cb, 0}, - FLUX_MSGHANDLER_TABLE_END}; - -static void show_help (void) -{ - DYAD_LOG_STDOUT ("dyad module options and arguments\n"); - DYAD_LOG_STDOUT (" -h, --help: Show help.\n"); - DYAD_LOG_STDOUT (" -d, --debug: Enable debugging log message.\n"); - DYAD_LOG_STDOUT ( - " -m, --mode: DTL mode. Need an argument.\n" - " Either 'FLUX_RPC' (default) or 'UCX'.\n"); - DYAD_LOG_STDOUT ( - " -i, --info_log: Specify the file into which to redirect\n" - " info logging. Does nothing if DYAD was not\n" - " configured with '-DDYAD_LOGGER=PRINTF'.\n" - " Need a filename as an argument.\n"); - DYAD_LOG_STDOUT ( - " -e, --error_log: Specify the file into which to redirect\n" - " error logging. Does nothing if DYAD was\n" - " not configured with '-DDYAD_LOGGER=PRINTF'\n" - " Need a filename as an argument.\n"); +static const struct flux_msg_handler_spec htab[] = { + {FLUX_MSGTYPE_REQUEST, DYAD_DTL_RPC_NAME, dyad_fetch_request_cb, 0}, + FLUX_MSGHANDLER_TABLE_END}; + +static void show_help(void) { + DYAD_LOG_STDOUT("dyad module options and arguments\n"); + DYAD_LOG_STDOUT(" -h, --help: Show help.\n"); + DYAD_LOG_STDOUT(" -d, --debug: Enable debugging log message.\n"); + DYAD_LOG_STDOUT(" -m, --mode: DTL mode. Need an argument.\n" + " Either 'FLUX_RPC' (default) or 'UCX'.\n"); + DYAD_LOG_STDOUT( + " -i, --info_log: Specify the file into which to redirect\n" + " info logging. Does nothing if DYAD was not\n" + " configured with '-DDYAD_LOGGER=PRINTF'.\n" + " Need a filename as an argument.\n"); + DYAD_LOG_STDOUT( + " -e, --error_log: Specify the file into which to redirect\n" + " error logging. Does nothing if DYAD was\n" + " not configured with '-DDYAD_LOGGER=PRINTF'\n" + " Need a filename as an argument.\n"); } struct opt_parse_out { - const char *prod_managed_path; - const char *dtl_mode; - bool debug; + const char *prod_managed_path; + const char *dtl_mode; + bool debug; }; typedef struct opt_parse_out opt_parse_out_t; -static int opt_parse (opt_parse_out_t *restrict opt, - const unsigned broker_rank, - dyad_dtl_mode_t *restrict dtl_mode, - int _argc, - char **restrict _argv) -{ +static int opt_parse(opt_parse_out_t *restrict opt, const unsigned broker_rank, + dyad_dtl_mode_t *restrict dtl_mode, int _argc, + char **restrict _argv) { #ifndef DYAD_LOGGER_NO_LOG - char log_file_name[PATH_MAX + 1] = {'\0'}; - char err_file_name[PATH_MAX + 1] = {'\0'}; - sprintf (log_file_name, "dyad_mod_%u.out", broker_rank); - sprintf (err_file_name, "dyad_mod_%d.err", broker_rank); -#endif // DYAD_LOGGER_NO_LOG - *dtl_mode = DYAD_DTL_END; - - int rc = DYAD_RC_OK; - int argc = 0; - char *argv[PATH_MAX] = {NULL}; - char *prod_managed_path = NULL; - - if (opt == NULL) - return rc; - - if ((argc = _argc + 1) > PATH_MAX) { - DYAD_LOG_STDERR ("DYAD_MOD: too many options.\n"); - return DYAD_RC_SYSFAIL; - } - - for (int i = 1; i < argc; ++i) { - argv[i] = _argv[i - 1]; - // DYAD_LOG_STDERR ("DYAD_MOD: argv[%d] = '%s'\n", i, argv[i]); + char log_file_name[PATH_MAX + 1] = {'\0'}; + char err_file_name[PATH_MAX + 1] = {'\0'}; + sprintf(log_file_name, "dyad_mod_%u.out", broker_rank); + sprintf(err_file_name, "dyad_mod_%d.err", broker_rank); +#endif // DYAD_LOGGER_NO_LOG + *dtl_mode = DYAD_DTL_END; + + int rc = DYAD_RC_OK; + int argc = 0; + char *argv[PATH_MAX] = {NULL}; + char *prod_managed_path = NULL; + + if (opt == NULL) + return rc; + + if ((argc = _argc + 1) > PATH_MAX) { + DYAD_LOG_STDERR("DYAD_MOD: too many options.\n"); + return DYAD_RC_SYSFAIL; + } + + for (int i = 1; i < argc; ++i) { + argv[i] = _argv[i - 1]; + // DYAD_LOG_STDERR ("DYAD_MOD: argv[%d] = '%s'\n", i, argv[i]); + } + + while (1) { + static struct option long_options[] = { + {"help", no_argument, 0, 'h'}, + {"debug", no_argument, 0, 'd'}, + {"mode", required_argument, 0, 'm'}, + {"info_log", required_argument, 0, 'i'}, + {"error_log", required_argument, 0, 'e'}, + {0, 0, 0, 0}}; + /* getopt_long stores the option index here. */ + int option_index = 0; + int c = -1; + + c = getopt_long(argc, argv, "hdm:i:e:", long_options, &option_index); + + /* Detect the end of the options. */ + if (c == -1) { + // DYAD_LOG_STDERR ("DYAD_MOD: no more option.\n"); + break; } - - while (1) { - static struct option long_options[] = {{"help", no_argument, 0, 'h'}, - {"debug", no_argument, 0, 'd'}, - {"mode", required_argument, 0, 'm'}, - {"info_log", required_argument, 0, 'i'}, - {"error_log", required_argument, 0, 'e'}, - {0, 0, 0, 0}}; - /* getopt_long stores the option index here. */ - int option_index = 0; - int c = -1; - - c = getopt_long (argc, argv, "hdm:i:e:", long_options, &option_index); - - /* Detect the end of the options. */ - if (c == -1) { - // DYAD_LOG_STDERR ("DYAD_MOD: no more option.\n"); - break; - } - DYAD_LOG_STDERR ("DYAD_MOD: opt %c, index %d\n", (char)c, optind); - - switch (c) { - case 'h': - show_help (); - break; - case 'd': - DYAD_LOG_STDERR ("DYAD_MOD: 'debug' option -d \n"); - opt->debug = true; - break; - case 'm': - // If the DTL is already initialized and it is set to the same - // mode as the option, then skip reinitializing - DYAD_LOG_STDERR ("DYAD_MOD: DTL 'mode' option -m with value `%s'\n", optarg); - opt->dtl_mode = optarg; - if (strcmp("UCX", optarg) == 0) *dtl_mode = DYAD_DTL_UCX; - else if (strcmp("FLUX_RPC", optarg) == 0) *dtl_mode = DYAD_DTL_FLUX_RPC; - break; - case 'i': + DYAD_LOG_STDERR("DYAD_MOD: opt %c, index %d\n", (char)c, optind); + + switch (c) { + case 'h': + show_help(); + break; + case 'd': + DYAD_LOG_STDERR("DYAD_MOD: 'debug' option -d \n"); + opt->debug = true; + break; + case 'm': + // If the DTL is already initialized and it is set to the same + // mode as the option, then skip reinitializing + DYAD_LOG_STDERR("DYAD_MOD: DTL 'mode' option -m with value `%s'\n", + optarg); + opt->dtl_mode = optarg; + if (strcmp("UCX", optarg) == 0) + *dtl_mode = DYAD_DTL_UCX; + else if (strcmp("FLUX_RPC", optarg) == 0) + *dtl_mode = DYAD_DTL_FLUX_RPC; + break; + case 'i': #ifndef DYAD_LOGGER_NO_LOG - DYAD_LOG_STDERR ("DYAD_MOD: 'info_log' option -i with value `%s'\n", optarg); - sprintf (log_file_name, "%s_%u.out", optarg, broker_rank); -#endif // DYAD_LOGGER_NO_LOG - break; - case 'e': + DYAD_LOG_STDERR("DYAD_MOD: 'info_log' option -i with value `%s'\n", + optarg); + sprintf(log_file_name, "%s_%u.out", optarg, broker_rank); +#endif // DYAD_LOGGER_NO_LOG + break; + case 'e': #ifndef DYAD_LOGGER_NO_LOG - DYAD_LOG_STDERR ("DYAD_MOD: 'error_log' option -e with value `%s'\n", optarg); - sprintf (err_file_name, "%s_%d.err", optarg, broker_rank); -#endif // DYAD_LOGGER_NO_LOG - break; - case '?': - /* getopt_long already printed an error message. */ - break; - default: - DYAD_LOG_STDERR ("DYAD_MOD: option parsing failed %d\n", c); - return DYAD_RC_SYSFAIL; - } + DYAD_LOG_STDERR("DYAD_MOD: 'error_log' option -e with value `%s'\n", + optarg); + sprintf(err_file_name, "%s_%d.err", optarg, broker_rank); +#endif // DYAD_LOGGER_NO_LOG + break; + case '?': + /* getopt_long already printed an error message. */ + break; + default: + DYAD_LOG_STDERR("DYAD_MOD: option parsing failed %d\n", c); + return DYAD_RC_SYSFAIL; } + } #ifndef DYAD_LOGGER_NO_LOG - DYAD_LOG_STDOUT_REDIRECT (log_file_name); - DYAD_LOG_STDERR_REDIRECT (err_file_name); -#endif // DYAD_LOGGER_NO_LOG - - if (*dtl_mode == DYAD_DTL_END) { - opt->dtl_mode = NULL; - } - - /* Print any remaining command line arguments (not options). */ - while (optind < argc) { - DYAD_LOG_STDERR ("DYAD_MOD: positional arguments %s\n", argv[optind]); - prod_managed_path = argv[optind++]; - } - opt->prod_managed_path = prod_managed_path; - - return DYAD_RC_OK; + DYAD_LOG_STDOUT_REDIRECT(log_file_name); + DYAD_LOG_STDERR_REDIRECT(err_file_name); +#endif // DYAD_LOGGER_NO_LOG + + if (*dtl_mode == DYAD_DTL_END) { + opt->dtl_mode = NULL; + } + + /* Print any remaining command line arguments (not options). */ + while (optind < argc) { + DYAD_LOG_STDERR("DYAD_MOD: positional arguments %s\n", argv[optind]); + prod_managed_path = argv[optind++]; + } + opt->prod_managed_path = prod_managed_path; + + return DYAD_RC_OK; } -dyad_rc_t dyad_module_ctx_init (const opt_parse_out_t *opt, flux_t *h) -{ // Initialize DYAD context - dyad_mod_ctx_t *mod_ctx = get_mod_ctx (h); - - if (mod_ctx == NULL || opt == NULL || h == NULL) { - return DYAD_RC_NOCTX; - } - - if (opt->prod_managed_path) { - setenv (DYAD_PATH_PRODUCER_ENV, opt->prod_managed_path, 1); - const mode_t m = (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH | S_ISGID); - mkdir_as_needed (opt->prod_managed_path, m); - DYAD_LOG_STDERR ("DYAD_MOD: Loading DYAD Module with Path %s", opt->prod_managed_path); - } - - if (opt->dtl_mode) { - setenv (DYAD_DTL_MODE_ENV, opt->dtl_mode, 1); - DYAD_LOG_STDERR ( - "DYAD_MOD: DTL 'mode' option set. " - "Setting env %s=%s", - DYAD_DTL_MODE_ENV, - opt->dtl_mode); - } else { - DYAD_LOG_STDERR ( - "DYAD_MOD: Did not find DTL 'mode' option. " - "Using env %s=%s", - DYAD_DTL_MODE_ENV, - getenv (DYAD_DTL_MODE_ENV)); - } - char* kvs = getenv("DYAD_KVS_NAMESPACE"); - if (kvs != NULL) { - DYAD_LOG_STDERR ("DYAD_MOD: DYAD_KVS_NAMESPACE is set to `%s'\n", kvs); - } else { - DYAD_LOG_STDERR ("DYAD_MOD: DYAD_KVS_NAMESPACE is not set\n"); - } - dyad_ctx_init (DYAD_COMM_SEND, h); - mod_ctx->ctx = dyad_ctx_get (); - dyad_ctx_t *ctx = mod_ctx->ctx; - - if (ctx == NULL) { - DYAD_LOG_STDERR ("DYAD_MOD: dyad_ctx_init() failed!"); - return DYAD_RC_NOCTX; - } - ctx->h = h; - ctx->debug = opt->debug; - - if (ctx->dtl_handle == NULL) { - DYAD_LOG_STDERR ("DYAD_MOD: dyad_ctx_init() failed to initialize DTL!"); - return DYAD_RC_NOCTX; - } - - return DYAD_RC_OK; +dyad_rc_t dyad_module_ctx_init(const opt_parse_out_t *opt, + flux_t *h) { // Initialize DYAD context + dyad_mod_ctx_t *mod_ctx = get_mod_ctx(h); + + if (mod_ctx == NULL || opt == NULL || h == NULL) { + return DYAD_RC_NOCTX; + } + + if (opt->prod_managed_path) { + setenv(DYAD_PATH_PRODUCER_ENV, opt->prod_managed_path, 1); + const mode_t m = (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH | S_ISGID); + mkdir_as_needed(opt->prod_managed_path, m); + DYAD_LOG_STDERR("DYAD_MOD: Loading DYAD Module with Path %s", + opt->prod_managed_path); + } + + if (opt->dtl_mode) { + setenv(DYAD_DTL_MODE_ENV, opt->dtl_mode, 1); + DYAD_LOG_STDERR("DYAD_MOD: DTL 'mode' option set. " + "Setting env %s=%s", + DYAD_DTL_MODE_ENV, opt->dtl_mode); + } else { + DYAD_LOG_STDERR("DYAD_MOD: Did not find DTL 'mode' option. " + "Using env %s=%s", + DYAD_DTL_MODE_ENV, getenv(DYAD_DTL_MODE_ENV)); + } + char *kvs = getenv(DYAD_KVS_NAMESPACE_ENV); + if (kvs != NULL) { + DYAD_LOG_STDERR("DYAD_MOD: %s is set to `%s'\n", DYAD_KVS_NAMESPACE_ENV, + kvs); + } else { + DYAD_LOG_STDERR("DYAD_MOD: %s is not set\n", DYAD_KVS_NAMESPACE_ENV); + // Required so that dyad_ctx_init can pass + setenv(DYAD_KVS_NAMESPACE_ENV, "dyad_module_dummy_env", 1); + } + + dyad_ctx_init(DYAD_COMM_SEND, h); + mod_ctx->ctx = dyad_ctx_get(); + dyad_ctx_t *ctx = mod_ctx->ctx; + + if (ctx == NULL) { + DYAD_LOG_STDERR("DYAD_MOD: dyad_ctx_init() failed!"); + return DYAD_RC_NOCTX; + } + ctx->h = h; + ctx->debug = opt->debug; + + if (ctx->dtl_handle == NULL) { + DYAD_LOG_STDERR("DYAD_MOD: dyad_ctx_init() failed to initialize DTL!"); + return DYAD_RC_NOCTX; + } + + return DYAD_RC_OK; } -DYAD_DLL_EXPORTED int mod_main (flux_t *h, int argc, char **argv) -{ - DYAD_LOGGER_INIT (); - DYAD_LOG_STDOUT ("Loading mod_main\n"); - dyad_mod_ctx_t *mod_ctx = NULL; - dyad_dtl_mode_t dtl_mode = DYAD_DTL_DEFAULT; +DYAD_DLL_EXPORTED int mod_main(flux_t *h, int argc, char **argv) { + DYAD_LOGGER_INIT(); + DYAD_LOG_STDOUT("Loading mod_main\n"); + dyad_mod_ctx_t *mod_ctx = NULL; + dyad_dtl_mode_t dtl_mode = DYAD_DTL_DEFAULT; - if (!h) { - DYAD_LOG_STDERR ("Failed to get flux handle\n"); - goto mod_done; - } + if (!h) { + DYAD_LOG_STDERR("Failed to get flux handle\n"); + goto mod_done; + } - mod_ctx = get_mod_ctx (h); + mod_ctx = get_mod_ctx(h); - uint32_t broker_rank; - flux_get_rank (h, &broker_rank); + uint32_t broker_rank; + flux_get_rank(h, &broker_rank); #ifdef DYAD_PROFILER_DFTRACER - int pid = broker_rank; - DFTRACER_C_INIT (NULL, NULL, &pid); + int pid = broker_rank; + DFTRACER_C_INIT(NULL, NULL, &pid); #endif - DYAD_C_FUNCTION_START (); - - opt_parse_out_t opt = {NULL, NULL, false}; - DYAD_LOG_STDERR ("DYAD_MOD: Parsing command line options"); - - if (DYAD_IS_ERROR (opt_parse (&opt, broker_rank, &dtl_mode, argc, argv))) { - DYAD_LOG_STDERR ("DYAD_MOD: Cannot parse command line arguments"); - goto mod_error; - } - - if (DYAD_IS_ERROR (dyad_module_ctx_init (&opt, h))) { - goto mod_error; - } - - if (flux_msg_handler_addvec (mod_ctx->ctx->h, htab, (void *)h, &mod_ctx->handlers) < 0) { - DYAD_LOG_ERROR (mod_ctx->ctx, "DYAD_MOD: flux_msg_handler_addvec: %s\n", strerror (errno)); - goto mod_error; - } - - if (flux_reactor_run (flux_get_reactor (mod_ctx->ctx->h), 0) < 0) { - DYAD_LOG_DEBUG (mod_ctx->ctx, "DYAD_MOD: flux_reactor_run: %s", strerror (errno)); - goto mod_error; - } - DYAD_LOG_DEBUG (mod_ctx->ctx, "DYAD_MOD: Finished"); - goto mod_done; + DYAD_C_FUNCTION_START(); + + opt_parse_out_t opt = {NULL, NULL, false}; + DYAD_LOG_STDERR("DYAD_MOD: Parsing command line options"); + + if (DYAD_IS_ERROR(opt_parse(&opt, broker_rank, &dtl_mode, argc, argv))) { + DYAD_LOG_STDERR("DYAD_MOD: Cannot parse command line arguments"); + goto mod_error; + } + + if (DYAD_IS_ERROR(dyad_module_ctx_init(&opt, h))) { + goto mod_error; + } + + if (flux_msg_handler_addvec(mod_ctx->ctx->h, htab, (void *)h, + &mod_ctx->handlers) < 0) { + DYAD_LOG_ERROR(mod_ctx->ctx, "DYAD_MOD: flux_msg_handler_addvec: %s\n", + strerror(errno)); + goto mod_error; + } + + if (flux_reactor_run(flux_get_reactor(mod_ctx->ctx->h), 0) < 0) { + DYAD_LOG_DEBUG(mod_ctx->ctx, "DYAD_MOD: flux_reactor_run: %s", + strerror(errno)); + goto mod_error; + } + DYAD_LOG_DEBUG(mod_ctx->ctx, "DYAD_MOD: Finished"); + goto mod_done; mod_error:; - DYAD_C_FUNCTION_END (); - return EXIT_FAILURE; + DYAD_C_FUNCTION_END(); + return EXIT_FAILURE; mod_done:; - DYAD_C_FUNCTION_END (); - return EXIT_SUCCESS; + DYAD_C_FUNCTION_END(); + return EXIT_SUCCESS; } -DYAD_DLL_EXPORTED MOD_NAME ("dyad"); +DYAD_DLL_EXPORTED MOD_NAME("dyad"); /* * vi:tabstop=4 shiftwidth=4 expandtab diff --git a/src/dyad/modules/test_opt_parse.c b/src/dyad/modules/test_opt_parse.c index 04293e87..cfc61666 100644 --- a/src/dyad/modules/test_opt_parse.c +++ b/src/dyad/modules/test_opt_parse.c @@ -7,12 +7,14 @@ #define DYAD_UTIL_LOGGER 1 #define DYAD_LOGGER_FLUX 1 +// clang-format off #include #include #include #include -#include +#include #include +// clang-format on #if defined(__cplusplus) #include @@ -29,188 +31,189 @@ #include #include #include -#endif // defined(__cplusplus) +#endif // defined(__cplusplus) #include +#include #include #include #include -#include /** This is a temporary measure until environment variable based initialization * is implemented */ -static dyad_dtl_mode_t get_dtl_mode_env () -{ - char* e = NULL; - - size_t dtl_mode_env_len = 0ul; - - if ((e = getenv (DYAD_DTL_MODE_ENV))) { - dtl_mode_env_len = strlen (e); - if (strncmp (e, "FLUX_RPC", dtl_mode_env_len) == 0) { - DYAD_LOG_STDERR ("DYAD MOD: FLUX_RPC DTL mode found in environment\n"); - return DYAD_DTL_FLUX_RPC; - } else if (strncmp (e, "UCX", dtl_mode_env_len) == 0) { - DYAD_LOG_STDERR ("DYAD MOD: UCX DTL mode found in environment\n"); - return DYAD_DTL_UCX; - } else { - DYAD_LOG_STDERR ("DYAD MOD: Invalid env %s = %s. Defaulting to %s\n", \ - DYAD_DTL_MODE_ENV, e, dyad_dtl_mode_name[DYAD_DTL_DEFAULT]); - return DYAD_DTL_DEFAULT; - } +static dyad_dtl_mode_t get_dtl_mode_env() { + char *e = NULL; + + size_t dtl_mode_env_len = 0ul; + + if ((e = getenv(DYAD_DTL_MODE_ENV))) { + dtl_mode_env_len = strlen(e); + if (strncmp(e, "FLUX_RPC", dtl_mode_env_len) == 0) { + DYAD_LOG_STDERR("DYAD MOD: FLUX_RPC DTL mode found in environment\n"); + return DYAD_DTL_FLUX_RPC; + } else if (strncmp(e, "UCX", dtl_mode_env_len) == 0) { + DYAD_LOG_STDERR("DYAD MOD: UCX DTL mode found in environment\n"); + return DYAD_DTL_UCX; } else { - DYAD_LOG_STDERR ("DYAD MOD: %s is not set. Defaulting to %s\n", \ - DYAD_DTL_MODE_ENV, dyad_dtl_mode_name[DYAD_DTL_DEFAULT]); - return DYAD_DTL_DEFAULT; + DYAD_LOG_STDERR("DYAD MOD: Invalid env %s = %s. Defaulting to %s\n", + DYAD_DTL_MODE_ENV, e, + dyad_dtl_mode_name[DYAD_DTL_DEFAULT]); + return DYAD_DTL_DEFAULT; } + } else { + DYAD_LOG_STDERR("DYAD MOD: %s is not set. Defaulting to %s\n", + DYAD_DTL_MODE_ENV, dyad_dtl_mode_name[DYAD_DTL_DEFAULT]); return DYAD_DTL_DEFAULT; + } + return DYAD_DTL_DEFAULT; } -static void show_help (void) -{ - DYAD_LOG_STDOUT ("dyad module options and arguments\n"); - DYAD_LOG_STDOUT (" -h, --help: Show help.\n"); - DYAD_LOG_STDOUT (" -d, --debug: Enable debugging log message.\n"); - DYAD_LOG_STDOUT (" -m, --mode: DTL mode. Need an argument.\n" - " Either 'FLUX_RPC' (default) or 'UCX'.\n"); - DYAD_LOG_STDOUT (" -i, --info_log: Specify the file into which to redirect\n" - " info logging. Does nothing if DYAD was not\n" - " configured with '-DDYAD_LOGGER=PRINTF'.\n" - " Need a filename as an argument.\n"); - DYAD_LOG_STDOUT (" -e, --error_log: Specify the file into which to redirect\n" - " error logging. Does nothing if DYAD was\n" - " not configured with '-DDYAD_LOGGER=PRINTF'\n" - " Need a filename as an argument.\n"); +static void show_help(void) { + DYAD_LOG_STDOUT("dyad module options and arguments\n"); + DYAD_LOG_STDOUT(" -h, --help: Show help.\n"); + DYAD_LOG_STDOUT(" -d, --debug: Enable debugging log message.\n"); + DYAD_LOG_STDOUT(" -m, --mode: DTL mode. Need an argument.\n" + " Either 'FLUX_RPC' (default) or 'UCX'.\n"); + DYAD_LOG_STDOUT( + " -i, --info_log: Specify the file into which to redirect\n" + " info logging. Does nothing if DYAD was not\n" + " configured with '-DDYAD_LOGGER=PRINTF'.\n" + " Need a filename as an argument.\n"); + DYAD_LOG_STDOUT( + " -e, --error_log: Specify the file into which to redirect\n" + " error logging. Does nothing if DYAD was\n" + " not configured with '-DDYAD_LOGGER=PRINTF'\n" + " Need a filename as an argument.\n"); } -static int opt_parse (dyad_ctx_t* ctx, const unsigned broker_rank, - dyad_dtl_mode_t *dtl_mode, int argc, char** argv) -{ - size_t arglen = 0ul; - #ifndef DYAD_LOGGER_NO_LOG - char log_file_name[PATH_MAX+1] = {'\0'}; - char err_file_name[PATH_MAX+1] = {'\0'}; - sprintf (log_file_name, "dyad_mod_%u.out", broker_rank); - sprintf (err_file_name, "dyad_mod_%d.err", broker_rank); - #endif // DYAD_LOGGER_NO_LOG - *dtl_mode = DYAD_DTL_END; - char* prod_managed_path = NULL; - bool debug = false; - (void) debug; - - while (1) - { - static struct option long_options[] = - { - {"help", no_argument, 0, 'h'}, - {"debug", no_argument, 0, 'd'}, - {"mode", required_argument, 0, 'm'}, - {"info_log", required_argument, 0, 'i'}, - {"error_log", required_argument, 0, 'e'}, - {0, 0, 0, 0} - }; - /* getopt_long stores the option index here. */ - int option_index = 0; - int c = -1; - - c = getopt_long (argc, argv, "hdm:i:e:", long_options, &option_index); - - /* Detect the end of the options. */ - if (c == -1) { - DYAD_LOG_DEBUG (ctx, "DYAD_MOD: no more option\n"); - break; - } - DYAD_LOG_DEBUG (ctx, "DYAD_MOD: opt %c, index %d\n", (char) c, optind); - - switch (c) - { - case 'h': - show_help (); - break; - case 'd': - DYAD_LOG_DEBUG (ctx, "DYAD_MOD: 'debug' option -d \n"); - debug = true; - break; - case 'm': - DYAD_LOG_DEBUG (ctx, "DYAD_MOD: DTL 'mode' option -m with value `%s'\n", optarg); - arglen = strlen (optarg); - if (strncmp (optarg, "FLUX_RPC", arglen) == 0) { - *dtl_mode = DYAD_DTL_FLUX_RPC; - } else if (strncmp (optarg, "UCX", arglen) == 0) { - *dtl_mode = DYAD_DTL_UCX; - } else { - DYAD_LOG_ERROR (ctx, "DYAD_MOD: Invalid DTL 'mode' (%s) provided\n", optarg); - *dtl_mode = DYAD_DTL_END; - show_help (); - return DYAD_RC_SYSFAIL ; - } - break; - case 'i': - #ifndef DYAD_LOGGER_NO_LOG - DYAD_LOG_DEBUG (ctx, "DYAD_MOD: 'info_log' option -i with value `%s'\n", optarg); - sprintf (log_file_name, "%s_%u.out", optarg, broker_rank); - #endif // DYAD_LOGGER_NO_LOG - break; - case 'e': - #ifndef DYAD_LOGGER_NO_LOG - DYAD_LOG_DEBUG (ctx, "DYAD_MOD: 'error_log' option -e with value `%s'\n", optarg); - sprintf (err_file_name, "%s_%d.err", optarg, broker_rank); - #endif // DYAD_LOGGER_NO_LOG - break; - case '?': - /* getopt_long already printed an error message. */ - break; - default: - DYAD_LOG_DEBUG (ctx, "DYAD_MOD: option parsing failed %d\n", c); - return DYAD_RC_SYSFAIL ; - } - } - - #ifndef DYAD_LOGGER_NO_LOG - DYAD_LOG_STDOUT_REDIRECT (log_file_name); - DYAD_LOG_STDERR_REDIRECT (err_file_name); - #endif // DYAD_LOGGER_NO_LOG - - if (*dtl_mode == DYAD_DTL_END) { - DYAD_LOG_DEBUG (ctx, "DYAD_MOD: Did not find DTL 'mode' option"); - *dtl_mode = get_dtl_mode_env (); +static int opt_parse(dyad_ctx_t *ctx, const unsigned broker_rank, + dyad_dtl_mode_t *dtl_mode, int argc, char **argv) { + size_t arglen = 0ul; +#ifndef DYAD_LOGGER_NO_LOG + char log_file_name[PATH_MAX + 1] = {'\0'}; + char err_file_name[PATH_MAX + 1] = {'\0'}; + sprintf(log_file_name, "dyad_mod_%u.out", broker_rank); + sprintf(err_file_name, "dyad_mod_%d.err", broker_rank); +#endif // DYAD_LOGGER_NO_LOG + *dtl_mode = DYAD_DTL_END; + char *prod_managed_path = NULL; + bool debug = false; + (void)debug; + + while (1) { + static struct option long_options[] = { + {"help", no_argument, 0, 'h'}, + {"debug", no_argument, 0, 'd'}, + {"mode", required_argument, 0, 'm'}, + {"info_log", required_argument, 0, 'i'}, + {"error_log", required_argument, 0, 'e'}, + {0, 0, 0, 0}}; + /* getopt_long stores the option index here. */ + int option_index = 0; + int c = -1; + + c = getopt_long(argc, argv, "hdm:i:e:", long_options, &option_index); + + /* Detect the end of the options. */ + if (c == -1) { + DYAD_LOG_DEBUG(ctx, "DYAD_MOD: no more option\n"); + break; } - - DYAD_LOG_DEBUG (ctx, "DYAD_MOD: optind %d argc %d\n", optind, argc); - /* Print any remaining command line arguments (not options). */ - while (optind < argc) { - DYAD_LOG_DEBUG (ctx, "DYAD_MOD: positional arguments %s\n", argv[optind]); - prod_managed_path = argv[optind++]; + DYAD_LOG_DEBUG(ctx, "DYAD_MOD: opt %c, index %d\n", (char)c, optind); + + switch (c) { + case 'h': + show_help(); + break; + case 'd': + DYAD_LOG_DEBUG(ctx, "DYAD_MOD: 'debug' option -d \n"); + debug = true; + break; + case 'm': + DYAD_LOG_DEBUG(ctx, "DYAD_MOD: DTL 'mode' option -m with value `%s'\n", + optarg); + arglen = strlen(optarg); + if (strncmp(optarg, "FLUX_RPC", arglen) == 0) { + *dtl_mode = DYAD_DTL_FLUX_RPC; + } else if (strncmp(optarg, "UCX", arglen) == 0) { + *dtl_mode = DYAD_DTL_UCX; + } else { + DYAD_LOG_ERROR(ctx, "DYAD_MOD: Invalid DTL 'mode' (%s) provided\n", + optarg); + *dtl_mode = DYAD_DTL_END; + show_help(); + return DYAD_RC_SYSFAIL; + } + break; + case 'i': +#ifndef DYAD_LOGGER_NO_LOG + DYAD_LOG_DEBUG(ctx, "DYAD_MOD: 'info_log' option -i with value `%s'\n", + optarg); + sprintf(log_file_name, "%s_%u.out", optarg, broker_rank); +#endif // DYAD_LOGGER_NO_LOG + break; + case 'e': +#ifndef DYAD_LOGGER_NO_LOG + DYAD_LOG_DEBUG(ctx, "DYAD_MOD: 'error_log' option -e with value `%s'\n", + optarg); + sprintf(err_file_name, "%s_%d.err", optarg, broker_rank); +#endif // DYAD_LOGGER_NO_LOG + break; + case '?': + /* getopt_long already printed an error message. */ + break; + default: + DYAD_LOG_DEBUG(ctx, "DYAD_MOD: option parsing failed %d\n", c); + return DYAD_RC_SYSFAIL; } - if (prod_managed_path) { - const size_t prod_path_len = strlen (prod_managed_path); - const char* tmp_ptr = prod_managed_path; - prod_managed_path = (char*)malloc (prod_path_len + 1); - if (prod_managed_path == NULL) { - DYAD_LOG_ERROR (ctx, "DYAD_MOD: Could not allocate buffer for " \ - "Producer managed path!\n"); - return DYAD_RC_SYSFAIL ; - } - strncpy (prod_managed_path, tmp_ptr, prod_path_len + 1); - DYAD_LOG_INFO (ctx, "DYAD_MOD: Loading DYAD Module with Path %s and DTL Mode %s", \ - prod_managed_path, dyad_dtl_mode_name[*dtl_mode]); + } + +#ifndef DYAD_LOGGER_NO_LOG + DYAD_LOG_STDOUT_REDIRECT(log_file_name); + DYAD_LOG_STDERR_REDIRECT(err_file_name); +#endif // DYAD_LOGGER_NO_LOG + + if (*dtl_mode == DYAD_DTL_END) { + DYAD_LOG_DEBUG(ctx, "DYAD_MOD: Did not find DTL 'mode' option"); + *dtl_mode = get_dtl_mode_env(); + } + + DYAD_LOG_DEBUG(ctx, "DYAD_MOD: optind %d argc %d\n", optind, argc); + /* Print any remaining command line arguments (not options). */ + while (optind < argc) { + DYAD_LOG_DEBUG(ctx, "DYAD_MOD: positional arguments %s\n", argv[optind]); + prod_managed_path = argv[optind++]; + } + if (prod_managed_path) { + const size_t prod_path_len = strlen(prod_managed_path); + const char *tmp_ptr = prod_managed_path; + prod_managed_path = (char *)malloc(prod_path_len + 1); + if (prod_managed_path == NULL) { + DYAD_LOG_ERROR(ctx, "DYAD_MOD: Could not allocate buffer for " + "Producer managed path!\n"); + return DYAD_RC_SYSFAIL; } - DYAD_LOG_INFO (NULL, "DYAD_MOD: debug flag set to %s\n", debug? "true": "false"); - DYAD_LOG_INFO (NULL, "DYAD_MOD: prod_managed_path set to %s\n", prod_managed_path); - return DYAD_RC_OK; + strncpy(prod_managed_path, tmp_ptr, prod_path_len + 1); + DYAD_LOG_INFO(ctx, + "DYAD_MOD: Loading DYAD Module with Path %s and DTL Mode %s", + prod_managed_path, dyad_dtl_mode_name[*dtl_mode]); + } + DYAD_LOG_INFO(NULL, "DYAD_MOD: debug flag set to %s\n", + debug ? "true" : "false"); + DYAD_LOG_INFO(NULL, "DYAD_MOD: prod_managed_path set to %s\n", + prod_managed_path); + return DYAD_RC_OK; } -int main (int argc, char **argv) -{ - dyad_dtl_mode_t dtl_mode = DYAD_DTL_DEFAULT; - uint32_t broker_rank = 0u; +int main(int argc, char **argv) { + dyad_dtl_mode_t dtl_mode = DYAD_DTL_DEFAULT; + uint32_t broker_rank = 0u; - DYAD_LOG_DEBUG (NULL, "DYAD_MOD: Parsing command line options"); - if (opt_parse (NULL, broker_rank, &dtl_mode, argc, argv) != DYAD_RC_OK) { - DYAD_LOG_ERROR (NULL, "DYAD_MOD: Cannot parse command line arguments"); - return EXIT_FAILURE; - } + DYAD_LOG_DEBUG(NULL, "DYAD_MOD: Parsing command line options"); + if (opt_parse(NULL, broker_rank, &dtl_mode, argc, argv) != DYAD_RC_OK) { + DYAD_LOG_ERROR(NULL, "DYAD_MOD: Cannot parse command line arguments"); + return EXIT_FAILURE; + } - return EXIT_SUCCESS; + return EXIT_SUCCESS; } - diff --git a/src/dyad/stream/CMakeLists.txt b/src/dyad/stream/CMakeLists.txt index 0c226401..9c7c3fd1 100644 --- a/src/dyad/stream/CMakeLists.txt +++ b/src/dyad/stream/CMakeLists.txt @@ -1,8 +1,8 @@ set(DYAD_FSTREAM_SRC ${CMAKE_CURRENT_SOURCE_DIR}/dyad_stream_core.cpp) -set(DYAD_FSTREAM_PRIVATE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_rc.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_dtl.h - ${CMAKE_CURRENT_SOURCE_DIR}/../core/dyad_ctx.h - ${CMAKE_CURRENT_SOURCE_DIR}/../core/dyad_core.h +set(DYAD_FSTREAM_PRIVATE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/common/dyad_rc.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/common/dyad_dtl.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/core/dyad_ctx.h + ${CMAKE_CURRENT_SOURCE_DIR}/../client/dyad_client_int.h ${CMAKE_CURRENT_SOURCE_DIR}/../utils/utils.h) set(DYAD_FSTREAM_PUBLIC_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/stream/dyad_stream_api.hpp ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/stream/dyad_params.hpp @@ -12,9 +12,9 @@ set(DYAD_FSTREAM_PUBLIC_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dya add_library(${PROJECT_NAME}_fstream SHARED ${DYAD_FSTREAM_SRC} ${DYAD_FSTREAM_PRIVATE_HEADERS} ${DYAD_FSTREAM_PUBLIC_HEADERS}) set_target_properties(${PROJECT_NAME}_fstream PROPERTIES CMAKE_INSTALL_RPATH - "${CMAKE_INSTALL_PREFIX}/${DYAD_LIBDIR}") + "${DYAD_INSTALL_LIBDIR}") target_compile_features(${PROJECT_NAME}_fstream PRIVATE) -target_link_libraries(${PROJECT_NAME}_fstream PRIVATE ${PROJECT_NAME}_ctx ${PROJECT_NAME}_core +target_link_libraries(${PROJECT_NAME}_fstream PRIVATE ${PROJECT_NAME}_client ${PROJECT_NAME}_utils flux::core) if (ENABLE_PERFFLOW) @@ -27,12 +27,10 @@ target_compile_definitions(${PROJECT_NAME}_fstream PUBLIC DYAD_HAS_CONFIG) target_include_directories(${PROJECT_NAME}_fstream PUBLIC $ $ - $) + $) target_include_directories(${PROJECT_NAME}_fstream SYSTEM PRIVATE ${FluxCore_INCLUDE_DIRS}) -if (TARGET DYAD_CXX_FLAGS_werror) - target_link_libraries(${PROJECT_NAME}_fstream PRIVATE DYAD_CXX_FLAGS_werror) -endif () +dyad_add_werror_if_needed(${PROJECT_NAME}_fstream) if(DYAD_PROFILER STREQUAL "DFTRACER") target_link_libraries(${PROJECT_NAME}_fstream PRIVATE ${DFTRACER_LIBRARIES}) endif() @@ -40,9 +38,9 @@ endif() install( TARGETS ${PROJECT_NAME}_fstream EXPORT ${DYAD_EXPORTED_TARGETS} - LIBRARY DESTINATION ${DYAD_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${DYAD_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${DYAD_INSTALL_BIN_DIR} + LIBRARY DESTINATION ${DYAD_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${DYAD_INSTALL_LIBDIR} + RUNTIME DESTINATION ${DYAD_INSTALL_BINDIR} ) if(NOT "${DYAD_FSTREAM_PUBLIC_HEADERS}" STREQUAL "") dyad_install_headers("${DYAD_FSTREAM_PUBLIC_HEADERS}" ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/src/dyad/stream/dyad_stream_core.cpp b/src/dyad/stream/dyad_stream_core.cpp index 900461bf..5a94bb9d 100644 --- a/src/dyad/stream/dyad_stream_core.cpp +++ b/src/dyad/stream/dyad_stream_core.cpp @@ -16,15 +16,14 @@ #include -#include -#include +#include #include #include #ifndef _GNU_SOURCE #define _GNU_SOURCE #include -#endif // _GNU_SOURCE +#endif // _GNU_SOURCE #include #include @@ -33,7 +32,7 @@ #include #include #include -using namespace std; // std::clock () +using namespace std; // std::clock () // #include // c++11 #include @@ -42,254 +41,216 @@ using namespace std; // std::clock () #include #include #include -#include // dirname +#include // dirname #include -namespace dyad -{ +namespace dyad { /***************************************************************************** * * * dyad_stream_core API * * * *****************************************************************************/ -dyad_stream_core::dyad_stream_core () - : m_ctx (NULL), m_ctx_mutable (NULL), m_initialized (false), m_is_prod (false), m_is_cons (false) -{ - DYAD_CPP_FUNCTION(); +dyad_stream_core::dyad_stream_core() + : m_ctx(NULL), m_ctx_mutable(NULL), m_initialized(false), m_is_prod(false), + m_is_cons(false) { + DYAD_CPP_FUNCTION(); } -dyad_stream_core::~dyad_stream_core () -{ - finalize (); -} +dyad_stream_core::~dyad_stream_core() { finalize(); } -void dyad_stream_core::finalize () -{ - DYAD_CPP_FUNCTION(); - if (m_ctx != NULL) { - //dyad_finalize (&m_ctx); - m_ctx = m_ctx_mutable = NULL; - m_initialized = false; - } +void dyad_stream_core::finalize() { + DYAD_CPP_FUNCTION(); + if (m_ctx != NULL) { + // dyad_finalize (&m_ctx); + m_ctx = m_ctx_mutable = NULL; + m_initialized = false; + } } -void dyad_stream_core::init (const bool reinit) -{ - DYAD_CPP_FUNCTION(); - bool reinit_env = false; - char *e = NULL; - - if ((e = getenv (DYAD_REINIT_ENV))) { - reinit_env = true; - } else { - reinit_env = false; - } - - if (!(reinit || reinit_env) && m_initialized) { - return; - } - - if ((e = getenv (DYAD_PATH_CONSUMER_ENV))) { - m_is_cons = (strlen (e) != 0); - } else { - m_is_cons = false; - } - if ((e = getenv (DYAD_PATH_PRODUCER_ENV))) { - m_is_prod = (strlen (e) != 0); - } else { - m_is_prod = false; - } - - if (reinit || reinit_env || - !(m_ctx = m_ctx_mutable = dyad_ctx_get ())) - { - dyad_ctx_init (DYAD_COMM_RECV, NULL); - m_ctx = m_ctx_mutable = dyad_ctx_get (); - log_info ("Stream core is initialized by env variables."); - } else { - log_info ("Stream core skips initialization as it has already been initialized."); - } - - // TODO figure out if we want to error if init fails - m_initialized = true; +void dyad_stream_core::init(const bool reinit) { + DYAD_CPP_FUNCTION(); + bool reinit_env = false; + char *e = NULL; + + if ((e = getenv(DYAD_REINIT_ENV))) { + reinit_env = true; + } else { + reinit_env = false; + } + + if (!(reinit || reinit_env) && m_initialized) { + return; + } + + if ((e = getenv(DYAD_PATH_CONSUMER_ENV))) { + m_is_cons = (strlen(e) != 0); + } else { + m_is_cons = false; + } + if ((e = getenv(DYAD_PATH_PRODUCER_ENV))) { + m_is_prod = (strlen(e) != 0); + } else { + m_is_prod = false; + } + + if (reinit || reinit_env || !(m_ctx = m_ctx_mutable = dyad_ctx_get())) { + dyad_ctx_init(DYAD_COMM_RECV, NULL); + m_ctx = m_ctx_mutable = dyad_ctx_get(); + log_info("Stream core is initialized by env variables."); + } else { + log_info( + "Stream core skips initialization as it has already been initialized."); + } + + // TODO figure out if we want to error if init fails + m_initialized = true; } -void dyad_stream_core::init (const dyad_params &p) -{ - DYAD_CPP_FUNCTION(); - DYAD_LOG_DEBUG(m_ctx, "DYAD_WRAPPER: Initializeing DYAD wrapper"); - dyad_rc_t rc = dyad_init (p.m_debug, - false, - p.m_shared_storage, - p.m_reinit, - p.m_async_publish, - p.m_fsync_write, - p.m_key_depth, - p.m_key_bins, - p.m_service_mux, - p.m_kvs_namespace.c_str (), - p.m_prod_managed_path.c_str (), - p.m_cons_managed_path.c_str (), - p.m_relative_to_managed_path, - dyad_dtl_mode_name[static_cast (p.m_dtl_mode)], - DYAD_COMM_RECV, NULL); +void dyad_stream_core::init(const dyad_params &p) { + DYAD_CPP_FUNCTION(); + DYAD_LOG_DEBUG(m_ctx, "DYAD_WRAPPER: Initializeing DYAD wrapper"); + dyad_rc_t rc = dyad_init( + p.m_debug, false, p.m_shared_storage, p.m_reinit, p.m_async_publish, + p.m_fsync_write, p.m_key_depth, p.m_key_bins, p.m_service_mux, + p.m_kvs_namespace.c_str(), p.m_prod_managed_path.c_str(), + p.m_cons_managed_path.c_str(), p.m_relative_to_managed_path, + dyad_dtl_mode_name[static_cast(p.m_dtl_mode)], + DYAD_COMM_RECV, NULL); #if defined(DYAD_HAS_STD_FSTREAM_FD) - m_ctx_mutable->use_fs_locks = true; + m_ctx_mutable->use_fs_locks = true; #else - // Rely on the KVS-based synchronization and disable checking for fs lock based logic. - m_ctx_mutable->use_fs_locks = false; + // Rely on the KVS-based synchronization and disable checking for fs lock + // based logic. + m_ctx_mutable->use_fs_locks = false; #endif - (void) rc; - // TODO figure out if we want to error if init fails - m_initialized = true; - log_info ("Stream core is initialized by parameters"); + (void)rc; + // TODO figure out if we want to error if init fails + m_initialized = true; + log_info("Stream core is initialized by parameters"); } -void dyad_stream_core::log_info (const std::string &msg_head) const -{ - DYAD_CPP_FUNCTION(); - DYAD_LOG_INFO (m_ctx, "=== %s ===", msg_head.c_str ()); - DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_PATH_CONSUMER_ENV, m_ctx->cons_managed_path); - DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_PATH_PRODUCER_ENV, m_ctx->prod_managed_path); - DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_PATH_RELATIVE_ENV, - (m_ctx->relative_to_managed_path) ? "true" : "false"); - DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_SYNC_DEBUG_ENV, - (m_ctx->debug) ? "true" : "false"); - DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_SHARED_STORAGE_ENV, - (m_ctx->shared_storage) ? "true" : "false"); - DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_ASYNC_PUBLISH_ENV, - (m_ctx->async_publish) ? "true" : "false"); - DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_FSYNC_WRITE_ENV, - (m_ctx->fsync_write) ? "true" : "false"); - DYAD_LOG_INFO (m_ctx, "%s=%u", DYAD_KEY_DEPTH_ENV, m_ctx->key_depth); - DYAD_LOG_INFO (m_ctx, "%s=%u", DYAD_KEY_BINS_ENV, m_ctx->key_bins); - DYAD_LOG_INFO (m_ctx, "%s=%u", DYAD_SERVICE_MUX_ENV, m_ctx->service_mux); - DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_KVS_NAMESPACE_ENV, m_ctx->kvs_namespace); - DYAD_LOG_INFO (m_ctx, "%s=%s", DYAD_DTL_MODE_ENV, getenv (DYAD_DTL_MODE_ENV)); +void dyad_stream_core::log_info(const std::string &msg_head) const { + DYAD_CPP_FUNCTION(); + DYAD_LOG_INFO(m_ctx, "=== %s ===", msg_head.c_str()); + DYAD_LOG_INFO(m_ctx, "%s=%s", DYAD_PATH_CONSUMER_ENV, + m_ctx->cons_managed_path); + DYAD_LOG_INFO(m_ctx, "%s=%s", DYAD_PATH_PRODUCER_ENV, + m_ctx->prod_managed_path); + DYAD_LOG_INFO(m_ctx, "%s=%s", DYAD_PATH_RELATIVE_ENV, + (m_ctx->relative_to_managed_path) ? "true" : "false"); + DYAD_LOG_INFO(m_ctx, "%s=%s", DYAD_SYNC_DEBUG_ENV, + (m_ctx->debug) ? "true" : "false"); + DYAD_LOG_INFO(m_ctx, "%s=%s", DYAD_SHARED_STORAGE_ENV, + (m_ctx->shared_storage) ? "true" : "false"); + DYAD_LOG_INFO(m_ctx, "%s=%s", DYAD_ASYNC_PUBLISH_ENV, + (m_ctx->async_publish) ? "true" : "false"); + DYAD_LOG_INFO(m_ctx, "%s=%s", DYAD_FSYNC_WRITE_ENV, + (m_ctx->fsync_write) ? "true" : "false"); + DYAD_LOG_INFO(m_ctx, "%s=%u", DYAD_KEY_DEPTH_ENV, m_ctx->key_depth); + DYAD_LOG_INFO(m_ctx, "%s=%u", DYAD_KEY_BINS_ENV, m_ctx->key_bins); + DYAD_LOG_INFO(m_ctx, "%s=%u", DYAD_SERVICE_MUX_ENV, m_ctx->service_mux); + DYAD_LOG_INFO(m_ctx, "%s=%s", DYAD_KVS_NAMESPACE_ENV, m_ctx->kvs_namespace); + DYAD_LOG_INFO(m_ctx, "%s=%s", DYAD_DTL_MODE_ENV, getenv(DYAD_DTL_MODE_ENV)); } -bool dyad_stream_core::is_dyad_producer () const -{ - return m_is_prod; -} +bool dyad_stream_core::is_dyad_producer() const { return m_is_prod; } -bool dyad_stream_core::is_dyad_consumer () const -{ - return m_is_cons; -} +bool dyad_stream_core::is_dyad_consumer() const { return m_is_cons; } -bool dyad_stream_core::open_sync (const char *path) -{ - DYAD_CPP_FUNCTION(); - DYAD_CPP_FUNCTION_UPDATE ("path", path); - DYAD_LOG_DEBUG (m_ctx, "DYAD_SYNC OPEN: enters sync (\"%s\").", path); - if (!m_initialized) { - // TODO log - return true; - } +bool dyad_stream_core::open_sync(const char *path) { + DYAD_CPP_FUNCTION(); + DYAD_CPP_FUNCTION_UPDATE("path", path); + DYAD_LOG_DEBUG(m_ctx, "DYAD_SYNC OPEN: enters sync (\"%s\").", path); + if (!m_initialized) { + // TODO log + return true; + } - if (!is_dyad_consumer ()) { - return true; - } + if (!is_dyad_consumer()) { + return true; + } - dyad_rc_t rc = dyad_consume (m_ctx_mutable, path); + dyad_rc_t rc = dyad_consume(m_ctx_mutable, path); - if (DYAD_IS_ERROR (rc)) { - DPRINTF (m_ctx, "DYAD_SYNC OPEN: failed sync (\"%s\").", path); - return false; - } + if (DYAD_IS_ERROR(rc)) { + DPRINTF(m_ctx, "DYAD_SYNC OPEN: failed sync (\"%s\").", path); + return false; + } - DYAD_LOG_DEBUG(m_ctx, "DYAD_SYNC OEPN: exists sync (\"%s\").\n", path); - return true; + DYAD_LOG_DEBUG(m_ctx, "DYAD_SYNC OEPN: exists sync (\"%s\").\n", path); + return true; } -bool dyad_stream_core::close_sync (const char *path) -{ - DYAD_CPP_FUNCTION(); - DYAD_CPP_FUNCTION_UPDATE ("path", path); - DYAD_LOG_DEBUG (m_ctx, "DYAD_SYNC CLOSE: enters sync (\"%s\").\n", path); - if (!m_initialized) { - // TODO log - return true; - } +bool dyad_stream_core::close_sync(const char *path) { + DYAD_CPP_FUNCTION(); + DYAD_CPP_FUNCTION_UPDATE("path", path); + DYAD_LOG_DEBUG(m_ctx, "DYAD_SYNC CLOSE: enters sync (\"%s\").\n", path); + if (!m_initialized) { + // TODO log + return true; + } - if (!is_dyad_producer ()) { - return true; - } + if (!is_dyad_producer()) { + return true; + } - dyad_rc_t rc = dyad_produce (m_ctx_mutable, path); + dyad_rc_t rc = dyad_produce(m_ctx_mutable, path); - if (DYAD_IS_ERROR (rc)) { - DPRINTF (m_ctx, "DYAD_SYNC CLOSE: failed sync (\"%s\").\n", path); - return false; - } + if (DYAD_IS_ERROR(rc)) { + DPRINTF(m_ctx, "DYAD_SYNC CLOSE: failed sync (\"%s\").\n", path); + return false; + } - DYAD_LOG_DEBUG (m_ctx, "DYAD_SYNC CLOSE: exists sync (\"%s\").\n", path); - return true; + DYAD_LOG_DEBUG(m_ctx, "DYAD_SYNC CLOSE: exists sync (\"%s\").\n", path); + return true; } -void dyad_stream_core::set_initialized () -{ - m_initialized = true; -} +void dyad_stream_core::set_initialized() { m_initialized = true; } -bool dyad_stream_core::chk_initialized () const -{ - return m_initialized; -} +bool dyad_stream_core::chk_initialized() const { return m_initialized; } -bool dyad_stream_core::chk_fsync_write () const -{ - return m_ctx->fsync_write; -} +bool dyad_stream_core::chk_fsync_write() const { return m_ctx->fsync_write; } -int dyad_stream_core::file_lock_exclusive (int fd) const -{ - struct flock exclusive_flock; +int dyad_stream_core::file_lock_exclusive(int fd) const { + struct flock exclusive_flock; - dyad_rc_t rc = dyad_excl_flock (m_ctx, fd, &exclusive_flock); + dyad_rc_t rc = dyad_excl_flock(m_ctx, fd, &exclusive_flock); - if (DYAD_IS_ERROR (rc)) { - dyad_release_flock (m_ctx, fd, &exclusive_flock); - } + if (DYAD_IS_ERROR(rc)) { + dyad_release_flock(m_ctx, fd, &exclusive_flock); + } - return rc; + return rc; } -int dyad_stream_core::file_lock_shared (int fd) const -{ - struct flock shared_flock; +int dyad_stream_core::file_lock_shared(int fd) const { + struct flock shared_flock; - dyad_rc_t rc = dyad_excl_flock (m_ctx, fd, &shared_flock); + dyad_rc_t rc = dyad_excl_flock(m_ctx, fd, &shared_flock); - if (DYAD_IS_ERROR (rc)) { - dyad_release_flock (m_ctx, fd, &shared_flock); - } + if (DYAD_IS_ERROR(rc)) { + dyad_release_flock(m_ctx, fd, &shared_flock); + } - return rc; + return rc; } -int dyad_stream_core::file_unlock (int fd) const -{ - struct flock a_flock; +int dyad_stream_core::file_unlock(int fd) const { + struct flock a_flock; - return dyad_release_flock (m_ctx, fd, &a_flock); + return dyad_release_flock(m_ctx, fd, &a_flock); } -bool dyad_stream_core::cmp_canonical_path_prefix ( - bool is_prod, const char* const __restrict__ path) -{ - memset (upath, '\0', PATH_MAX); - return ::cmp_canonical_path_prefix (m_ctx, is_prod, path, upath, PATH_MAX); +bool dyad_stream_core::cmp_canonical_path_prefix( + bool is_prod, const char *const __restrict__ path) { + memset(upath, '\0', PATH_MAX); + return ::cmp_canonical_path_prefix(m_ctx, is_prod, path, upath, PATH_MAX); } -std::string dyad_stream_core::get_upath () const -{ - return std::string {upath}; -} +std::string dyad_stream_core::get_upath() const { return std::string{upath}; } -} // end of namespace dyad +} // end of namespace dyad /* * vi: ts=4 sw=4 expandtab diff --git a/src/dyad/utils/CMakeLists.txt b/src/dyad/utils/CMakeLists.txt index 6596eae4..ed2be4af 100644 --- a/src/dyad/utils/CMakeLists.txt +++ b/src/dyad/utils/CMakeLists.txt @@ -3,7 +3,7 @@ add_subdirectory(base64) set(DYAD_UTILS_SRC ${CMAKE_CURRENT_SOURCE_DIR}/utils.c ${CMAKE_CURRENT_SOURCE_DIR}/read_all.c) set(DYAD_UTILS_PRIVATE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/utils.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_structures.h + ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_structures_int.h ${CMAKE_CURRENT_SOURCE_DIR}/read_all.h) set(DYAD_UTILS_PUBLIC_HEADERS) @@ -14,7 +14,7 @@ set(DYAD_MURMUR3_PUBLIC_HEADERS) add_library(${PROJECT_NAME}_utils SHARED ${DYAD_UTILS_SRC} ${DYAD_UTILS_PRIVATE_HEADERS} ${DYAD_UTILS_PUBLIC_HEADERS}) set_target_properties(${PROJECT_NAME}_utils PROPERTIES CMAKE_INSTALL_RPATH - "${CMAKE_INSTALL_PREFIX}/${DYAD_LIBDIR}") + "${DYAD_INSTALL_LIBDIR}") target_link_libraries(${PROJECT_NAME}_utils PUBLIC ${PROJECT_NAME}_base64 ${PROJECT_NAME}_murmur3) @@ -29,14 +29,14 @@ endif() target_compile_definitions(${PROJECT_NAME}_utils PUBLIC DYAD_HAS_CONFIG) target_include_directories(${PROJECT_NAME}_utils PUBLIC $ - $) + $) add_library(${PROJECT_NAME}_murmur3 SHARED ${DYAD_MURMUR3_SRC}) set_target_properties(${PROJECT_NAME}_murmur3 PROPERTIES CMAKE_INSTALL_RPATH - "${CMAKE_INSTALL_PREFIX}/${DYAD_LIBDIR}") + "${DYAD_INSTALL_LIBDIR}") add_executable(test_cmp_canonical_path_prefix test_cmp_canonical_path_prefix.c - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_structures.h) + ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_structures_int.h) target_compile_definitions(test_cmp_canonical_path_prefix PUBLIC DYAD_HAS_CONFIG) target_link_libraries(test_cmp_canonical_path_prefix PUBLIC ${PROJECT_NAME}_utils) @@ -51,20 +51,17 @@ if(DYAD_PROFILER STREQUAL "DFTRACER") target_link_libraries(test_cmp_canonical_path_prefix PRIVATE ${DTRACER_LIBRARIES}) endif() - -if (TARGET DYAD_C_FLAGS_werror) - target_link_libraries(${PROJECT_NAME}_utils PRIVATE DYAD_C_FLAGS_werror) - target_link_libraries(${PROJECT_NAME}_murmur3 PRIVATE DYAD_C_FLAGS_werror) - target_link_libraries(test_murmur3 PRIVATE DYAD_C_FLAGS_werror) - target_link_libraries(test_cmp_canonical_path_prefix PRIVATE DYAD_C_FLAGS_werror) -endif () +dyad_add_werror_if_needed(${PROJECT_NAME}_utils) +dyad_add_werror_if_needed(${PROJECT_NAME}_murmur3) +dyad_add_werror_if_needed(test_murmur3) +dyad_add_werror_if_needed(test_cmp_canonical_path_prefix) install( TARGETS ${PROJECT_NAME}_utils EXPORT ${DYAD_EXPORTED_TARGETS} - LIBRARY DESTINATION ${DYAD_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${DYAD_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${DYAD_INSTALL_BIN_DIR} + LIBRARY DESTINATION ${DYAD_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${DYAD_INSTALL_LIBDIR} + RUNTIME DESTINATION ${DYAD_INSTALL_BINDIR} ) if(NOT "${DYAD_UTILS_PUBLIC_HEADERS}" STREQUAL "") dyad_install_headers("${DYAD_UTILS_PUBLIC_HEADERS}" ${CMAKE_CURRENT_SOURCE_DIR}) @@ -72,9 +69,9 @@ endif() install( TARGETS ${PROJECT_NAME}_murmur3 EXPORT ${DYAD_EXPORTED_TARGETS} - LIBRARY DESTINATION ${DYAD_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${DYAD_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${DYAD_INSTALL_BIN_DIR} + LIBRARY DESTINATION ${DYAD_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${DYAD_INSTALL_LIBDIR} + RUNTIME DESTINATION ${DYAD_INSTALL_BINDIR} ) if(NOT "${DYAD_MURMUR3_PUBLIC_HEADERS}" STREQUAL "") dyad_install_headers("${DYAD_UTILS_PUBLIC_HEADERS}" ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/src/dyad/utils/base64/CMakeLists.txt b/src/dyad/utils/base64/CMakeLists.txt index bbfaf5f1..d6708e5e 100644 --- a/src/dyad/utils/base64/CMakeLists.txt +++ b/src/dyad/utils/base64/CMakeLists.txt @@ -5,17 +5,17 @@ set(DYAD_BASE64_PUBLIC_HEADERS) add_library(${PROJECT_NAME}_base64 SHARED ${DYAD_BASE64_SRC} ${DYAD_BASE64_PRIVATE_HEADERS} ${DYAD_BASE64_PUBLIC_HEADERS}) set_target_properties(${PROJECT_NAME}_base64 PROPERTIES CMAKE_INSTALL_RPATH - "${CMAKE_INSTALL_PREFIX}/${DYAD_LIBDIR}") + "${DYAD_INSTALL_LIBDIR}") target_include_directories(${PROJECT_NAME}_base64 PUBLIC $ - $) + $) install( TARGETS ${PROJECT_NAME}_base64 EXPORT ${DYAD_EXPORTED_TARGETS} - LIBRARY DESTINATION ${DYAD_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${DYAD_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${DYAD_INSTALL_BIN_DIR} + LIBRARY DESTINATION ${DYAD_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${DYAD_INSTALL_LIBDIR} + RUNTIME DESTINATION ${DYAD_INSTALL_BINDIR} ) if(NOT "${DYAD_BASE64_PUBLIC_HEADERS}" STREQUAL "") dyad_install_headers("${DYAD_BASE64_PUBLIC_HEADERS}" ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/src/dyad/utils/test_cmp_canonical_path_prefix.c b/src/dyad/utils/test_cmp_canonical_path_prefix.c index 7a92e9ec..e71d90ce 100644 --- a/src/dyad/utils/test_cmp_canonical_path_prefix.c +++ b/src/dyad/utils/test_cmp_canonical_path_prefix.c @@ -4,54 +4,58 @@ #error "no config" #endif -#include +// clang-format off +#include #include #include #include #include #include +// clang-format on -int main (int argc, char** argv) -{ - char* prefix = NULL; - char* can_prefix = NULL; - char* path = NULL; - bool ret = false; - char upath[PATH_MAX] = {'\0'}; +int main(int argc, char **argv) { + char *prefix = NULL; + char *can_prefix = NULL; + char *path = NULL; + bool ret = false; + char upath[PATH_MAX] = {'\0'}; - if (argc < 3 || 4 < argc) { - printf ("Usage: %s prefix path [canonical_prefix]\n", argv[0]); - return EXIT_FAILURE; - } - if (argc >= 3) { - prefix = argv[1]; - path = argv[2]; - } - if (argc == 4) { - can_prefix = argv[3]; - } + if (argc < 3 || 4 < argc) { + printf("Usage: %s prefix path [canonical_prefix]\n", argv[0]); + return EXIT_FAILURE; + } + if (argc >= 3) { + prefix = argv[1]; + path = argv[2]; + } + if (argc == 4) { + can_prefix = argv[3]; + } - const size_t prefix_len = strlen (prefix); - const size_t can_prefix_len = can_prefix? strlen (can_prefix) : 0u;; - uint32_t prefix_hash = hash_str (prefix, DYAD_SEED); - uint32_t can_prefix_hash = hash_str (can_prefix, DYAD_SEED); + const size_t prefix_len = strlen(prefix); + const size_t can_prefix_len = can_prefix ? strlen(can_prefix) : 0u; + ; + uint32_t prefix_hash = hash_str(prefix, DYAD_SEED); + uint32_t can_prefix_hash = hash_str(can_prefix, DYAD_SEED); - dyad_ctx_t ctx; - ctx.cons_managed_path = prefix; - ctx.cons_real_path = can_prefix; - ctx.cons_managed_len = prefix_len; - ctx.cons_real_len = can_prefix_len; - ctx.cons_managed_hash = prefix_hash; - ctx.cons_real_hash = can_prefix_hash; - bool is_prod = false; + dyad_ctx_t ctx; + ctx.cons_managed_path = prefix; + ctx.cons_real_path = can_prefix; + ctx.cons_managed_len = prefix_len; + ctx.cons_real_len = can_prefix_len; + ctx.cons_managed_hash = prefix_hash; + ctx.cons_real_hash = can_prefix_hash; + bool is_prod = false; - ret = cmp_canonical_path_prefix (&ctx, is_prod, path, upath, PATH_MAX); - if (!ret) { - printf ("path '%s' does not include prefix '%s' ('%s')\n", path, prefix, can_prefix); - return EXIT_FAILURE; - } - printf ("path '%s' include prefix '%s' (canonical form: '%s')\n", path, prefix, can_prefix); - printf ("'%s' is under '%s'\n", upath, prefix); + ret = cmp_canonical_path_prefix(&ctx, is_prod, path, upath, PATH_MAX); + if (!ret) { + printf("path '%s' does not include prefix '%s' ('%s')\n", path, prefix, + can_prefix); + return EXIT_FAILURE; + } + printf("path '%s' include prefix '%s' (canonical form: '%s')\n", path, prefix, + can_prefix); + printf("'%s' is under '%s'\n", upath, prefix); - return EXIT_SUCCESS; + return EXIT_SUCCESS; } diff --git a/src/dyad/utils/utils.h b/src/dyad/utils/utils.h index 80533aeb..e259269e 100644 --- a/src/dyad/utils/utils.h +++ b/src/dyad/utils/utils.h @@ -19,86 +19,86 @@ #ifndef _GNU_SOURCE #define _GNU_SOURCE -#endif // _GNU_SOURCE +#endif // _GNU_SOURCE #if defined(__cplusplus) // #include // c++11 #include -#include #include +#include #else #include #include -#include #include -#endif // defined(__cplusplus) +#include +#endif // defined(__cplusplus) -#include -#include #include +#include +#include #if defined(__cplusplus) extern "C" { -#endif // defined(__cplusplus) +#endif // defined(__cplusplus) -bool file_in_read_mode (FILE* f); -bool fd_in_read_mode (int fd); -bool oflag_is_read (int oflag); -bool mode_is_read (const char* mode); +bool file_in_read_mode(FILE *f); +bool fd_in_read_mode(int fd); +bool oflag_is_read(int oflag); +bool mode_is_read(const char *mode); -void enable_debug_dyad_utils (void); -void disable_debug_dyad_utils (void); -bool check_debug_dyad_utils (void); +void enable_debug_dyad_utils(void); +void disable_debug_dyad_utils(void); +bool check_debug_dyad_utils(void); /** Return the hash value computed for the entire string with a given seed - * The return value 0 means that the string was not valid or the length was 0. */ -uint32_t hash_str (const char* str, const uint32_t seed); + * The return value 0 means that the string was not valid or the length was 0. + */ +uint32_t hash_str(const char *str, const uint32_t seed); /** Return the hash value computed for the path string up to len characters * with a given seed. - * The return value 0 means that the string was not valid or the length was 0. */ -uint32_t hash_path_prefix (const char* str, const uint32_t seed, const size_t len); + * The return value 0 means that the string was not valid or the length was 0. + */ +uint32_t hash_path_prefix(const char *str, const uint32_t seed, + const size_t len); -char* concat_str (char* __restrict__ str, - const char* __restrict__ to_append, - const char* __restrict__ connector, - size_t str_capacity); +char *concat_str(char *__restrict__ str, const char *__restrict__ to_append, + const char *__restrict__ connector, size_t str_capacity); -bool extract_user_path (const char* __restrict__ prefix, - const char* __restrict__ full, - const char* __restrict__ delim, - char* __restrict__ upath, - const size_t upath_capacity); +bool extract_user_path(const char *__restrict__ prefix, + const char *__restrict__ full, + const char *__restrict__ delim, char *__restrict__ upath, + const size_t upath_capacity); -bool cmp_canonical_path_prefix (const dyad_ctx_t* __restrict__ ctx, - const bool is_prod, - const char* __restrict__ path, - char* __restrict__ upath, - const size_t upath_capacity); +bool cmp_canonical_path_prefix(const dyad_ctx_t *__restrict__ ctx, + const bool is_prod, + const char *__restrict__ path, + char *__restrict__ upath, + const size_t upath_capacity); -int mkdir_as_needed (const char* path, const mode_t m); +int mkdir_as_needed(const char *path, const mode_t m); /// Obtain path from the file descriptor -int get_path (const int fd, const size_t max_size, char* path); +int get_path(const int fd, const size_t max_size, char *path); /// Check if the path is a directory -bool is_path_dir (const char* path); +bool is_path_dir(const char *path); /// Check if the file identified by the file descriptor is a directory -bool is_fd_dir (int fd); +bool is_fd_dir(int fd); #if DYAD_SPIN_WAIT /// Try access file info -bool get_stat (const char* path, unsigned int max_retry, long ns_sleep); -#endif // DYAD_SPIN_WAIT +bool get_stat(const char *path, unsigned int max_retry, long ns_sleep); +#endif // DYAD_SPIN_WAIT -ssize_t get_file_size (int fd); +ssize_t get_file_size(int fd); -dyad_rc_t dyad_excl_flock (const dyad_ctx_t* __restrict__ ctx, int fd, - struct flock* __restrict__ lock); -dyad_rc_t dyad_shared_flock (const dyad_ctx_t* __restrict__ ctx, int fd, - struct flock* __restrict__ lock); -dyad_rc_t dyad_release_flock (const dyad_ctx_t* __restrict__ ctx, int fd, - struct flock* __restrict__ lock); +dyad_rc_t dyad_excl_flock(const dyad_ctx_t *__restrict__ ctx, int fd, + struct flock *__restrict__ lock); +dyad_rc_t dyad_shared_flock(const dyad_ctx_t *__restrict__ ctx, int fd, + struct flock *__restrict__ lock); +dyad_rc_t dyad_release_flock(const dyad_ctx_t *__restrict__ ctx, int fd, + struct flock *__restrict__ lock); #if DYAD_SYNC_DIR /** @@ -106,11 +106,11 @@ dyad_rc_t dyad_release_flock (const dyad_ctx_t* __restrict__ ctx, int fd, * For example, if path is "/a/b", then fsync on "/a". * This cannot be used with DYAD interception. */ -int sync_containing_dir (const char* path); -#endif // DYAD_SYNC_DIR +int sync_containing_dir(const char *path); +#endif // DYAD_SYNC_DIR #if defined(__cplusplus) } -#endif // defined(__cplusplus) +#endif // defined(__cplusplus) -#endif // DYAD_UTILS_UTILS_H +#endif // DYAD_UTILS_UTILS_H diff --git a/src/dyad/wrapper/CMakeLists.txt b/src/dyad/wrapper/CMakeLists.txt index 0978e19e..f208456c 100644 --- a/src/dyad/wrapper/CMakeLists.txt +++ b/src/dyad/wrapper/CMakeLists.txt @@ -1,26 +1,24 @@ set(DYAD_WRAPPER_SRC ${CMAKE_CURRENT_SOURCE_DIR}/wrapper.c) set(DYAD_WRAPPER_PRIVATE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../utils/utils.h - ${CMAKE_CURRENT_SOURCE_DIR}/../common/dyad_dtl.h - ${CMAKE_CURRENT_SOURCE_DIR}/../core/dyad_ctx.h - ${CMAKE_CURRENT_SOURCE_DIR}/../core/dyad_core.h) + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/common/dyad_dtl.h + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include/dyad/core/dyad_ctx.h + ${CMAKE_CURRENT_SOURCE_DIR}/../client/dyad_client_int.h) set(DYAD_WRAPPER_PUBLIC_HEADERS) add_library(${PROJECT_NAME}_wrapper SHARED ${DYAD_WRAPPER_SRC} ${DYAD_WRAPPER_PRIVATE_HEADERS} ${DYAD_WRAPPER_PUBLIC_HEADERS}) set_target_properties(${PROJECT_NAME}_wrapper PROPERTIES CMAKE_INSTALL_RPATH - "${CMAKE_INSTALL_PREFIX}/${DYAD_LIBDIR}") -target_link_libraries(${PROJECT_NAME}_wrapper PRIVATE ${PROJECT_NAME}_ctx ${PROJECT_NAME}_core) + "${DYAD_INSTALL_LIBDIR}") +target_link_libraries(${PROJECT_NAME}_wrapper PRIVATE ${PROJECT_NAME}_ctx ${PROJECT_NAME}_client) target_link_libraries(${PROJECT_NAME}_wrapper PRIVATE ${PROJECT_NAME}_utils flux::core) target_compile_definitions(${PROJECT_NAME}_wrapper PUBLIC BUILDING_DYAD=1) target_compile_definitions(${PROJECT_NAME}_wrapper PUBLIC DYAD_HAS_CONFIG) target_include_directories(${PROJECT_NAME}_wrapper PUBLIC $ - $) + $) target_include_directories(${PROJECT_NAME}_wrapper SYSTEM PRIVATE ${FluxCore_INCLUDE_DIRS}) -if (TARGET DYAD_C_FLAGS_werror) - target_link_libraries(${PROJECT_NAME}_wrapper PRIVATE DYAD_C_FLAGS_werror) -endif () +dyad_add_werror_if_needed(${PROJECT_NAME}_wrapper) set(FLUX_BARRIER_SRC ${CMAKE_CURRENT_SOURCE_DIR}/flux_barrier.c) set(FLUX_BARRIER_PRIVATE_HEADERS) @@ -33,9 +31,9 @@ target_include_directories(${PROJECT_NAME}_flux_barrier SYSTEM PRIVATE ${FluxCor install( TARGETS ${PROJECT_NAME}_wrapper EXPORT ${DYAD_EXPORTED_TARGETS} - LIBRARY DESTINATION ${DYAD_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${DYAD_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${DYAD_INSTALL_BIN_DIR} + LIBRARY DESTINATION ${DYAD_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${DYAD_INSTALL_LIBDIR} + RUNTIME DESTINATION ${DYAD_INSTALL_BINDIR} ) if(NOT "${DYAD_WRAPPER_PUBLIC_HEADERS}" STREQUAL "") @@ -45,9 +43,9 @@ endif() install( TARGETS ${PROJECT_NAME}_flux_barrier EXPORT ${DYAD_EXPORTED_TARGETS} - LIBRARY DESTINATION ${DYAD_INSTALL_LIB_DIR} - ARCHIVE DESTINATION ${DYAD_INSTALL_LIB_DIR} - RUNTIME DESTINATION ${DYAD_INSTALL_BIN_DIR} + LIBRARY DESTINATION ${DYAD_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${DYAD_INSTALL_LIBDIR} + RUNTIME DESTINATION ${DYAD_INSTALL_BINDIR} ) if(NOT "${FLUX_BARRIER_PUBLIC_HEADERS}" STREQUAL "") diff --git a/src/dyad/wrapper/wrapper.c b/src/dyad/wrapper/wrapper.c index 4a174a7c..035baa4c 100644 --- a/src/dyad/wrapper/wrapper.c +++ b/src/dyad/wrapper/wrapper.c @@ -16,7 +16,7 @@ #ifndef _GNU_SOURCE #define _GNU_SOURCE -#endif // _GNU_SOURCE +#endif // _GNU_SOURCE #if defined(__cplusplus) #include @@ -26,7 +26,7 @@ #include #include #include -using namespace std; // std::clock () +using namespace std; // std::clock () // #include // c++11 #else #include @@ -37,18 +37,17 @@ using namespace std; // std::clock () #include #include #include -#endif // defined(__cplusplus) +#endif // defined(__cplusplus) #include +#include #include #include #include #include -#include -#include #include #include -#include // dirname +#include // dirname #include #ifdef __cplusplus @@ -57,12 +56,12 @@ extern "C" { static __thread const dyad_ctx_t *ctx = NULL; static __thread dyad_ctx_t *ctx_mutable = NULL; -static void dyad_wrapper_init (void) __attribute__ ((constructor)); -static void dyad_wrapper_fini (void) __attribute__ ((destructor)); +static void dyad_wrapper_init(void) __attribute__((constructor)); +static void dyad_wrapper_fini(void) __attribute__((destructor)); #if DYAD_SYNC_DIR -int sync_directory (const char *path); -#endif // DYAD_SYNC_DIR +int sync_directory(const char *path); +#endif // DYAD_SYNC_DIR /***************************************************************************** * * @@ -78,14 +77,13 @@ int sync_directory (const char *path); * @return 1 if the file descriptor is write-only, 0 if not, and -1 * if there was an error in fcntl */ -static inline int is_wronly (int fd) -{ - int rc = fcntl (fd, F_GETFL); - if (rc == -1) - return -1; - if ((rc & O_ACCMODE) == O_WRONLY) - return 1; - return 0; +static inline int is_wronly(int fd) { + int rc = fcntl(fd, F_GETFL); + if (rc == -1) + return -1; + if ((rc & O_ACCMODE) == O_WRONLY) + return 1; + return 0; } /***************************************************************************** @@ -94,324 +92,322 @@ static inline int is_wronly (int fd) * * *****************************************************************************/ -void dyad_wrapper_init (void) -{ +void dyad_wrapper_init(void) { #if DYAD_PROFILER == 3 - DFTRACER_C_FINI (); + DFTRACER_C_FINI(); #endif - DYAD_C_FUNCTION_START (); - dyad_ctx_init (DYAD_COMM_RECV, NULL); - ctx = ctx_mutable = dyad_ctx_get (); - DYAD_LOG_INFO (ctx, "DYAD Wrapper Initialized"); - DYAD_C_FUNCTION_END (); + DYAD_C_FUNCTION_START(); + dyad_ctx_init(DYAD_COMM_RECV, NULL); + ctx = ctx_mutable = dyad_ctx_get(); + DYAD_LOG_INFO(ctx, "DYAD Wrapper Initialized"); + DYAD_C_FUNCTION_END(); } -void dyad_wrapper_fini () -{ - DYAD_C_FUNCTION_START (); - DYAD_LOG_INFO (ctx, "DYAD Wrapper Finalized"); - dyad_ctx_fini (); - DYAD_C_FUNCTION_END (); +void dyad_wrapper_fini() { + DYAD_C_FUNCTION_START(); + DYAD_LOG_INFO(ctx, "DYAD Wrapper Finalized"); + dyad_ctx_fini(); + DYAD_C_FUNCTION_END(); #if DYAD_PROFILER == 3 - DFTRACER_C_FINI (); + DFTRACER_C_FINI(); #endif } -DYAD_DLL_EXPORTED int open (const char *path, int oflag, ...) -{ - DYAD_C_FUNCTION_START (); - DYAD_C_FUNCTION_UPDATE_STR ("path", "path"); - char *error = NULL; - typedef int (*open_ptr_t) (const char *, int, mode_t, ...); - open_ptr_t func_ptr = NULL; - int mode = 0; - char upath[PATH_MAX + 1] = {'\0'}; - - if (oflag & O_CREAT) { - va_list arg; - va_start (arg, oflag); - mode = va_arg (arg, int); - va_end (arg); - } - - // https://stackoverflow.com/questions/14134245/iso-c-void-and-function-pointers - // func_ptr = (open_ptr_t)dlsym (RTLD_NEXT, "open"); - *(void **)&func_ptr = dlsym (RTLD_NEXT, "open"); - if ((error = dlerror ())) { - DPRINTF (ctx, "DYAD_SYNC: error in dlsym: %s", error); - DYAD_C_FUNCTION_END (); - return -1; - } - - if ((mode != O_RDONLY) || is_path_dir (path)) { - // TODO: make sure if the directory mode is consistent - goto real_call; - } - - if (!(ctx && ctx->h) || (ctx && !ctx->reenter)) { - IPRINTF (ctx, "DYAD_SYNC: open sync not applicable for \"%s\".", path); - goto real_call; - } - - IPRINTF (ctx, "DYAD_SYNC: enters open sync (\"%s\").", path); - if (DYAD_IS_ERROR (dyad_consume (ctx_mutable, path))) { - DPRINTF (ctx, "DYAD_SYNC: failed open sync (\"%s\").", path); - goto real_call; - } - IPRINTF (ctx, "DYAD_SYNC: exists open sync (\"%s\").", path); +DYAD_DLL_EXPORTED int open(const char *path, int oflag, ...) { + DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_UPDATE_STR("path", "path"); + char *error = NULL; + typedef int (*open_ptr_t)(const char *, int, mode_t, ...); + open_ptr_t func_ptr = NULL; + int mode = 0; + char upath[PATH_MAX + 1] = {'\0'}; + + if (oflag & O_CREAT) { + va_list arg; + va_start(arg, oflag); + mode = va_arg(arg, int); + va_end(arg); + } + + // https://stackoverflow.com/questions/14134245/iso-c-void-and-function-pointers + // func_ptr = (open_ptr_t)dlsym (RTLD_NEXT, "open"); + *(void **)&func_ptr = dlsym(RTLD_NEXT, "open"); + if ((error = dlerror())) { + DPRINTF(ctx, "DYAD_SYNC: error in dlsym: %s", error); + DYAD_C_FUNCTION_END(); + return -1; + } + + if ((mode != O_RDONLY) || is_path_dir(path)) { + // TODO: make sure if the directory mode is consistent + goto real_call; + } + + if (!(ctx && ctx->h) || (ctx && !ctx->reenter)) { + IPRINTF(ctx, "DYAD_SYNC: open sync not applicable for \"%s\".", path); + goto real_call; + } + + IPRINTF(ctx, "DYAD_SYNC: enters open sync (\"%s\").", path); + if (DYAD_IS_ERROR(dyad_consume(ctx_mutable, path))) { + DPRINTF(ctx, "DYAD_SYNC: failed open sync (\"%s\").", path); + goto real_call; + } + IPRINTF(ctx, "DYAD_SYNC: exists open sync (\"%s\").", path); real_call:; - int ret = (func_ptr (path, oflag, mode)); - - // This lock is to protect the file being produced by a producer - // from a consumer that has direct access to the file. For example, - // either the file is on a shared storage or the consumer is on - // the same node as where the producer is. - if ((ret > 0) && (mode == O_WRONLY || mode == O_APPEND) && !is_path_dir (path)) { - if ((ctx->relative_to_managed_path - && (strncmp (path, DYAD_PATH_DELIM, ctx->delim_len) != 0)) - || cmp_canonical_path_prefix (ctx, true, path, upath, PATH_MAX)) { - struct flock exclusive_lock; - dyad_rc_t rc = dyad_excl_flock (ctx, ret, &exclusive_lock); - if (DYAD_IS_ERROR (rc)) { - dyad_release_flock (ctx, ret, &exclusive_lock); - } - } + int ret = (func_ptr(path, oflag, mode)); + + // This lock is to protect the file being produced by a producer + // from a consumer that has direct access to the file. For example, + // either the file is on a shared storage or the consumer is on + // the same node as where the producer is. + if ((ret > 0) && (mode == O_WRONLY || mode == O_APPEND) && + !is_path_dir(path)) { + if ((ctx->relative_to_managed_path && + (strncmp(path, DYAD_PATH_DELIM, ctx->delim_len) != 0)) || + cmp_canonical_path_prefix(ctx, true, path, upath, PATH_MAX)) { + struct flock exclusive_lock; + dyad_rc_t rc = dyad_excl_flock(ctx, ret, &exclusive_lock); + if (DYAD_IS_ERROR(rc)) { + dyad_release_flock(ctx, ret, &exclusive_lock); + } } + } - DYAD_C_FUNCTION_END (); - return ret; + DYAD_C_FUNCTION_END(); + return ret; } -DYAD_DLL_EXPORTED FILE *fopen (const char *path, const char *mode) -{ - DYAD_C_FUNCTION_START (); - DYAD_C_FUNCTION_UPDATE_STR ("path", "path"); - char *error = NULL; - typedef FILE *(*fopen_ptr_t) (const char *, const char *); - fopen_ptr_t func_ptr = NULL; - char upath[PATH_MAX + 1] = {'\0'}; - - // func_ptr = (fopen_ptr_t)dlsym (RTLD_NEXT, "fopen"); - *(void **)&func_ptr = dlsym (RTLD_NEXT, "fopen"); - if ((error = dlerror ())) { - DPRINTF (ctx, "DYAD_SYNC: error in dlsym: %s\n", error); - DYAD_C_FUNCTION_END (); - return NULL; - } - - if ((strcmp (mode, "r") != 0) || is_path_dir (path)) { - // TODO: make sure if the directory mode is consistent - goto real_call; - } - - if (!(ctx && ctx->h) || (ctx && !ctx->reenter) || !path) { - IPRINTF (ctx, "DYAD_SYNC: fopen sync not applicable for \"%s\".\n", ((path) ? path : "")); - goto real_call; - } - - IPRINTF (ctx, "DYAD_SYNC: enters fopen sync (\"%s\").\n", path); - if (DYAD_IS_ERROR (dyad_consume (ctx_mutable, path))) { - DPRINTF (ctx, "DYAD_SYNC: failed fopen sync (\"%s\").\n", path); - goto real_call; - } - IPRINTF (ctx, "DYAD_SYNC: exits fopen sync (\"%s\").\n", path); +DYAD_DLL_EXPORTED FILE *fopen(const char *path, const char *mode) { + DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_UPDATE_STR("path", "path"); + char *error = NULL; + typedef FILE *(*fopen_ptr_t)(const char *, const char *); + fopen_ptr_t func_ptr = NULL; + char upath[PATH_MAX + 1] = {'\0'}; + + // func_ptr = (fopen_ptr_t)dlsym (RTLD_NEXT, "fopen"); + *(void **)&func_ptr = dlsym(RTLD_NEXT, "fopen"); + if ((error = dlerror())) { + DPRINTF(ctx, "DYAD_SYNC: error in dlsym: %s\n", error); + DYAD_C_FUNCTION_END(); + return NULL; + } + + if ((strcmp(mode, "r") != 0) || is_path_dir(path)) { + // TODO: make sure if the directory mode is consistent + goto real_call; + } + + if (!(ctx && ctx->h) || (ctx && !ctx->reenter) || !path) { + IPRINTF(ctx, "DYAD_SYNC: fopen sync not applicable for \"%s\".\n", + ((path) ? path : "")); + goto real_call; + } + + IPRINTF(ctx, "DYAD_SYNC: enters fopen sync (\"%s\").\n", path); + if (DYAD_IS_ERROR(dyad_consume(ctx_mutable, path))) { + DPRINTF(ctx, "DYAD_SYNC: failed fopen sync (\"%s\").\n", path); + goto real_call; + } + IPRINTF(ctx, "DYAD_SYNC: exits fopen sync (\"%s\").\n", path); real_call:; - FILE *fh = (func_ptr (path, mode)); - - // This lock is to protect the file being produced by a producer - // from a consumer that has direct access to the file. For example, - // either the file is on a shared storage or the consumer is on - // the same node as where the producer is. - if ((fh != NULL) && ((strcmp (mode, "w") == 0) || (strcmp (mode, "a") == 0)) - && !is_path_dir (path)) { - if ((ctx->relative_to_managed_path - && (strncmp (path, DYAD_PATH_DELIM, ctx->delim_len) != 0)) - || cmp_canonical_path_prefix (ctx, true, path, upath, PATH_MAX)) { - int fd = fileno (fh); - struct flock exclusive_lock; - dyad_rc_t rc = dyad_excl_flock (ctx, fd, &exclusive_lock); - if (DYAD_IS_ERROR (rc)) { - dyad_release_flock (ctx, fd, &exclusive_lock); - } - } + FILE *fh = (func_ptr(path, mode)); + + // This lock is to protect the file being produced by a producer + // from a consumer that has direct access to the file. For example, + // either the file is on a shared storage or the consumer is on + // the same node as where the producer is. + if ((fh != NULL) && ((strcmp(mode, "w") == 0) || (strcmp(mode, "a") == 0)) && + !is_path_dir(path)) { + if ((ctx->relative_to_managed_path && + (strncmp(path, DYAD_PATH_DELIM, ctx->delim_len) != 0)) || + cmp_canonical_path_prefix(ctx, true, path, upath, PATH_MAX)) { + int fd = fileno(fh); + struct flock exclusive_lock; + dyad_rc_t rc = dyad_excl_flock(ctx, fd, &exclusive_lock); + if (DYAD_IS_ERROR(rc)) { + dyad_release_flock(ctx, fd, &exclusive_lock); + } } - DYAD_C_FUNCTION_END (); - return fh; + } + DYAD_C_FUNCTION_END(); + return fh; } -DYAD_DLL_EXPORTED int close (int fd) -{ - DYAD_C_FUNCTION_START (); - DYAD_C_FUNCTION_UPDATE_INT ("fd", fd); - bool to_sync = false; - char *error = NULL; - typedef int (*close_ptr_t) (int); - close_ptr_t func_ptr = NULL; - char path[PATH_MAX + 1] = {'\0'}; - int rc = 0; - - // func_ptr = (close_ptr_t)dlsym (RTLD_NEXT, "close"); - *(void **)&func_ptr = dlsym (RTLD_NEXT, "close"); - if ((error = dlerror ())) { - DPRINTF (ctx, "DYAD_SYNC: error in dlsym: %s\n", error); - DYAD_C_FUNCTION_END (); - return -1; // return the failure code - } - - if ((fd < 0) || (ctx == NULL) || (ctx->h == NULL) || !ctx->reenter) { +DYAD_DLL_EXPORTED int close(int fd) { + DYAD_C_FUNCTION_START(); + DYAD_C_FUNCTION_UPDATE_INT("fd", fd); + bool to_sync = false; + char *error = NULL; + typedef int (*close_ptr_t)(int); + close_ptr_t func_ptr = NULL; + char path[PATH_MAX + 1] = {'\0'}; + int rc = 0; + + // func_ptr = (close_ptr_t)dlsym (RTLD_NEXT, "close"); + *(void **)&func_ptr = dlsym(RTLD_NEXT, "close"); + if ((error = dlerror())) { + DPRINTF(ctx, "DYAD_SYNC: error in dlsym: %s\n", error); + DYAD_C_FUNCTION_END(); + return -1; // return the failure code + } + + if ((fd < 0) || (ctx == NULL) || (ctx->h == NULL) || !ctx->reenter) { #if defined(IPRINTF_DEFINED) - if (ctx == NULL) { - IPRINTF (ctx, "DYAD_SYNC: close sync not applicable. (no context)\n"); - } else if (ctx->h == NULL) { - IPRINTF (ctx, "DYAD_SYNC: close sync not applicable. (no flux)\n"); - } else if (!ctx->reenter) { - IPRINTF (ctx, "DYAD_SYNC: close sync not applicable. (no reenter)\n"); - } else if (fd >= 0) { - IPRINTF (ctx, - "DYAD_SYNC: close sync not applicable. (invalid file " - "descriptor)\n"); - } -#endif // defined(IPRINTF_DEFINED) - to_sync = false; - goto real_call; + if (ctx == NULL) { + IPRINTF(ctx, "DYAD_SYNC: close sync not applicable. (no context)\n"); + } else if (ctx->h == NULL) { + IPRINTF(ctx, "DYAD_SYNC: close sync not applicable. (no flux)\n"); + } else if (!ctx->reenter) { + IPRINTF(ctx, "DYAD_SYNC: close sync not applicable. (no reenter)\n"); + } else if (fd >= 0) { + IPRINTF(ctx, "DYAD_SYNC: close sync not applicable. (invalid file " + "descriptor)\n"); } +#endif // defined(IPRINTF_DEFINED) + to_sync = false; + goto real_call; + } - if (is_fd_dir (fd)) { - // TODO: make sure if the directory mode is consistent - goto real_call; - } + if (is_fd_dir(fd)) { + // TODO: make sure if the directory mode is consistent + goto real_call; + } - if (get_path (fd, PATH_MAX - 1, path) < 0) { - DYAD_LOG_DEBUG (ctx, "DYAD_SYNC: unable to obtain file path from a descriptor.\n"); - to_sync = false; - goto real_call; - } + if (get_path(fd, PATH_MAX - 1, path) < 0) { + DYAD_LOG_DEBUG( + ctx, "DYAD_SYNC: unable to obtain file path from a descriptor.\n"); + to_sync = false; + goto real_call; + } - to_sync = true; + to_sync = true; -real_call:; // semicolon here to avoid the error - // "a label can only be part of a statement and a declaration is not a - // statement" +real_call:; // semicolon here to avoid the error + // "a label can only be part of a statement and a declaration is not a + // statement" - int wronly = is_wronly (fd); + int wronly = is_wronly(fd); - if (wronly == -1) { - DPRINTF (ctx, "Failed to check the mode of the file with fcntl: %s\n", strerror (errno)); - } + if (wronly == -1) { + DPRINTF(ctx, "Failed to check the mode of the file with fcntl: %s\n", + strerror(errno)); + } - if (to_sync && wronly == 1) { - if (ctx->fsync_write) { - fsync (fd); + if (to_sync && wronly == 1) { + if (ctx->fsync_write) { + fsync(fd); #if DYAD_SYNC_DIR - dyad_sync_directory (ctx, path); -#endif // DYAD_SYNC_DIR - } - - struct flock exclusive_lock; - dyad_release_flock (ctx, fd, &exclusive_lock); - rc = func_ptr (fd); - if (rc != 0) { - DPRINTF (ctx, "Failed close (\"%s\").: %s\n", path, strerror (errno)); - } - IPRINTF (ctx, "DYAD_SYNC: enters close sync (\"%s\").\n", path); - if (DYAD_IS_ERROR (dyad_produce (ctx_mutable, path))) { - DPRINTF (ctx, "DYAD_SYNC: failed close sync (\"%s\").\n", path); - } - IPRINTF (ctx, "DYAD_SYNC: exits close sync (\"%s\").\n", path); - } else { - rc = func_ptr (fd); + dyad_sync_directory(ctx, path); +#endif // DYAD_SYNC_DIR } - DYAD_C_FUNCTION_END (); - return rc; -} -DYAD_DLL_EXPORTED int fclose (FILE *fp) -{ - DYAD_C_FUNCTION_START (); - bool to_sync = false; - char *error = NULL; - typedef int (*fclose_ptr_t) (FILE *); - fclose_ptr_t func_ptr = NULL; - char path[PATH_MAX + 1] = {'\0'}; - int rc = 0; - int fd = 0; - - // func_ptr = (fclose_ptr_t)dlsym (RTLD_NEXT, "fclose"); - *(void **)&func_ptr = dlsym (RTLD_NEXT, "fclose"); - if ((error = dlerror ())) { - DYAD_LOG_DEBUG (ctx, "DYAD_SYNC: error in dlsym: %s\n", error); - DYAD_C_FUNCTION_END (); - return EOF; // return the failure code + struct flock exclusive_lock; + dyad_release_flock(ctx, fd, &exclusive_lock); + rc = func_ptr(fd); + if (rc != 0) { + DPRINTF(ctx, "Failed close (\"%s\").: %s\n", path, strerror(errno)); + } + IPRINTF(ctx, "DYAD_SYNC: enters close sync (\"%s\").\n", path); + if (DYAD_IS_ERROR(dyad_produce(ctx_mutable, path))) { + DPRINTF(ctx, "DYAD_SYNC: failed close sync (\"%s\").\n", path); } + IPRINTF(ctx, "DYAD_SYNC: exits close sync (\"%s\").\n", path); + } else { + rc = func_ptr(fd); + } + DYAD_C_FUNCTION_END(); + return rc; +} - if ((fp == NULL) || (ctx == NULL) || (ctx->h == NULL) || !ctx->reenter) { +DYAD_DLL_EXPORTED int fclose(FILE *fp) { + DYAD_C_FUNCTION_START(); + bool to_sync = false; + char *error = NULL; + typedef int (*fclose_ptr_t)(FILE *); + fclose_ptr_t func_ptr = NULL; + char path[PATH_MAX + 1] = {'\0'}; + int rc = 0; + int fd = 0; + + // func_ptr = (fclose_ptr_t)dlsym (RTLD_NEXT, "fclose"); + *(void **)&func_ptr = dlsym(RTLD_NEXT, "fclose"); + if ((error = dlerror())) { + DYAD_LOG_DEBUG(ctx, "DYAD_SYNC: error in dlsym: %s\n", error); + DYAD_C_FUNCTION_END(); + return EOF; // return the failure code + } + + if ((fp == NULL) || (ctx == NULL) || (ctx->h == NULL) || !ctx->reenter) { #if defined(IPRINTF_DEFINED) - if (ctx == NULL) { - IPRINTF (ctx, "DYAD_SYNC: fclose sync not applicable. (no context)\n"); - } else if (ctx->h == NULL) { - IPRINTF (ctx, "DYAD_SYNC: fclose sync not applicable. (no flux)\n"); - } else if (!ctx->reenter) { - IPRINTF (ctx, "DYAD_SYNC: fclose sync not applicable. (no reenter)\n"); - } else if (fp == NULL) { - IPRINTF (ctx, - "DYAD_SYNC: fclose sync not applicable. (invalid file " - "pointer)\n"); - } -#endif // defined(IPRINTF_DEFINED) - to_sync = false; - goto real_call; + if (ctx == NULL) { + IPRINTF(ctx, "DYAD_SYNC: fclose sync not applicable. (no context)\n"); + } else if (ctx->h == NULL) { + IPRINTF(ctx, "DYAD_SYNC: fclose sync not applicable. (no flux)\n"); + } else if (!ctx->reenter) { + IPRINTF(ctx, "DYAD_SYNC: fclose sync not applicable. (no reenter)\n"); + } else if (fp == NULL) { + IPRINTF(ctx, "DYAD_SYNC: fclose sync not applicable. (invalid file " + "pointer)\n"); } +#endif // defined(IPRINTF_DEFINED) + to_sync = false; + goto real_call; + } - if (is_fd_dir (fileno (fp))) { - // TODO: make sure if the directory mode is consistent - goto real_call; - } + if (is_fd_dir(fileno(fp))) { + // TODO: make sure if the directory mode is consistent + goto real_call; + } - if (get_path (fileno (fp), PATH_MAX - 1, path) < 0) { - DYAD_LOG_DEBUG (ctx, "DYAD_SYNC: unable to obtain file path from a descriptor.\n"); - to_sync = false; - goto real_call; - } + if (get_path(fileno(fp), PATH_MAX - 1, path) < 0) { + DYAD_LOG_DEBUG( + ctx, "DYAD_SYNC: unable to obtain file path from a descriptor.\n"); + to_sync = false; + goto real_call; + } - to_sync = true; + to_sync = true; real_call:; - fd = fileno (fp); + fd = fileno(fp); - int wronly = is_wronly (fd); + int wronly = is_wronly(fd); - if (wronly == -1) { - DPRINTF (ctx, "Failed to check the mode of the file with fcntl: %s\n", strerror (errno)); - } + if (wronly == -1) { + DPRINTF(ctx, "Failed to check the mode of the file with fcntl: %s\n", + strerror(errno)); + } - if (to_sync && wronly == 1) { - if (ctx->fsync_write) { - fflush (fp); - fsync (fd); + if (to_sync && wronly == 1) { + if (ctx->fsync_write) { + fflush(fp); + fsync(fd); #if DYAD_SYNC_DIR - dyad_sync_directory (ctx, path); -#endif // DYAD_SYNC_DIR - } - - struct flock exclusive_lock; - dyad_release_flock (ctx, fd, &exclusive_lock); - rc = func_ptr (fp); - if (rc != 0) { - DPRINTF (ctx, "Failed fclose (\"%s\").\n", path); - } - IPRINTF (ctx, "DYAD_SYNC: enters fclose sync (\"%s\").\n", path); - if (DYAD_IS_ERROR (dyad_produce (ctx_mutable, path))) { - DPRINTF (ctx, "DYAD_SYNC: failed fclose sync (\"%s\").\n", path); - } - IPRINTF (ctx, "DYAD_SYNC: exits fclose sync (\"%s\").\n", path); - } else { - rc = func_ptr (fp); + dyad_sync_directory(ctx, path); +#endif // DYAD_SYNC_DIR + } + + struct flock exclusive_lock; + dyad_release_flock(ctx, fd, &exclusive_lock); + rc = func_ptr(fp); + if (rc != 0) { + DPRINTF(ctx, "Failed fclose (\"%s\").\n", path); + } + IPRINTF(ctx, "DYAD_SYNC: enters fclose sync (\"%s\").\n", path); + if (DYAD_IS_ERROR(dyad_produce(ctx_mutable, path))) { + DPRINTF(ctx, "DYAD_SYNC: failed fclose sync (\"%s\").\n", path); } - DYAD_C_FUNCTION_END (); - return rc; + IPRINTF(ctx, "DYAD_SYNC: exits fclose sync (\"%s\").\n", path); + } else { + rc = func_ptr(fp); + } + DYAD_C_FUNCTION_END(); + return rc; } #ifdef __cplusplus diff --git a/tests/unit/data_plane/data_plane.cpp b/tests/unit/data_plane/data_plane.cpp index b69b6173..49bdab81 100644 --- a/tests/unit/data_plane/data_plane.cpp +++ b/tests/unit/data_plane/data_plane.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include @@ -59,7 +59,7 @@ TEST_CASE("RemoteDataBandwidth", "[files= " + std::to_string(args.number_of_file dyad_metadata_t mdata; mdata.owner_rank = neighour_broker_idx; size_t data_len = args.request_size * args.iteration; - char* file_data = NULL; + char *file_data = NULL; for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { sprintf(filename, "%s_%u_%zu.bat", args.filename.c_str(), neighour_broker_idx, file_idx); @@ -100,7 +100,8 @@ TEST_CASE("RemoteDataAggBandwidth", "[files= " + std::to_string(args.number_of_f dyad_metadata_t mdata; mdata.owner_rank = neighour_broker_idx; size_t data_len = args.request_size * args.iteration; - if (info.rank % args.process_per_node != 0) usleep(10000); + if (info.rank % args.process_per_node != 0) + usleep(10000); for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { sprintf(upath, "%s_%u_%zu.bat", args.filename.c_str(), neighour_broker_idx, file_idx); @@ -140,7 +141,7 @@ TEST_CASE("LocalProcessDataBandwidth", "[files= " + std::to_string(args.number_o Timer data_time; char filename[4096]; size_t data_len = args.request_size * args.iteration; - char* file_data = (char*)malloc(data_len); + char *file_data = (char *)malloc(data_len); for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { sprintf(filename, "%s/%s_%u_%zu.bat", args.dyad_managed_dir.c_str(), args.filename.c_str(), info.broker_idx, file_idx); @@ -184,7 +185,7 @@ TEST_CASE("LocalNodeDataBandwidth", "[files= " + std::to_string(args.number_of_f Timer data_time; char filename[4096]; size_t data_len = args.request_size * args.iteration; - char* file_data = (char*)malloc(data_len); + char *file_data = (char *)malloc(data_len); for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { sprintf(filename, "%s/%s_%u_%zu.bat", args.dyad_managed_dir.c_str(), args.filename.c_str(), info.broker_idx, file_idx); diff --git a/tests/unit/dyad_core/core_functions.cpp b/tests/unit/dyad_core/core_functions.cpp index 11125435..0915737b 100644 --- a/tests/unit/dyad_core/core_functions.cpp +++ b/tests/unit/dyad_core/core_functions.cpp @@ -1,33 +1,32 @@ -#include +#include /** * Test cases */ -TEST_CASE("gen_path_key", - "[module=dyad_core]" - "[method=gen_path_key]") { +TEST_CASE("gen_path_key", "[module=dyad_core]" + "[method=gen_path_key]") { SECTION("should generate path key") { - const char* str = "test_string"; + const char *str = "test_string"; char path_key[256] = {'\0'}; int result = gen_path_key(str, path_key, sizeof(path_key), 3, 5); REQUIRE(result == 0); REQUIRE(strcmp(path_key, "") != 0); } SECTION("should_return_minus_one_when_input_string_is_null") { - const char* str = nullptr; + const char *str = nullptr; char path_key[256] = {'\0'}; int result = gen_path_key(str, path_key, sizeof(path_key), 3, 5); REQUIRE(result == -1); } SECTION("should_handle_input_string_of_length_less_than_128_bytes") { - const char* str = "short_string"; + const char *str = "short_string"; char path_key[256] = {'\0'}; int result = gen_path_key(str, path_key, sizeof(path_key), 3, 5); REQUIRE(result == 0); REQUIRE(strcmp(path_key, "") != 0); } SECTION("should_handle_input_string_of_length_more_than_128_bytes") { - const char* str = + const char *str = "this_is_a_very_long_string_that_is_more_than_128_bytes_long_this_is_a_" "very_long_string_that_is_more_than_128_bytes_long_this_is_a_very_long_" "string_that_is_more_than_128_bytes_long_this_is_a_very_long_string_" @@ -43,28 +42,28 @@ TEST_CASE("gen_path_key", REQUIRE(strcmp(path_key, "") != 0); } SECTION("should_generate_path_key_with_depth_and_width_specified") { - const char* str = "test_string"; + const char *str = "test_string"; char path_key[256] = {'\0'}; int result = gen_path_key(str, path_key, sizeof(path_key), 3, 5); REQUIRE(result == 0); REQUIRE(strcmp(path_key, "") != 0); } SECTION("should_generate_path_key_with_depth_and_width_set_to_0") { - const char* str = "test_string"; + const char *str = "test_string"; char path_key[256] = {'\0'}; int result = gen_path_key(str, path_key, sizeof(path_key), 0, 0); REQUIRE(result == 0); REQUIRE(strcmp(path_key, "") != 0); } SECTION("should_return_minus_1_when_input_string_is_NULL") { - const char* str = NULL; + const char *str = NULL; char path_key[256] = {'\0'}; int result = gen_path_key(str, path_key, sizeof(path_key), 3, 5); REQUIRE(result == -1); } SECTION("should_return_minus_1_when_path_key_is_NULL") { - const char* str = "test_string"; - char* path_key = NULL; + const char *str = "test_string"; + char *path_key = NULL; int result = gen_path_key(str, path_key, sizeof(path_key), 3, 5); REQUIRE(result == -1); }