changes to support cpp-logger and dlio profiler

1. switch to c++ 17
2. added dlio_profiler events to functions.
3. fixed header for flux log ucx

.github/prod-cons/dyad_producer.sh

@@ -8,8 +8,8 @@ export DYAD_PATH_PRODUCER=${DYAD_PATH}_producer
 mkdir -p ${DYAD_PATH_PRODUCER}
 echo "Loading DYAD module"
 
-echo flux module load ${DYAD_INSTALL_PREFIX}/lib/dyad.so  $DYAD_PATH_PRODUCER $DYAD_DTL_MODE
-flux module load ${DYAD_INSTALL_PREFIX}/lib/dyad.so  $DYAD_PATH_PRODUCER $DYAD_DTL_MODE
+echo flux module load ${DYAD_INSTALL_PREFIX}/lib/dyad.so  --mode="$DYAD_DTL_MODE" $DYAD_PATH_PRODUCER
+flux module load ${DYAD_INSTALL_PREFIX}/lib/dyad.so  --mode="$DYAD_DTL_MODE" $DYAD_PATH_PRODUCER
 
 if [[ "$mode" == "${valid_modes[0]}" ]]; then
     echo ${GITHUB_WORKSPACE}/docs/demos/ecp_feb_2023/c_prod 10 $DYAD_PATH_PRODUCER

.github/workflows/compile_test.yaml

@@ -214,6 +214,7 @@ jobs:
           fi
           spack external find
           spack spec flux-core@${FLUX_VERSION}
+          spack spec cpp-logger@dev
           if [[ $DYAD_DTL_MODE == 'UCX' ]]; then
             spack spec [email protected]
           fi
@@ -227,8 +228,9 @@ jobs:
           if [[ $DYAD_PROFILER_MODE == 'CALIPER' ]]; then
             spack install -j4 caliper
           fi
+          spack install cpp-logger@dev
           mkdir -p ${DYAD_INSTALL_PREFIX}
-          spack view --verbose symlink ${DYAD_INSTALL_PREFIX} flux-core@${FLUX_VERSION}
+          spack view --verbose symlink ${DYAD_INSTALL_PREFIX} flux-core@${FLUX_VERSION} cpp-logger@dev
           if [[ $DYAD_DTL_MODE == 'UCX' ]]; then
             spack view --verbose symlink ${DYAD_INSTALL_PREFIX} [email protected]
           fi
@@ -250,7 +252,7 @@ jobs:
           if [[ $DYAD_DTL_MODE == 'UCX' ]]; then
             CONFIGURE_FLAGS="-DDYAD_ENABLE_UCX_DATA=ON -DDYAD_PROFILER=NONE"
           fi
-          TEST_CONF="-DDYAD_LIBDIR_AS_LIB:BOOL=ON -DCMAKE_BUILD_TYPE=Debug"
+          TEST_CONF="-DDYAD_LIBDIR_AS_LIB:BOOL=ON -DCMAKE_BUILD_TYPE=Debug -DDYAD_LOGGER=CPP_LOGGER -DDYAD_LOGGER_LEVEL=DEBUG"
           cmake --version
           cmake -DCMAKE_INSTALL_PREFIX=${DYAD_INSTALL_PREFIX} ${CONFIGURE_FLAGS} ${TEST_CONF} ..
           make VERBOSE=1 install -j

CMakeLists.txt

@@ -98,7 +98,11 @@ if (DYAD_ENABLE_UCX_DATA)
     set (DYAD_ENABLE_UCX_DTL 1)
 endif ()
 set(DYAD_PROFILER "NONE" CACHE STRING "Profiler to use for DYAD")
-set_property(CACHE DYAD_PROFILER PROPERTY STRINGS PERFFLOW_ASPECT CALIPER NONE)
+set_property(CACHE DYAD_PROFILER PROPERTY STRINGS PERFFLOW_ASPECT CALIPER DLIO_PROFILER NONE)
+set(DYAD_LOGGER "NONE" CACHE STRING "Logger to use for DYAD")
+set_property(CACHE DYAD_LOGGER PROPERTY STRINGS FLUX CPP_LOGGER NONE)
+set(DYAD_LOGGER_LEVEL "NONE" CACHE STRING "Logging level to use for DYAD")
+set_property(CACHE DYAD_LOGGER_LEVEL PROPERTY STRINGS DEBUG INFO WARN ERROR NONE)
 
 
 #------------------------------------------------------------------------------
@@ -116,9 +120,51 @@ if(CMAKE_BUILD_TYPE STREQUAL "")
     set(CMAKE_BUILD_TYPE "Release") # Setting default as Release
 endif()
 
+if(DYAD_PROFILER STREQUAL "PERFFLOW_ASPECT")
+    set(DYAD_PROFILER_PERFFLOW_ASPECT 1)
+elseif(DYAD_PROFILER STREQUAL "CALIPER")
+    set(DYAD_PROFILER_CALIPER 1)
+elseif(DYAD_PROFILER STREQUAL "DLIO_PROFILER")
+    set(DYAD_PROFILER_DLIO_PROFILER 1)
+else()
+    set(DYAD_PROFILER_NONE 1)
+endif()
+message(STATUS "[${PROJECT_NAME}] setting DYAD_PROFILER as ${DYAD_PROFILER}")
+
+if(DYAD_LOGGER STREQUAL "FLUX")
+    set(DYAD_LOGGER_FLUX 1)
+elseif(DYAD_LOGGER STREQUAL "CPP_LOGGER")
+    set(DYAD_LOGGER_CPP_LOGGER 1)
+endif()
+message(STATUS "[${PROJECT_NAME}] setting DYAD_LOGGER as ${DYAD_LOGGER}")
+
+
 if(CMAKE_BUILD_TYPE STREQUAL "Debug")
-    set(DYAD_FULL_DEBUG 1)
-    set(DYAD_LOGGING_ON 1)
+    if (DYAD_LOGGER STREQUAL "NONE")
+        message(STATUS "-- [${PROJECT_NAME}] disabling logger as no logger is selected")
+        set(DYAD_LOGGER_NO_LOG 1)
+    else()
+        if (DYAD_LOGGER_LEVEL STREQUAL "NONE" OR DYAD_LOGGER_LEVEL STREQUAL "DEBUG")
+            set(DYAD_LOGGER_LEVEL_ERROR 1)
+            set(DYAD_LOGGER_LEVEL_WARN 1)
+            set(DYAD_LOGGER_LEVEL_INFO 1)
+            set(DYAD_LOGGER_LEVEL_DEBUG 1)
+        elseif(DYAD_LOGGER_LEVEL STREQUAL "ERROR")
+            set(DYAD_LOGGER_LEVEL_ERROR 1)
+        elseif(DYAD_LOGGER_LEVEL STREQUAL "WARN")
+            set(DYAD_LOGGER_LEVEL_ERROR 1)
+            set(DYAD_LOGGER_LEVEL_WARN 1)
+        elseif(DYAD_LOGGER_LEVEL STREQUAL "INFO")
+            set(DYAD_LOGGER_LEVEL_ERROR 1)
+            set(DYAD_LOGGER_LEVEL_WARN 1)
+            set(DYAD_LOGGER_LEVEL_INFO 1)
+        else()
+            set(DYAD_LOGGER_NO_LOG 1)
+        endif()
+        message(STATUS "[${PROJECT_NAME}] setting DYAD_LOGGER_LEVEL as ${DYAD_LOGGER_LEVEL}")
+    endif()
+else()
+    set(DYAD_LOGGER_NO_LOG 1)
 endif()
 
 if (NOT DEFINED BUILD_SHARED_LIBS)
@@ -172,12 +218,31 @@ endif ()
 
 # Optional Dependencies
 # =============================================================================
+if (DYAD_LOGGER STREQUAL "CPP_LOGGER")
+    find_package(cpp-logger REQUIRED)
+    if (${CPP_LOGGER_FOUND})
+        message(STATUS "[${PROJECT_NAME}] found cpp-logger at ${CPP_LOGGER_INCLUDE_DIRS}")
+        include_directories(${CPP_LOGGER_INCLUDE_DIRS})
+        set(DEPENDENCY_LIB ${DEPENDENCY_LIB} ${CPP_LOGGER_LIBRARIES})
+    else ()
+        message(FATAL_ERROR "-- [${PROJECT_NAME}] cpp-logger is not found but selected in cmake options for ${PROJECT_NAME} build")
+    endif ()
+endif()
 if(DYAD_PROFILER STREQUAL "PERFFLOW_ASPECT")
     find_package(perfflowaspect REQUIRED)
     if (${perfflowaspect_FOUND})
         message(STATUS "[${PROJECT_NAME}] found perfflowaspect at ${perfflowaspect_INCLUDE_DIRS}")
     else ()
-        message(FATAL_ERROR "-- [${PROJECT_NAME}] ucx is needed for ${PROJECT_NAME} build")
+        message(FATAL_ERROR "-- [${PROJECT_NAME}] perfflowaspect is not found but selected in cmake options for ${PROJECT_NAME} build")
+    endif ()
+elseif(DYAD_PROFILER STREQUAL "DLIO_PROFILER")
+    find_package(dlio_profiler REQUIRED IMPORTED)
+    if (${DLIO_PROFILER_FOUND})
+        message(STATUS "[${PROJECT_NAME}] found dlio_profiler at ${DLIO_PROFILER_INCLUDE_DIRS}")
+        include_directories(${DLIO_PROFILER_INCLUDE_DIRS})
+        set(DEPENDENCY_LIB ${DEPENDENCY_LIB} ${DLIO_PROFILER_LIBRARIES})
+    else ()
+        message(FATAL_ERROR "-- [${PROJECT_NAME}] dlio_profiler is not found but selected in cmake options ${PROJECT_NAME} build")
     endif ()
 endif()
 if(DYAD_ENABLE_UCX_DATA)
@@ -248,7 +313,7 @@ export(EXPORT ${DYAD_EXPORTED_TARGETS}
 
 # Write the configure file
 configure_file("${CMAKE_SOURCE_DIR}/cmake/configure_files/dyad_config.hpp.in"
-               "${CMAKE_BINARY_DIR}/src/dyad/dyad_config.hpp" @ONLY)
+               "${CMAKE_INCLUDE_OUTPUT_DIRECTORY}/dyad/dyad_config.hpp" @ONLY)
 
 
 ################################################################
@@ -315,7 +380,7 @@ endif (WIP_package_config)
 #...........................................
 
 install(FILES
-        "${CMAKE_BINARY_DIR}/src/dyad/dyad_config.hpp"
+        "${CMAKE_INCLUDE_OUTPUT_DIRECTORY}/dyad/dyad_config.hpp"
         DESTINATION
         ${DYAD_INSTALL_INCLUDE_DIR}/dyad)
 
@@ -385,6 +450,10 @@ string(APPEND _str
   "  DYAD_DATA_PLANE:             ${DYAD_DATA_PLANE}\n")
 string(APPEND _str
   "  DYAD_PROFILER:               ${DYAD_PROFILER}\n")
+string(APPEND _str
+  "  DYAD_LOGGER:                 ${DYAD_LOGGER}\n")
+string(APPEND _str
+  "  DYAD_LOGGER_LEVEL:           ${DYAD_LOGGER_LEVEL}\n")
 string(APPEND _str
   "  FluxCore_ROOT_DIR:           ${FluxCore_ROOT_DIR}\n")
 string(APPEND _str

cmake/configure_files/dyad_config.hpp.in

@@ -25,10 +25,22 @@
 
 /* Macro flags */
 #cmakedefine DYAD_GNU_LINUX 1
-#cmakedefine DYAD_LOGGING_ON 1
-#cmakedefine DYAD_FULL_DEBUG 1
 #cmakedefine DYAD_ENABLE_UCX_DTL 1
 #cmakedefine DYAD_HAS_STD_FILESYSTEM 1
+// Profiler
+#cmakedefine DYAD_PROFILER_PERFFLOW_ASPECT 1
+#cmakedefine DYAD_PROFILER_CALIPER 1
+#cmakedefine DYAD_PROFILER_DLIO_PROFILER 1
+#cmakedefine DYAD_PROFILER_NONE 1
+// Logger
+#cmakedefine DYAD_LOGGER_FLUX 1
+#cmakedefine DYAD_LOGGER_CPP_LOGGER 1
+// Logger level
+#cmakedefine DYAD_LOGGER_NO_LOG 1
+#cmakedefine DYAD_LOGGER_LEVEL_ERROR 1
+#cmakedefine DYAD_LOGGER_LEVEL_WARN 1
+#cmakedefine DYAD_LOGGER_LEVEL_INFO 1
+#cmakedefine DYAD_LOGGER_LEVEL_DEBUG 1
 
 
 #endif /* DYAD_CONFIG_H */

cmake/modules/SetupCompiler.cmake

@@ -9,7 +9,7 @@ set(CMAKE_C_EXTENSIONS OFF)
 # Add _POSIX_C_SOURCE to avoid warning on setenv() as C_EXTENSIONS is off
 add_definitions(-D_POSIX_C_SOURCE=200809L)
 
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 #set(CMAKE_C_VISIBILITY_PRESET hidden)

include/dyad/stream/dyad_params.hpp

@@ -12,7 +12,7 @@
 #define DYAD_STREAM_DYAD_PARAMS_HPP
 
 #if defined(DYAD_HAS_CONFIG)
-#include "dyad/dyad_config.hpp"
+#include <dyad/dyad_config.hpp>
 #else
 #error "no config"
 #endif

include/dyad/stream/dyad_stream_api.hpp

@@ -12,7 +12,7 @@
 #define DYAD_STREAM_DYAD_STREAM_API_HPP
 
 #if defined(DYAD_HAS_CONFIG)
-#include "dyad/dyad_config.hpp"
+#include <dyad/dyad_config.hpp>
 #else
 #error "no config"
 #endif

include/dyad/stream/dyad_stream_core.hpp

@@ -12,7 +12,7 @@
 #define DYAD_STREAM_DYAD_STREAM_CORE_HPP
 
 #if defined(DYAD_HAS_CONFIG)
-#include "dyad/dyad_config.hpp"
+#include <dyad/dyad_config.hpp>
 #else
 #error "no config"
 #endif

pydyad/pydyad/bindings.py

@@ -3,7 +3,8 @@
 import enum
 from pathlib import Path
 import warnings
-
+from dlio_profiler.logger import dlio_logger, fn_interceptor
+dlio_log = fn_interceptor("DYAD_PY")
 
 DYAD_LIB_DIR = None
 
@@ -20,6 +21,7 @@ class DyadCtxWrapper(ctypes.Structure):
     _fields_ = [
         ("h", ctypes.POINTER(FluxHandle)),
         ("dtl_handle", ctypes.POINTER(DyadDTLHandle)),
+        ("fname", ctypes.c_char_p),
         ("debug", ctypes.c_bool),
         ("check", ctypes.c_bool),
         ("reenter", ctypes.c_bool),
@@ -72,7 +74,7 @@ class DTLMode(enum.IntEnum):
 
 
 class Dyad:
-
+    @dlio_log.log_init
     def __init__(self):
         self.initialized = False
         self.dyad_core_lib = None
@@ -145,7 +147,9 @@ def __init__(self):
         self.dyad_finalize.restype = ctypes.c_int
         self.cons_path = None
         self.prod_path = None
-
+        self.log_inst = None
+
+    @dlio_log.log
     def init(
         self,
         debug=False,
@@ -160,6 +164,7 @@ def init(
         cons_managed_path=None,
         dtl_mode=DTLMode.DYAD_DTL_FLUX_RPC,
     ):
+        self.log_inst = dlio_logger.initialize_log(logfile=None, data_dir=None, process_id=-1)
         if self.dyad_init is None:
             warnings.warn(
                 "Trying to initialize DYAD when libdyad_core.so was not found",
@@ -191,7 +196,8 @@ def init(
         else:
             self.cons_path = Path(self.ctx.contents.cons_managed_path.decode("utf-8")).expanduser().resolve()
         self.initialized = True
-
+
+    @dlio_log.log
     def init_env(self):
         if self.dyad_init_env is None:
             warnings.warn(
@@ -215,7 +221,8 @@ def init_env(self):
 
     def __del__(self):
         self.finalize()
-
+
+    @dlio_log.log
     def produce(self, fname):
         if self.dyad_produce is None:
             warnings.warn(
@@ -229,7 +236,8 @@ def produce(self, fname):
         )
         if int(res) != 0:
             raise RuntimeError("Cannot produce data with DYAD!")
-
+
+    @dlio_log.log
     def get_metadata(self, fname, should_wait=False, raw=False):
         if self.dyad_get_metadata is None:
             warnings.warn(
@@ -249,7 +257,8 @@ def get_metadata(self, fname, should_wait=False, raw=False):
         if not raw:
             return DyadMetadata(mdata, self)
         return mdata
-
+
+    @dlio_log.log
     def free_metadata(self, metadata_wrapper):
         if self.dyad_free_metadata is None:
             warnings.warn("Trying to free DYAD metadata when libdyad_core.so was not found", RuntimeWarning)
@@ -259,7 +268,8 @@ def free_metadata(self, metadata_wrapper):
         )
         if int(res) != 0:
             raise RuntimeError("Could not free DYAD metadata")
-
+
+    @dlio_log.log
     def consume(self, fname):
         if self.dyad_consume is None:
             warnings.warn(
@@ -273,10 +283,13 @@ def consume(self, fname):
         )
         if int(res) != 0:
             raise RuntimeError("Cannot consume data with DYAD!")
-
+
+    @dlio_log.log
     def finalize(self):
         if not self.initialized:
             return
+        if self.log_inst:
+            self.log_inst.finalize()
         if self.dyad_finalize is None:
             warnings.warn(
                 "Trying to finalize DYAD when libdyad_core.so was not found",

pydyad/pydyad/context.py

@@ -4,13 +4,15 @@
 import io
 from pathlib import Path
 
-
+from dlio_profiler.logger import fn_interceptor
+dlio_log = fn_interceptor("DYAD_PY")
 DYAD_IO = None
 
 
 # The design of dyad_open is based on the PEP for Python's 'with' syntax:
 # https://peps.python.org/pep-0343/
 @contextmanager
+@dlio_log.log
 def dyad_open(*args, dyad_ctx=None, register_dyad_ctx=False, **kwargs):
     global DYAD_IO
     local_dyad_io = dyad_ctx

pydyad/setup.cfg

@@ -7,4 +7,4 @@ classifier =
 [options]
 python_requires = >=3.7
 install_requires =
-    numpy
+    numpy

src/dyad/CMakeLists.txt

@@ -1,10 +1,9 @@
 include_directories(${CMAKE_SOURCE_DIR}/src)  # usual source code of the current project
-
+include_directories(${CMAKE_BINARY_DIR}/include)
 add_subdirectory(utils)
 add_subdirectory(dtl)
 add_subdirectory(core)
 add_subdirectory(modules)
 add_subdirectory(wrapper)
 add_subdirectory(stream)
-add_subdirectory(perf)