Merge pull request ceph#15055 from mogeb/wip-with-instrument-functions

cmake: Add -finstrument-functions flag to OSD code

Reviewed-by: Kefu Chai <[email protected]>

CMakeLists.txt

@@ -435,6 +435,8 @@ if(${WITH_LTTNG})
   endif()
 endif(${WITH_LTTNG})
 
+option(WITH_OSD_INSTRUMENT_FUNCTIONS OFF)
+
 #option for Babeltrace
 option(HAVE_BABELTRACE "Babeltrace libraries are enabled" ON)
 if(${HAVE_BABELTRACE})

src/CMakeLists.txt

@@ -39,7 +39,9 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unknown-pragmas")
 if(CMAKE_CXX_COMPILER_ID STREQUAL GNU)
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -rdynamic")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wstrict-null-sentinel -Woverloaded-virtual")
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pie")
+  if(NOT WITH_OSD_INSTRUMENT_FUNCTIONS)
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pie")
+  endif()
 elseif(CMAKE_CXX_COMPILER_ID STREQUAL Clang)
   set(CMAKE_EXE_LINKER_FLAGS "-Wl,-export-dynamic")
   set(CMAKE_LINKER_FLAGS "${CMAKE_LINKER_FLAGS} -rdynamic -Wl,-export-dynamic -export-dynamic")

src/ceph_osd.cc

@@ -56,6 +56,10 @@ TracepointProvider::Traits osd_tracepoint_traits("libosd_tp.so",
                                                  "osd_tracing");
 TracepointProvider::Traits os_tracepoint_traits("libos_tp.so",
                                                 "osd_objectstore_tracing");
+#ifdef WITH_OSD_INSTRUMENT_FUNCTIONS
+TracepointProvider::Traits cyg_profile_traits("libcyg_profile_tp.so",
+                                                 "osd_function_tracing");
+#endif
 
 } // anonymous namespace
 
@@ -579,6 +583,9 @@ int main(int argc, const char **argv)
 
   TracepointProvider::initialize<osd_tracepoint_traits>(g_ceph_context);
   TracepointProvider::initialize<os_tracepoint_traits>(g_ceph_context);
+#ifdef WITH_OSD_INSTRUMENT_FUNCTIONS
+  TracepointProvider::initialize<cyg_profile_traits>(g_ceph_context);
+#endif
 
   MonClient mc(g_ceph_context);
   if (mc.build_initial_monmap() < 0)

src/common/config_opts.h

@@ -900,6 +900,7 @@ OPTION(osd_fast_fail_on_connection_refused, OPT_BOOL, true) // immediately mark
 
 OPTION(osd_pg_object_context_cache_count, OPT_INT, 64)
 OPTION(osd_tracing, OPT_BOOL, false) // true if LTTng-UST tracepoints should be enabled
+OPTION(osd_function_tracing, OPT_BOOL, false) // true if function instrumentation should use LTTng
 
 OPTION(osd_fast_info, OPT_BOOL, true) // use fast info attr, if we can
 

src/include/config-h.in.cmake

@@ -169,6 +169,9 @@
 /* Define if you want to use LTTng */
 #cmakedefine WITH_LTTNG
 
+/* Define if you want to OSD function instrumentation */
+#cmakedefine WITH_OSD_INSTRUMENT_FUNCTIONS
+
 /* Define if you want to use Babeltrace */
 #cmakedefine WITH_BABELTRACE
 

src/osd/CMakeLists.txt

@@ -2,6 +2,13 @@ set(osdc_osd_srcs
   ${CMAKE_SOURCE_DIR}/src/osdc/Objecter.cc
   ${CMAKE_SOURCE_DIR}/src/osdc/Striper.cc)
 
+if(WITH_OSD_INSTRUMENT_FUNCTIONS AND CMAKE_CXX_COMPILER_ID STREQUAL GNU)
+  set(GCC_C_FLAGS "-finstrument-functions")
+  set(GCC_C_FLAGS "${GCC_C_FLAGS} -finstrument-functions-exclude-function-list=_mm_loadu_si128,_mm_cmpeq_epi32,_mm_movemask_epi8")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${GCC_C_FLAGS}")
+  set(osd_cyg_functions_src ${CMAKE_SOURCE_DIR}/src/tracing/cyg_profile_functions.c)
+endif()
+
 set(osd_srcs
   OSD.cc
   Watch.cc
@@ -23,6 +30,7 @@ set(osd_srcs
   ECUtil.cc
   ExtentCache.cc
   ${CMAKE_SOURCE_DIR}/src/common/TrackedOp.cc
+  ${osd_cyg_functions_src}
   ${osdc_osd_srcs})
 if(HAS_VTA)
   set_source_files_properties(osdcap.cc
@@ -41,3 +49,6 @@ endif()
 if(WITH_LTTNG AND WITH_EVENTTRACE)
   add_dependencies(osd eventtrace_tp)
 endif()
+if(WITH_OSD_INSTRUMENT_FUNCTIONS)
+  add_dependencies(osd cyg_profile_tp)
+endif()

src/tracing/CMakeLists.txt

@@ -48,6 +48,10 @@ add_tracing_library(rbd_tp librbd.tp 1.0.0)
 add_tracing_library(os_tp objectstore.tp 1.0.0)
 
 install(TARGETS rados_tp osd_tp rbd_tp os_tp DESTINATION ${CMAKE_INSTALL_LIBDIR})
+if(WITH_OSD_INSTRUMENT_FUNCTIONS)
+  add_tracing_library(cyg_profile_tp cyg_profile.tp 1.0.0)
+  install(TARGETS cyg_profile_tp DESTINATION ${CMAKE_INSTALL_LIBDIR})
+endif()
 if(WITH_LTTNG AND WITH_EVENTTRACE)
   add_tracing_library(eventtrace_tp eventtrace.tp 1.0.0)
   install(TARGETS eventtrace_tp DESTINATION ${CMAKE_INSTALL_LIBDIR})

src/tracing/README.md

@@ -26,4 +26,26 @@ provider shared object, in which `TRACEPOINT_DEFINE` should be defined. See
 Place the `.tp` and the `.c` files into the `src/tracing` directory
 and modify the CMake file `src/tracing/CMakeLists.txt` accordingly.
 
+Function Instrumentation
+========================
+Ceph supports instrumentation using GCC's `-finstrument-functions` flag.
+Supported CMake flags are:
+
+*   `-DWITH_OSD_INSTRUMENT_FUNCTIONS=ON`: instrument OSD code
+
+Note that this instrumentation adds an extra function call on each function entry
+and exit of Ceph code. This option is currently only supported with GCC. Using it
+with Clang has no effect.
+
+The only function tracing implementation at the moment is done using LTTng UST.
+In order to use it, Ceph needs to be configured with LTTng using `-DWITH_LTTNG=ON`.
+[TraceCompass](http://www.tracecompass.org) can be used to generate flame
+charts/graphs and other metrics.
+
+It is also possible to use [libbabeltrace](http://diamon.org/babeltrace/#docs)
+to write custom analysis. The entry and exit tracepoints are called
+`lttng_ust_cyg_profile:func_enter` and `lttng_ust_cyg_profile:func_exit`
+respectively. The payload variable `addr` holds the address of the function
+called and the payload variable `call_site` holds the address where it is called.
+`nm` can be used to resolve function addresses (`addr` to function name).
 

src/tracing/cyg_profile.c

@@ -0,0 +1,6 @@
+#define TRACEPOINT_CREATE_PROBES
+/*
+ * The header containing our TRACEPOINT_EVENTs.
+ */
+#include "tracing/cyg_profile.h"
+

src/tracing/cyg_profile.tp

@@ -0,0 +1,23 @@
+#include "include/int_types.h"
+
+TRACEPOINT_EVENT(lttng_ust_cyg_profile, func_entry,
+    TP_ARGS(
+        void *,  func_addr,
+        void *,  call_site),
+    TP_FIELDS(
+          ctf_integer_hex(unsigned long, addr, func_addr)
+          ctf_integer_hex(unsigned long, call_site, call_site)
+    )
+)
+
+TRACEPOINT_EVENT(lttng_ust_cyg_profile, func_exit,
+    TP_ARGS(
+          void *,  func_addr,
+          void *,  call_site
+    ),
+    TP_FIELDS(
+          ctf_integer_hex(unsigned long, addr, func_addr)
+          ctf_integer_hex(unsigned long, call_site, call_site)
+    )
+)
+

src/tracing/cyg_profile_functions.c

@@ -0,0 +1,31 @@
+#include "acconfig.h"
+
+#ifdef WITH_LTTNG
+#define TRACEPOINT_DEFINE
+#define TRACEPOINT_PROBE_DYNAMIC_LINKAGE
+#include "tracing/cyg_profile.h"
+#undef TRACEPOINT_PROBE_DYNAMIC_LINKAGE
+#undef TRACEPOINT_DEFINE
+#endif
+
+void __cyg_profile_func_enter(void *this_fn, void *call_site)
+    __attribute__((no_instrument_function));
+
+void __cyg_profile_func_exit(void *this_fn, void *call_site)
+    __attribute__((no_instrument_function));
+
+
+void __cyg_profile_func_enter(void *this_fn, void *call_site)
+{
+#ifdef WITH_LTTNG
+    tracepoint(lttng_ust_cyg_profile, func_entry, this_fn, call_site);
+#endif
+}
+
+void __cyg_profile_func_exit(void *this_fn, void *call_site)
+{
+#ifdef WITH_LTTNG
+    tracepoint(lttng_ust_cyg_profile, func_exit, this_fn, call_site);
+#endif
+}
+