Merge pull request ceph#18672 from jcsp/wip-21999

mgr: emit cluster log message on serve() exception Reviewed-by: Sage Weil <[email protected]>
ijjorama · Nov 4, 2017 · 15e0224 · 15e0224
2 parents b6fe5d3 + f4763c3
commit 15e0224
Show file tree

Hide file tree

Showing 7 changed files with 39 additions and 11 deletions.
diff --git a/src/mgr/ActivePyModule.h b/src/mgr/ActivePyModule.h
@@ -60,8 +60,9 @@ class ActivePyModule : public PyModuleRunner
 public:
   ActivePyModule(const std::string &module_name_,
       PyObject *pClass_,
-      const SafeThreadState &my_ts_)
-    : PyModuleRunner(module_name_, pClass_, my_ts_)
+      const SafeThreadState &my_ts_,
+      LogChannelRef clog_)
+    : PyModuleRunner(module_name_, pClass_, my_ts_, clog_)
   {}
 
   int load(ActivePyModules *py_modules);

diff --git a/src/mgr/ActivePyModules.cc b/src/mgr/ActivePyModules.cc
@@ -330,7 +330,7 @@ int ActivePyModules::start_one(std::string const &module_name,
 
   modules[module_name].reset(new ActivePyModule(
       module_name, pClass,
-      pMyThreadState));
+      pMyThreadState, clog));
 
   int r = modules[module_name]->load(this);
   if (r != 0) {

diff --git a/src/mgr/PyModuleRegistry.cc b/src/mgr/PyModuleRegistry.cc
@@ -346,7 +346,7 @@ void PyModuleRegistry::standby_start(MonClient *monc)
 
   dout(4) << "Starting modules in standby mode" << dendl;
 
-  standby_modules.reset(new StandbyPyModules(monc, mgr_map));
+  standby_modules.reset(new StandbyPyModules(monc, mgr_map, clog));
 
   std::set<std::string> failed_modules;
   for (const auto &i : modules) {

diff --git a/src/mgr/PyModuleRunner.cc b/src/mgr/PyModuleRunner.cc
@@ -54,6 +54,23 @@ int PyModuleRunner::serve()
   if (pValue != NULL) {
     Py_DECREF(pValue);
   } else {
+    // This is not a very informative log message because it's an
+    // unknown/unexpected exception that we can't say much about.
+
+    // Peek at the exception for the cluster log, before
+    // dumping the backtrace to log log next.
+    PyObject *ptype, *pvalue, *ptraceback;
+    PyErr_Fetch(&ptype, &pvalue, &ptraceback);
+    assert(ptype);
+    assert(pvalue);
+    PyObject *pvalue_str = PyObject_Str(pvalue);
+    std::string exc_msg = PyString_AsString(pvalue_str);
+    Py_DECREF(pvalue_str);
+    PyErr_Restore(ptype, pvalue, ptraceback);
+
+    clog->error() << "Unhandled exception from module '" << module_name
+                  << "' while running on mgr." << g_conf->name.get_id()
+                  << ": " << exc_msg;
     derr << module_name << ".serve:" << dendl;
     derr << handle_pyerror() << dendl;
     return -EINVAL;

diff --git a/src/mgr/PyModuleRunner.h b/src/mgr/PyModuleRunner.h
@@ -15,6 +15,7 @@
 #pragma once
 
 #include "common/Thread.h"
+#include "common/LogClient.h"
 #include "mgr/Gil.h"
 
 /**
@@ -36,6 +37,8 @@ class PyModuleRunner
   // Populated when we construct our instance of pClass in load()
   PyObject *pClassInstance = nullptr;
 
+  LogChannelRef clog;
+
   class PyModuleRunnerThread : public Thread
   {
     PyModuleRunner *mod;
@@ -55,10 +58,12 @@ class PyModuleRunner
   PyModuleRunner(
       const std::string &module_name_,
       PyObject *pClass_,
-      const SafeThreadState &pMyThreadState_)
+      const SafeThreadState &pMyThreadState_,
+      LogChannelRef clog_)
     : 
       module_name(module_name_),
       pClass(pClass_), pMyThreadState(pMyThreadState_),
+      clog(clog_),
       thread(this)
   {
     assert(pClass != nullptr);

diff --git a/src/mgr/StandbyPyModules.cc b/src/mgr/StandbyPyModules.cc
@@ -34,8 +34,9 @@
 std::string handle_pyerror();
 
 
-StandbyPyModules::StandbyPyModules(MonClient *monc_, const MgrMap &mgr_map_)
-    : monc(monc_), load_config_thread(monc, &state)
+StandbyPyModules::StandbyPyModules(MonClient *monc_, const MgrMap &mgr_map_,
+    LogChannelRef clog_)
+    : monc(monc_), load_config_thread(monc, &state), clog(clog_)
 {
   state.set_mgr_map(mgr_map_);
 }
@@ -86,7 +87,7 @@ int StandbyPyModules::start_one(std::string const &module_name,
   modules[module_name].reset(new StandbyPyModule(
       state,
       module_name, pClass,
-      pMyThreadState));
+      pMyThreadState, clog));
 
   if (modules.size() == 1) {
     load_config_thread.create("LoadConfig");

diff --git a/src/mgr/StandbyPyModules.h b/src/mgr/StandbyPyModules.h
@@ -91,9 +91,10 @@ class StandbyPyModule : public PyModuleRunner
       StandbyPyModuleState &state_,
       const std::string &module_name_,
       PyObject *pClass_,
-      const SafeThreadState &pMyThreadState_)
+      const SafeThreadState &pMyThreadState_,
+      LogChannelRef clog_)
     :
-      PyModuleRunner(module_name_, pClass_, pMyThreadState_),
+      PyModuleRunner(module_name_, pClass_, pMyThreadState_, clog_),
       state(state_)
   {
   }
@@ -129,11 +130,14 @@ class StandbyPyModules
 
   LoadConfigThread load_config_thread;
 
+  LogChannelRef clog;
+
 public:
 
   StandbyPyModules(
       MonClient *monc_,
-      const MgrMap &mgr_map_);
+      const MgrMap &mgr_map_,
+      LogChannelRef clog_);
 
   int start_one(std::string const &module_name,
                 PyObject *pClass,