Merge pull request ceph#1057 from ceph/wip-exception-logging

Improve exception logging

Reviewed-by: Dan Mick <[email protected]>

teuthology/__init__.py

@@ -19,6 +19,7 @@
     subprocess=False,
 )
 import sys
+from gevent.hub import Hub
 
 # Don't write pyc files
 sys.dont_write_bytecode = True
@@ -80,3 +81,30 @@ def setup_log_file(log_path):
     handler.setFormatter(formatter)
     root_logger.addHandler(handler)
     root_logger.info('teuthology version: %s', __version__)
+
+
+def install_except_hook():
+    """
+    Install an exception hook that first logs any uncaught exception, then
+    raises it.
+    """
+    def log_exception(exc_type, exc_value, exc_traceback):
+        if not issubclass(exc_type, KeyboardInterrupt):
+            log.critical("Uncaught exception", exc_info=(exc_type, exc_value,
+                                                         exc_traceback))
+        sys.__excepthook__(exc_type, exc_value, exc_traceback)
+    sys.excepthook = log_exception
+
+
+def patch_gevent_hub_error_handler():
+    Hub._origin_handle_error = Hub.handle_error
+
+    def custom_handle_error(self, context, type, value, tb):
+        if not issubclass(type, Hub.SYSTEM_ERROR + Hub.NOT_ERROR):
+            log.error("Uncaught exception (Hub)", exc_info=(type, value, tb))
+
+        self._origin_handle_error(context, type, value, tb)
+
+    Hub.handle_error = custom_handle_error
+
+patch_gevent_hub_error_handler()

teuthology/run.py

@@ -4,9 +4,9 @@
 import contextlib
 import sys
 import logging
-from traceback import format_tb
 
 import teuthology
+from teuthology import install_except_hook
 from . import report
 from .job_status import get_status
 from .misc import get_user, merge_configs
@@ -33,18 +33,6 @@ def set_up_logging(verbose, archive):
     install_except_hook()
 
 
-def install_except_hook():
-    def log_exception(exception_class, exception, traceback):
-        logging.critical(''.join(format_tb(traceback)))
-        if not exception.message:
-            logging.critical(exception_class.__name__)
-            return
-        logging.critical('{0}: {1}'.format(
-            exception_class.__name__, exception))
-
-    sys.excepthook = log_exception
-
-
 def write_initial_metadata(archive, config, name, description, owner):
     if archive is not None:
         with file(os.path.join(archive, 'pid'), 'w') as f:

teuthology/worker.py

@@ -8,7 +8,7 @@
 
 from datetime import datetime
 
-from teuthology import setup_log_file
+from teuthology import setup_log_file, install_except_hook
 from . import beanstalk
 from . import report
 from . import safepath
@@ -58,19 +58,6 @@ def load_config(ctx=None):
             teuth_config.archive_base = ctx.archive_dir
 
 
-def install_except_hook():
-    """
-    Install an exception hook that first logs any uncaught exception, then
-    raises it.
-    """
-    def log_exception(exc_type, exc_value, exc_traceback):
-        if not issubclass(exc_type, KeyboardInterrupt):
-            log.critical("Uncaught exception", exc_info=(exc_type, exc_value,
-                                                         exc_traceback))
-        sys.__excepthook__(exc_type, exc_value, exc_traceback)
-    sys.excepthook = log_exception
-
-
 def main(ctx):
     loglevel = logging.INFO
     if ctx.verbose: