qa/tasks: introduce Thrasher base class

* Introduced a Thrasher base class. * Updated thrashers to inherit from Thrasher. * Replaced the magic variable e with Thrasher.exception as per the discussion. Now the exception variable sets by default as the thrashers are inheriting from the Thrasher class. Fixes: ceph#28378 (comment) Fixes: https://tracker.ceph.com/issues/41133 Signed-off-by: Jos Collin <[email protected]>
atta · Aug 21, 2019 · f31791e · f31791e
1 parent 5a29627
commit f31791e
Show file tree

Hide file tree

Showing 7 changed files with 53 additions and 18 deletions.
diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py
@@ -22,6 +22,7 @@
 from teuthology.orchestra.remote import Remote
 from teuthology.orchestra import run
 from teuthology.exceptions import CommandFailedError
+from tasks.thrasher import Thrasher
 
 try:
     from subprocess import DEVNULL # py3k
@@ -100,11 +101,12 @@ class PoolType:
     ERASURE_CODED = 3
 
 
-class Thrasher:
+class OSDThrasher(Thrasher):
     """
     Object used to thrash Ceph
     """
     def __init__(self, manager, config, logger):
+        super(OSDThrasher, self).__init__()
         self.ceph_manager = manager
         self.cluster = manager.cluster
         self.ceph_manager.wait_for_clean()
@@ -955,6 +957,19 @@ def choose_action(self):
             val -= prob
         return None
 
+    def do_thrash(self):
+        """
+        _do_thrash() wrapper.
+        """
+        try:
+            self._do_thrash()
+        except Exception as e:
+            # See _run exception comment for MDSThrasher
+            self.exception = e
+            self.logger.exception("exception:")
+            # Allow successful completion so gevent doesn't see an exception.
+            # The DaemonWatchdog will observe the error and tear down the test.
+
     def log_exc(func):
         @wraps(func)
         def wrapper(self):
@@ -1047,7 +1062,7 @@ def do_noscrub_toggle(self):
         self.ceph_manager.raw_cluster_cmd('osd', 'unset', 'nodeep-scrub')
 
     @log_exc
-    def do_thrash(self):
+    def _do_thrash(self):
         """
         Loop to select random actions to thrash ceph manager with.
         """

diff --git a/qa/tasks/daemonwatchdog.py b/qa/tasks/daemonwatchdog.py
@@ -23,7 +23,7 @@ class DaemonWatchdog(Greenlet):
     """
 
     def __init__(self, ctx, config, thrashers):
-        Greenlet.__init__(self)
+        super(DaemonWatchdog, self).__init__()
         self.ctx = ctx
         self.config = config
         self.e = None
@@ -106,7 +106,7 @@ def watch(self):
                     del daemon_failure_time[name]
 
             for thrasher in self.thrashers:
-                if thrasher.e is not None:
+                if thrasher.exception is not None:
                     self.log("thrasher on fs.{name} failed".format(name=thrasher.fs.name))
                     bark = True
 

diff --git a/qa/tasks/mds_thrash.py b/qa/tasks/mds_thrash.py
@@ -14,10 +14,11 @@
 from teuthology import misc as teuthology
 
 from tasks.cephfs.filesystem import MDSCluster, Filesystem
+from tasks.thrasher import Thrasher
 
 log = logging.getLogger(__name__)
 
-class MDSThrasher(Greenlet):
+class MDSThrasher(Greenlet, Thrasher, object):
     """
     MDSThrasher::
 
@@ -97,11 +98,10 @@ class MDSThrasher(Greenlet):
     """
 
     def __init__(self, ctx, manager, config, fs, max_mds):
-        Greenlet.__init__(self)
+        super(MDSThrasher, self).__init__()
 
         self.config = config
         self.ctx = ctx
-        self.e = None
         self.logger = log.getChild('fs.[{f}]'.format(f = fs.name))
         self.fs = fs
         self.manager = manager
@@ -136,7 +136,7 @@ def _run(self):
             #   File "/usr/lib/python2.7/traceback.py", line 13, in _print
             #     file.write(str+terminator)
             # 2017-02-03T14:34:01.261 CRITICAL:root:IOError
-            self.e = e
+            self.exception = e
             self.logger.exception("exception:")
             # allow successful completion so gevent doesn't see an exception...
 
@@ -425,7 +425,7 @@ def task(ctx, config):
     finally:
         log.info('joining mds_thrasher')
         thrasher.stop()
-        if thrasher.e:
+        if thrasher.exception is not None:
             raise RuntimeError('error during thrashing')
         thrasher.join()
         log.info('done joining')
diff --git a/qa/tasks/mon_thrash.py b/qa/tasks/mon_thrash.py
@@ -11,6 +11,7 @@
 import math
 from teuthology import misc as teuthology
 from tasks.cephfs.filesystem import MDSCluster
+from tasks.thrasher import Thrasher
 
 log = logging.getLogger(__name__)
 
@@ -21,7 +22,7 @@ def _get_mons(ctx):
     mons = [f[len('mon.'):] for f in teuthology.get_mon_names(ctx)]
     return mons
 
-class MonitorThrasher:
+class MonitorThrasher(Thrasher):
     """
     How it works::
 
@@ -85,11 +86,11 @@ class MonitorThrasher:
             - mon/workloadgen.sh
     """
     def __init__(self, ctx, manager, config, logger):
+        super(MonitorThrasher, self).__init__()
         self.ctx = ctx
         self.manager = manager
         self.manager.wait_for_clean()
 
-        self.e = None
         self.stopping = False
         self.logger = logger
         self.config = config
@@ -228,7 +229,7 @@ def do_thrash(self):
             self._do_thrash()
         except Exception as e:
             # See _run exception comment for MDSThrasher
-            self.e = e
+            self.exception = e
             self.logger.exception("exception:")
             # Allow successful completion so gevent doesn't see an exception.
             # The DaemonWatchdog will observe the error and tear down the test.

diff --git a/qa/tasks/rbd_mirror_thrash.py b/qa/tasks/rbd_mirror_thrash.py
@@ -17,11 +17,12 @@
 from teuthology.exceptions import CommandFailedError
 from teuthology.task import Task
 from teuthology.orchestra import run
+from tasks.thrasher import Thrasher
 
 log = logging.getLogger(__name__)
 
 
-class RBDMirrorThrasher(Greenlet):
+class RBDMirrorThrasher(Greenlet, Thrasher, object):
     """
     RBDMirrorThrasher::
 
@@ -63,14 +64,13 @@ class RBDMirrorThrasher(Greenlet):
     """
 
     def __init__(self, ctx, config, cluster, daemons):
-        Greenlet.__init__(self)
+        super(RBDMirrorThrasher, self).__init__()
 
         self.ctx = ctx
         self.config = config
         self.cluster = cluster
         self.daemons = daemons
 
-        self.e = None
         self.logger = log
         self.name = 'thrasher.rbd_mirror.[{cluster}]'.format(cluster = cluster)
         self.stopping = Event()
@@ -85,8 +85,11 @@ def _run(self):
         try:
             self.do_thrash()
         except Exception as e:
-            self.e = e
+            # See _run exception comment for MDSThrasher
+            self.exception = e
             self.logger.exception("exception:")
+            # Allow successful completion so gevent doesn't see an exception.
+            # The DaemonWatchdog will observe the error and tear down the test.
 
     def log(self, x):
         """Write data to logger assigned to this RBDMirrorThrasher"""
@@ -211,7 +214,7 @@ def task(ctx, config):
     finally:
         log.info('joining rbd_mirror_thrash')
         thrasher.stop()
-        if thrasher.e:
+        if thrasher.exception is not None:
             raise RuntimeError('error during thrashing')
         thrasher.join()
         log.info('done joining')
diff --git a/qa/tasks/thrasher.py b/qa/tasks/thrasher.py
@@ -0,0 +1,16 @@
+"""
+Thrasher base class
+"""
+class Thrasher(object):
+
+    def __init__(self):
+        super(Thrasher, self).__init__()
+        self.exception = None
+
+    @property
+    def exception(self):
+        return self._exception
+
+    @exception.setter
+    def exception(self, e):
+        self._exception = e
diff --git a/qa/tasks/thrashosds.py b/qa/tasks/thrashosds.py
@@ -200,7 +200,7 @@ def task(ctx, config):
             'true')
 
     log.info('Beginning thrashosds...')
-    thrash_proc = ceph_manager.Thrasher(
+    thrash_proc = ceph_manager.OSDThrasher(
         cluster_manager,
         config,
         logger=log.getChild('thrasher')