Merge branch 'master' into wip-ethdev

requirements.txt

@@ -9,6 +9,8 @@ apache-libcloud==1.5.0
 appdirs==1.4.3            # via os-client-config
 argparse==1.4.0
 asn1crypto==0.22.0        # via cryptography
+atomicwrites==1.1.5       # via pytest
+attrs==18.1.0             # via pytest
 babel==2.4.0              # via osc-lib, oslo.i18n, python-cinderclient, python-glanceclient, python-neutronclient, python-novaclient, python-openstackclient
 backports.ssl-match-hostname==3.5.0.1
 beanstalkc==0.4.0
@@ -27,7 +29,7 @@ deprecation==1.0          # via openstacksdk
 docopt==0.6.2
 enum34==1.1.6             # via cryptography
 first==2.0.1              # via pip-tools
-funcsigs==1.0.2           # via debtcollector, oslo.utils
+funcsigs==1.0.2           # via debtcollector, oslo.utils, pytest
 functools32==3.2.3.post2  # via jsonschema
 gevent==1.2.1
 greenlet==0.4.12          # via gevent
@@ -40,10 +42,10 @@ jsonpatch==1.15           # via warlock
 jsonpointer==1.10         # via jsonpatch
 jsonschema==2.6.0         # via warlock
 keystoneauth1==2.19.0     # via openstacksdk, os-client-config, osc-lib, python-cinderclient, python-glanceclient, python-keystoneclient, python-neutronclient, python-novaclient, python-openstackclient
-libvirt-python
 manhole==1.3.0
 markupsafe==1.0           # via jinja2
 monotonic==1.3            # via oslo.utils
+more-itertools==4.3.0     # via pytest
 msgpack-python==0.4.8     # via oslo.serialization
 mysql-python==1.2.3
 ndg-httpsclient==0.4.2
@@ -58,19 +60,21 @@ oslo.i18n==3.15.0         # via osc-lib, oslo.config, oslo.utils, python-cinderc
 oslo.serialization==2.18.0  # via python-keystoneclient, python-neutronclient, python-novaclient
 oslo.utils==3.25.0        # via osc-lib, oslo.serialization, python-cinderclient, python-glanceclient, python-keystoneclient, python-neutronclient, python-novaclient, python-openstackclient
 paramiko==2.1.2
+pathlib2==2.3.2           # via pytest
 pbr==2.0.0                # via cliff, debtcollector, keystoneauth1, openstacksdk, osc-lib, oslo.i18n, oslo.serialization, oslo.utils, positional, python-cinderclient, python-glanceclient, python-keystoneclient, python-neutronclient, python-novaclient, python-openstackclient, requestsexceptions, stevedore
 pexpect==4.2.1
 pip-tools==1.10.1
-pluggy==0.6.0             # via tox
+pluggy==0.7.1             # via pytest, tox
 positional==1.1.1         # via keystoneauth1, python-keystoneclient
 prettytable==0.7.2
 psutil==5.2.2
 ptyprocess==0.5.1         # via pexpect
-py==1.5.3                 # via tox
+py==1.5.3                 # via pytest, tox
 pyasn1==0.2.3
 pycparser==2.17           # via cffi
 pyopenssl==16.2.0
 pyparsing==2.2.0          # via cliff, cmd2, oslo.utils
+pytest==3.7.1
 python-cinderclient==2.0.1  # via python-openstackclient
 python-dateutil==2.6.0
 python-glanceclient==2.6.0  # via python-openstackclient
@@ -84,6 +88,7 @@ raven==6.0.0
 requests==2.12.5
 requestsexceptions==1.2.0  # via os-client-config
 rfc3986==0.4.1            # via oslo.config
+scandir==1.8              # via pathlib2
 simplejson==3.10.0        # via osc-lib, python-cinderclient, python-neutronclient, python-novaclient
 six==1.10.0
 stevedore==1.21.0         # via cliff, keystoneauth1, openstacksdk, osc-lib, oslo.config, python-keystoneclient

scripts/results.py

@@ -1,5 +1,5 @@
 """
-usage: teuthology-results [-h] [-v] [--dry-run] [--email EMAIL] [--timeout TIMEOUT] --archive-dir DIR --name NAME
+usage: teuthology-results [-h] [-v] [--dry-run] [--email EMAIL] [--timeout TIMEOUT] --archive-dir DIR --name NAME [--subset SUBSET] [--seed SEED]
 
 Email teuthology suite results
 
@@ -12,6 +12,8 @@
                      [default: 0]
   --archive-dir DIR  path under which results for the suite are stored
   --name NAME        name of the suite
+  --subset SUBSET    subset passed to teuthology-suite
+  --seed SEED        random seed used in teuthology-suite
 """
 import docopt
 import teuthology.results

scripts/schedule.py

@@ -26,6 +26,9 @@
   -N <num>, --num <num>                Number of times to run/queue the job
                                        [default: 1]
 
+  --first-in-suite                     Mark the first job in a suite so suite
+                                       can note down the rerun-related info
+                                       [default: False]
   --last-in-suite                      Mark the last job in a suite so suite
                                        post-processing can be run
                                        [default: False]
@@ -34,6 +37,10 @@
   --timeout <timeout>                  How many seconds to wait for jobs to
                                        finish before emailing results. Only
                                        applies to the last job in a suite.
+  --seed <seed>                        The random seed for rerunning the suite.
+                                       Only applies to the last job in a suite.
+  --subset <subset>                    The subset option passed to teuthology-suite.
+                                       Only applies to the last job in a suite.
   --dry-run                            Instead of scheduling, just output the
                                        job config.
 

scripts/suite.py

@@ -127,7 +127,8 @@
                               [default: fail,dead]
  --seed SEED                  An random number mostly useful when used along
                               with --rerun argument. This number can be found
-                              in the output of teuthology-suite command.
+                              in the output of teuthology-suite command. -1
+                              for a random seed [default: -1].
 
 """.format(
     default_machine_type=config.default_machine_type,

setup.py

@@ -63,6 +63,7 @@
                       'httplib2',
                       'paramiko',
                       'pexpect',
+                      'pytest', # for tox.ini
                       'nose', # for qa/tasks/rgw_multisite_tests.py',
                       'requests != 2.13.0',
                       'raven',

teuthology/ceph.conf.template

@@ -83,17 +83,9 @@
 	mon osd prime pg temp = true
 	mon reweight min bytes per osd = 10
 
-[mds]
-        mds debug scatterstat = true
-        mds verify scatter = true
-        mds debug frag = true
-
 [client]
-        rgw cache enabled = true
+	rgw cache enabled = true
 	rgw enable ops log = true
 	rgw enable usage log = true
 	log file = /var/log/ceph/$cluster-$name.$pid.log
 	admin socket = /var/run/ceph/$cluster-$name.$pid.asok
-
-
-	client mount timeout = 600

teuthology/openstack/__init__.py

@@ -528,7 +528,13 @@ def main(self):
         """
         self.setup_logs()
         set_config_attr(self.args)
-        self.key_filename = self.args.key_filename
+        for keyfile in [self.args.key_filename,
+                        os.environ['HOME'] + '/.ssh/id_rsa',
+                        os.environ['HOME'] + '/.ssh/id_dsa',
+                        os.environ['HOME'] + '/.ssh/id_ecdsa']:
+            if (keyfile and os.path.isfile(keyfile)):
+                self.key_filename = keyfile
+                break
         self.verify_openstack()
         self.setup()
         exit_code = 0

teuthology/orchestra/opsys.py

@@ -1,18 +1,23 @@
-import ast
 import re
 
 DISTRO_CODENAME_MAP = {
     "ubuntu": {
         "18.04": "bionic",
+        "17.10": "artful",
+        "17.04": "zesty",
+        "16.10": "yakkety",
         "16.04": "xenial",
+        "15.10": "wily",
         "15.04": "vivid",
+        "14.10": "utopic",
         "14.04": "trusty",
         "13.10": "saucy",
         "12.04": "precise",
     },
     "debian": {
         "7": "wheezy",
         "8": "jessie",
+        "9": "stretch",
     },
     "rhel": {
         "7": "maipo",
@@ -23,13 +28,25 @@
         "6": "core",
     },
     "fedora": {
+        "28": "28",
+        "27": "27",
+        "26": "26",
         "25": "25",
         "24": "24",
         "23": "23",
         "22": "22",
         "21": "21",
         "20": "heisenbug",
     },
+    "opensuse": {
+        "42.2": "leap",
+        "42.3": "leap",
+    },
+    "opensuse-leap": {
+        "42.2": "leap",
+        "42.3": "leap",
+        "15.0": "leap",
+    },
 }
 
 DEFAULT_OS_VERSION = dict(
@@ -54,7 +71,7 @@ class OS(object):
     __slots__ = ['name', 'version', 'codename', 'package_type']
 
     _deb_distros = ('debian', 'ubuntu')
-    _rpm_distros = ('fedora', 'rhel', 'centos', 'opensuse')
+    _rpm_distros = ('fedora', 'rhel', 'centos', 'opensuse', 'opensuse-leap')
 
     def __init__(self, name=None, version=None, codename=None):
         self.name = name
@@ -78,37 +95,6 @@ def _codename_to_version(name, codename):
             codename,
         ))
 
-    @classmethod
-    def from_python(cls, python_val):
-        """
-        Parse output from platform.linux_distribution() and populate attributes
-
-        Given a tuple or str()'ed tuple like this:
-            ('Ubuntu', '14.04', 'trusty')
-
-        Attributes will be:
-            name = 'ubuntu'
-            version = '14.04'
-            codename = 'trusty'
-        Additionally, we set the package type:
-            package_type = 'deb'
-        """
-        if not isinstance(python_val, tuple):
-            python_val = ast.literal_eval(python_val)
-
-        (name, version, codename) = python_val
-        name = name.lower().replace(' ', '')
-        if name.startswith('redhat'):
-            name = 'rhel'
-        elif name.startswith('centos'):
-            name = 'centos'
-        elif name.startswith('fedora'):
-            name = 'fedora'
-        elif name.startswith('opensuse'):
-            name = 'opensuse'
-        obj = cls(name=name, version=version, codename=codename.lower())
-        return obj
-
     @classmethod
     def from_lsb_release(cls, lsb_release_str):
         """

teuthology/orchestra/remote.py

@@ -391,15 +391,6 @@ def get_tar_stream(self, path, sudo=False):
     @property
     def os(self):
         if not hasattr(self, '_os'):
-            proc = self.run(
-                args=[
-                    'python', '-c',
-                    'import platform; print platform.linux_distribution()'],
-                stdout=StringIO(), stderr=StringIO(), check_status=False)
-            if proc.exitstatus == 0:
-                self._os = OS.from_python(proc.stdout.getvalue().strip())
-                return self._os
-
             proc = self.run(args=['cat', '/etc/os-release'], stdout=StringIO(),
                             stderr=StringIO(), check_status=False)
             if proc.exitstatus == 0:

teuthology/orchestra/run.py

@@ -139,10 +139,16 @@ def wait(self):
 
         :returns: self.returncode
         """
-        for greenlet in self.greenlets:
-            greenlet.get()
 
         status = self._get_exitstatus()
+        if status != 0:
+            log.debug("got remote process result: {}".format(status))
+        for greenlet in self.greenlets:
+            try:
+                greenlet.get(block=True,timeout=60)
+            except gevent.Timeout:
+                log.debug("timed out waiting; will kill: {}".format(greenlet))
+                greenlet.kill(block=False)
         for stream in ('stdout', 'stderr'):
             if hasattr(self, stream):
                 stream_obj = getattr(self, stream)