misc: Reimplement host key scanning

We're seeing very intermittent issues with ssh-keyscan; sometimes given
N hostnames, it only returns N-1 keys. Lack of an error message adds to
the confusion. The solution is to call ssh-keyscan once for each host
instead of batching them together - with a few retries - so that we can
easily ensure we get the right amount. If we do not, raise a
RuntimeError.

Signed-off-by: Zack Cerza <[email protected]>
(cherry picked from commit daa28ae)

teuthology/misc.py

@@ -1132,28 +1132,50 @@ def get_valgrind_args(testdir, name, preamble, v):
 def ssh_keyscan(hostnames):
     """
     Fetch the SSH public key of one or more hosts
+
+    :param hostnames: A list of hostnames, or a dict keyed by hostname
+    :returns: A dict keyed by hostname, with the host keys as values
     """
     if isinstance(hostnames, basestring):
         raise TypeError("'hostnames' must be a list")
     hostnames = [canonicalize_hostname(name, user=None) for name in
                  hostnames]
-    args = ['ssh-keyscan', '-T', '1', '-t', 'rsa'] + hostnames
+    keys_dict = dict()
+    for hostname in hostnames:
+        with safe_while(
+                sleep=1, tries=5, action="ssh_keyscan " + hostname) as proceed:
+            while proceed():
+                key = _ssh_keyscan(hostname)
+                if key:
+                    keys_dict[hostname] = key
+                    break
+    if len(keys_dict) != len(hostnames):
+        missing = set(hostnames) - set(keys_dict.keys())
+        raise RuntimeError("Unable to scan these host keys: %s" % missing)
+    return keys_dict
+
+
+def _ssh_keyscan(hostname):
+    """
+    Fetch the SSH public key of one or more hosts
+
+    :param hostname: The hostname
+    :returns: The host key
+    """
+    args = ['ssh-keyscan', '-T', '1', '-t', 'rsa', hostname]
     p = subprocess.Popen(
         args=args,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
     )
     p.wait()
-
-    keys_dict = dict()
     for line in p.stderr.readlines():
         line = line.strip()
         if line and not line.startswith('#'):
             log.error(line)
     for line in p.stdout.readlines():
         host, key = line.strip().split(' ', 1)
-        keys_dict[host] = key
-    return keys_dict
+        return key
 
 
 def ssh_keyscan_wait(hostname):