rgw: start_rgw() polls gateway until it accepts connections

resolves various races between radosgw startup and further operations - both within the rgw task itself (such as the 'radosgw-admin realm pull'), and in later tasks Fixes: http://tracker.ceph.com/issues/17794 Fixes: http://tracker.ceph.com/issues/17872 Signed-off-by: Casey Bodley <[email protected]>
att · Nov 16, 2016 · 5e6538e · 5e6538e
1 parent 69bbafa
commit 5e6538e
Showing 1 changed file with 14 additions and 0 deletions.
diff --git a/tasks/rgw.py b/tasks/rgw.py
@@ -10,6 +10,9 @@
 import errno
 import util.rgw as rgw_utils
 
+from requests.packages.urllib3 import PoolManager
+from requests.packages.urllib3.util import Retry
+
 from cStringIO import StringIO
 
 from teuthology.orchestra import run
@@ -353,6 +356,17 @@ def start_rgw(ctx, config, on_client = None, except_client = None):
             wait=False,
             )
 
+    # XXX: add_daemon() doesn't let us wait until radosgw finishes startup
+    # use a connection pool with retry/backoff to poll each gateway until it starts listening
+    http = PoolManager(retries=Retry(connect=8, backoff_factor=1))
+    for client in clients_to_run:
+        if client == except_client:
+            continue
+        host, port = ctx.rgw.role_endpoints[client]
+        endpoint = 'http://{host}:{port}/'.format(host=host, port=port)
+        log.info('Polling {client} until it starts accepting connections on {endpoint}'.format(client=client, endpoint=endpoint))
+        http.request('GET', endpoint)
+
     try:
         yield
     finally: