fix(pytest): create explicit readiness_check in MayastorHandle

Attempt to work around gRPC bug by retrying once on failure.
There appears to be a bug in which, under certain conditions,
the python code does not honour the wait_for_ready
flag when it is set in gRPC calls.
Upon such failure we now perform a single retry.

It has been claimed that this is sufficient seem to resolve
the issue. However, it has not been possible to verify this
as we have been unable to reproduce the conditions under
which the bug is triggered. At the very least,
a single retry will not do any harm
and cannot make matters worse.

test/python/common/hdl.py

@@ -18,8 +18,7 @@ def __init__(self, ip_v4):
         self.channel = grpc.insecure_channel(("%s:10124") % self.ip_v4)
         self.bdev = rpc.BdevRpcStub(self.channel)
         self.ms = rpc.MayastorStub(self.channel)
-        self.bdev_list()
-        self.pool_list()
+        self._readiness_check()
 
     def set_timeout(self, timeout):
         self.timeout = timeout
@@ -34,12 +33,21 @@ def install_stub(self, name):
                 setattr(stub, f, partial(h, timeout=self.timeout))
         return stub
 
+    def _readiness_check(self):
+        try:
+            self.bdev_list()
+            self.pool_list()
+        except grpc._channel._InactiveRpcError:
+            # This is to get around a gRPC bug.
+            # Retry once before failing
+            self.bdev_list()
+            self.pool_list()
+
     def reconnect(self):
         self.channel = grpc.insecure_channel(("%s:10124") % self.ip_v4)
         self.bdev = self.install_stub("BdevRpcStub")
         self.ms = self.install_stub("MayastorStub")
-        self.bdev_list()
-        self.pool_list()
+        self._readiness_check()
 
     def __del__(self):
         del self.channel