Dell PowerFlex: Added retry after disconnect volume

Implemented a retry mechanism post-volume
disconnection to ensure reliability.
Integrated a function execution to validate
the existence of the volume, safeguarding
against connectivity issues arising from the
removal of the old path during a hard reboot.

Closes-Bug: #2034685
Change-Id: Iae1fd01f85bc429c7976e71fb9eeb8dbda795789

os_brick/initiator/connectors/scaleio.py

@@ -531,6 +531,10 @@ def disconnect_volume(self, connection_properties, device_info,
                         'err': response['message']})
                 LOG.error(msg)
                 raise exception.BrickException(message=msg)
+        else:
+            if 'device_path' in connection_properties:
+                path = connection_properties['device_path']
+                self._wait_for_remove_volume_path(path)
 
     @utils.connect_volume_undo_prepare_result
     def extend_volume(self, connection_properties):
@@ -549,3 +553,21 @@ def extend_volume(self, connection_properties):
         msg = (_("Error extending ScaleIO volume"))
         LOG.error(msg)
         raise exception.BrickException(message=msg)
+
+    # NOTE: Usually 5 retries is enough to find the volume.
+    # If there are network issues, it could take much longer. Set
+    # the max retries to 15 to make sure we can find the volume.
+    @utils.retry(exception.BrickException,
+                 retries=15,
+                 backoff_rate=1)
+    def _wait_for_remove_volume_path(self, path):
+        if os.path.exists(path):
+            msg = (_("ScaleIO volume %(volume_id)s found "
+                     "after disconnect operation at path %(path)s") %
+                   {'volume_id': self.volume_id, 'path': path})
+            LOG.debug(msg)
+            raise exception.BrickException(message=msg)
+        else:
+            LOG.info("ScaleIO disconnect volume %(volume_id)s "
+                     "removed at path %(path)s.",
+                     {'volume_id': self.volume_id, 'path': path})

os_brick/tests/initiator/connectors/test_scaleio.py

@@ -357,3 +357,30 @@ def test_connection_properties_without_failed_over(self):
             scaleio.CONNECTOR_CONF_PATH,
             connection_properties['config_group'],
             False)
+
+    @mock.patch('tenacity.wait_exponential')
+    @mock.patch.object(os.path, 'exists', return_value=True)
+    def test_disconnect_volume_wait_for_path_not_removed(self,
+                                                         path_mock,
+                                                         wait_mock):
+        self.fake_connection_properties['device_path'] = ('/dev/'
+                                                          'disk/by-id/'
+                                                          'emc-vol-'
+                                                          '00df72815d3b900f'
+                                                          '-d4f7289200000023')
+        self.assertRaises(exception.BrickException,
+                          self.test_disconnect_volume)
+        wait_mock.assert_called_once_with(multiplier=1, min=0, exp_base=1)
+
+    @mock.patch('tenacity.wait_exponential')
+    @mock.patch.object(os.path, 'exists', return_value=False)
+    def test_disconnect_volume_wait_for_path_removed(self,
+                                                     path_mock,
+                                                     wait_mock):
+        self.fake_connection_properties['device_path'] = ('/dev/'
+                                                          'disk/by-id/'
+                                                          'emc-vol-'
+                                                          '00df72815d3b900f'
+                                                          '-d4f7289200000023')
+        self.test_disconnect_volume()
+        wait_mock.assert_called_once_with(multiplier=1, min=0, exp_base=1)

releasenotes/notes/bug-2034685-dell-powerflex-scaleio-disconnect-volume-c69017cc120306df.yaml

@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    Dell PowerFlex driver `Bug #2034685
+    <https://bugs.launchpad.net/os-brick/+bug/2034685>`_: Added
+    a retry mechanism to check if the disconnected device is
+    actually removed from the host ensuring that subsequent
+    connections succeed.