fix(cinder-understack): refactor init error cases to start cleanly

If any error occurred during startup the driver would just enter a
failure loop because there would already be another instance for a prior
SVM and things could never recover. This refactors the error paths to
clean up and ensure things can startup successfully.

Author: cardoe

python/cinder-understack/cinder_understack/dynamic_netapp_driver.py

@@ -233,10 +233,9 @@ def do_setup(self, ctxt):
             svm_lib.do_setup(ctxt)
             self._libraries[svm_name] = svm_lib
 
-    def _remove_svm_lib(self, svm_name: str, svm_lib: NetAppMinimalLibrary):
+    def _remove_svm_lib(self, svm_lib: NetAppMinimalLibrary):
         """Remove resources for a given SVM library."""
         # TODO: Need to free up resources here.
-        LOG.info("Removing resources for SVM library %s", svm_name)
         for task in svm_lib.loopingcalls.tasks:
             task.looping_call.stop()
         svm_lib.loopingcalls.tasks = []
@@ -257,17 +256,16 @@ def _actual_refresh_svm_libraries(self, ctxt):
         stale_svms = existing_libs - current_svms
         for svm_name in stale_svms:
             LOG.info("Removing stale NVMe library for SVM: %s", svm_name)
-            # TODO : stop looping calls, free resources.
-            svm_lib = self._libraries.get(svm_name)
-            self._remove_svm_lib(svm_name, svm_lib)
+            svm_lib = self._libraries[svm_name]
+            self._remove_svm_lib(svm_lib)
             del self._libraries[svm_name]
 
         # Add new SVM libraries
         new_svms = current_svms - existing_libs
         for svm_name in new_svms:
             LOG.info("Creating NVMe library for new SVM: %s", svm_name)
+            lib = self._create_svm_lib(svm_name)
             try:
-                lib = self._create_svm_lib(svm_name)
                 # Call do_setup to initialize the library
                 lib.do_setup(ctxt)
                 lib.check_for_setup_error()
@@ -278,13 +276,21 @@ def _actual_refresh_svm_libraries(self, ctxt):
                     "Failed to create library for SVM %s",
                     svm_name,
                 )
+                self._remove_svm_lib(lib)
         LOG.info("Final libraries loaded: %s", list(self._libraries.keys()))
 
     def check_for_setup_error(self):
         """Check for setup errors."""
-        for svm_name, svm_lib in self._libraries.items():
+        svm_to_init = set(self._libraries.keys())
+        for svm_name in svm_to_init:
             LOG.info("Checking NVMe library for errors for SVM %s", svm_name)
-            svm_lib.check_for_setup_error()
+            svm_lib = self._libraries[svm_name]
+            try:
+                svm_lib.check_for_setup_error()
+            except Exception:
+                LOG.exception("Failed to initialize SVM %s, skipping", svm_name)
+                self._remove_svm_lib(svm_lib)
+                del self._libraries[svm_name]
 
         # looping call to refresh SVM libraries
         if not self._looping_call:

python/cinder-understack/cinder_understack/tests/test_dynamic_netapp_driver.py

@@ -201,8 +201,11 @@ def test_refresh_svm_libraries_handles_lib_creation_failure(
         self, mock_get_svms, mock_create_svm_lib
     ):
         """Ensure that failure in lib creation is caught and logged, not raised."""
-        mock_get_svms.return_value = ["os-new-failing-svm"]
-        mock_create_svm_lib.side_effect = Exception("Simulated failure")
+        test_svm_name = "os-new-failing_svm"
+        mock_get_svms.return_value = [test_svm_name]
+        mock_svm_lib = _create_mock_svm_lib(test_svm_name)
+        mock_svm_lib.check_for_setup_error.side_effect = Exception("Simulated failure")
+        mock_create_svm_lib.return_value = mock_svm_lib
 
         self.driver._libraries = {}