tests: Fix NVMe device timing issues with polling mechanism

Fixes #1080

Replace immediate device lookup with intelligent polling to handle
timing issues where NVMe namespace devices may not be immediately
available after controller creation.

- Add exponential backoff polling (100ms-500ms, 5s timeout)
- Include udevadm settle for device readiness
- Maintain backward compatibility
- More reliable than simple sleep() approach

Author: ermezavx

tests/utils.py

@@ -274,11 +274,12 @@ def clean_scsi_debug(scsi_debug_dev):
     except:
         pass
 
-def find_nvme_ctrl_devs_for_subnqn(subnqn):
+def find_nvme_ctrl_devs_for_subnqn(subnqn, timeout=5):
     """
     Find NVMe controller devices for the specified subsystem nqn
 
     :param str subnqn: subsystem nqn
+    :param int timeout: timeout in seconds to wait for devices to appear
     """
 
     def _check_subsys(subsys, dev_paths):
@@ -295,33 +296,54 @@ def _check_subsys(subsys, dev_paths):
                 except:
                     pass
 
-    ret, out, err = run_command("nvme list --output-format=json --verbose")
-    if ret != 0:
-        raise RuntimeError("Error getting NVMe list: '%s %s'" % (out, err))
+    def _find_devices():
+        ret, out, err = run_command("nvme list --output-format=json --verbose")
+        if ret != 0:
+            return []
+
+        try:
+            decoder = json.JSONDecoder()
+            decoded = decoder.decode(out)
+            if not decoded or 'Devices' not in decoded:
+                return []
+
+            dev_paths = []
+            for dev in decoded['Devices']:
+                # nvme-cli 2.x
+                if 'Subsystems' in dev:
+                    for subsys in dev['Subsystems']:
+                        _check_subsys(subsys, dev_paths)
+                # nvme-cli 1.x
+                if 'SubsystemNQN' in dev:
+                    _check_subsys(dev, dev_paths)
+
+            return dev_paths
+        except:
+            return []
 
-    decoder = json.JSONDecoder()
-    decoded = decoder.decode(out)
-    if not decoded or 'Devices' not in decoded:
-        return []
+    # Poll for devices with exponential backoff
+    start_time = time.time()
+    wait_time = 0.1  # Start with 100ms
+    max_wait = 0.5   # Cap at 500ms
 
-    dev_paths = []
-    for dev in decoded['Devices']:
-        # nvme-cli 2.x
-        if 'Subsystems' in dev:
-            for subsys in dev['Subsystems']:
-                _check_subsys(subsys, dev_paths)
-        # nvme-cli 1.x
-        if 'SubsystemNQN' in dev:
-            _check_subsys(dev, dev_paths)
+    while time.time() - start_time < timeout:
+        dev_paths = _find_devices()
+        if dev_paths:
+            return dev_paths
+        
+        time.sleep(wait_time)
+        wait_time = min(wait_time * 1.2, max_wait)  # Slower exponential backoff
+        os.system("udevadm settle")  # Ensure udev has processed any pending events
 
-    return dev_paths
+    return _find_devices()
 
 
-def find_nvme_ns_devs_for_subnqn(subnqn):
+def find_nvme_ns_devs_for_subnqn(subnqn, timeout=5):
     """
     Find NVMe namespace block devices for the specified subsystem nqn
 
     :param str subnqn: subsystem nqn
+    :param int timeout: timeout in seconds to wait for devices to appear
     """
 
     def _check_namespaces(node, ns_dev_paths):
@@ -344,26 +366,46 @@ def _check_subsys(subsys, ns_dev_paths):
                     if 'Namespaces' in ctrl:
                         _check_namespaces(ctrl, ns_dev_paths)
 
-    ret, out, err = run_command("nvme list --output-format=json --verbose")
-    if ret != 0:
-        raise RuntimeError("Error getting NVMe list: '%s %s'" % (out, err))
-
-    decoder = json.JSONDecoder()
-    decoded = decoder.decode(out)
-    if not decoded or 'Devices' not in decoded:
-        return []
-
-    ns_dev_paths = []
-    for dev in decoded['Devices']:
-        # nvme-cli 2.x
-        if 'Subsystems' in dev:
-            for subsys in dev['Subsystems']:
-                _check_subsys(subsys, ns_dev_paths)
-        # nvme-cli 1.x
-        if 'SubsystemNQN' in dev:
-            _check_subsys(dev, ns_dev_paths)
-
-    return ns_dev_paths
+    def _find_devices():
+        ret, out, err = run_command("nvme list --output-format=json --verbose")
+        if ret != 0:
+            return []
+
+        try:
+            decoder = json.JSONDecoder()
+            decoded = decoder.decode(out)
+            if not decoded or 'Devices' not in decoded:
+                return []
+
+            ns_dev_paths = []
+            for dev in decoded['Devices']:
+                # nvme-cli 2.x
+                if 'Subsystems' in dev:
+                    for subsys in dev['Subsystems']:
+                        _check_subsys(subsys, ns_dev_paths)
+                # nvme-cli 1.x
+                if 'SubsystemNQN' in dev:
+                    _check_subsys(dev, ns_dev_paths)
+
+            return ns_dev_paths
+        except:
+            return []
+
+    # Poll for devices with exponential backoff
+    start_time = time.time()
+    wait_time = 0.1  # Start with 100ms
+    max_wait = 0.5   # Cap at 500ms
+
+    while time.time() - start_time < timeout:
+        ns_dev_paths = _find_devices()
+        if ns_dev_paths:  # Return as soon as we find any namespace devices
+            return ns_dev_paths
+        
+        time.sleep(wait_time)
+        wait_time = min(wait_time * 1.2, max_wait)  # Slower exponential backoff
+        os.system("udevadm settle")  # Ensure udev has processed any pending events
+
+    return _find_devices()
 
 
 def get_nvme_hostnqn():