Improve NetBox sync lock error handling to prevent blocking (#1852)

This change improves error handling when NetBox sync operations fail to
acquire locks. Previously, lock acquisition timeouts would block the entire
sync and continue without proper tracking. Now:

1. NetBox functions (set_maintenance, set_provision_state, set_power_state)
   return True/False based on lock acquisition success
2. sync_netbox_from_ironic tracks failed devices and reports them at the end
3. Sync continues with remaining devices when individual devices fail
4. Failed devices are reported with a clear warning message

This prevents silent failures and provides better visibility into which
devices failed to sync due to lock timeouts, while allowing the sync
operation to continue for other devices.

AI-assisted: Claude Code

Signed-off-by: Christian Berendt <[email protected]>

Author: berendt

osism/tasks/conductor/ironic.py

@@ -605,6 +605,7 @@ def sync_netbox_from_ironic(request_id, node_name=None, netbox_filter=None):
             return
 
     # Sync each node to NetBox
+    failed_devices = []
     for node in nodes:
         # Adjust message based on whether secondaries are actually being synced
         if reachable_secondaries:
@@ -616,26 +617,44 @@ def sync_netbox_from_ironic(request_id, node_name=None, netbox_filter=None):
 
         osism_utils.push_task_output(request_id, sync_msg)
 
+        # Track if this device failed to sync
+        device_failed = False
+
         # Update all three states (each function handles primary + secondary NetBox instances)
         # Pass netbox_filter to only update matching NetBox instances
         # Pass reachable_secondaries to only use reachable secondary instances
-        netbox.set_provision_state(
+        if not netbox.set_provision_state(
             node["name"],
             node["provision_state"],
             netbox_filter=netbox_filter,
             secondary_nb_list=reachable_secondaries,
-        )
-        netbox.set_power_state(
+        ):
+            device_failed = True
+
+        if not netbox.set_power_state(
             node["name"],
             node["power_state"],
             netbox_filter=netbox_filter,
             secondary_nb_list=reachable_secondaries,
-        )
-        netbox.set_maintenance(
+        ):
+            device_failed = True
+
+        if not netbox.set_maintenance(
             node["name"],
             state=node["is_maintenance"],
             netbox_filter=netbox_filter,
             secondary_nb_list=reachable_secondaries,
+        ):
+            device_failed = True
+
+        if device_failed:
+            failed_devices.append(node["name"])
+
+    # Report failed devices if any
+    if failed_devices:
+        osism_utils.push_task_output(
+            request_id,
+            f"WARNING: Failed to sync {len(failed_devices)} device(s) due to lock timeout: {', '.join(failed_devices)}\n",
         )
 
     osism_utils.finish_task_output(request_id, rc=0)

osism/tasks/netbox.py

@@ -127,6 +127,9 @@ def set_maintenance(
                       Use 'primary' to match the primary NetBox instance.
         secondary_nb_list: Optional list of secondary NetBox instances to use.
                           If not provided, uses utils.secondary_nb_list.
+
+    Returns:
+        bool: True if lock was acquired and operation succeeded, False if lock could not be acquired.
     """
     # Check if tasks are locked before execution
     utils.check_task_lock_and_exit()
@@ -177,8 +180,10 @@ def set_maintenance(
                     )
         finally:
             lock.release()
+        return True
     else:
         logger.error(f"Could not acquire lock for node {device_name}")
+        return False
 
 
 @app.task(bind=True, name="osism.tasks.netbox.set_provision_state")
@@ -195,6 +200,9 @@ def set_provision_state(
                       Use 'primary' to match the primary NetBox instance.
         secondary_nb_list: Optional list of secondary NetBox instances to use.
                           If not provided, uses utils.secondary_nb_list.
+
+    Returns:
+        bool: True if lock was acquired and operation succeeded, False if lock could not be acquired.
     """
     # Check if tasks are locked before execution
     utils.check_task_lock_and_exit()
@@ -245,8 +253,10 @@ def set_provision_state(
                     )
         finally:
             lock.release()
+        return True
     else:
         logger.error(f"Could not acquire lock for node {device_name}")
+        return False
 
 
 @app.task(bind=True, name="osism.tasks.netbox.set_power_state")
@@ -263,6 +273,9 @@ def set_power_state(
                       Use 'primary' to match the primary NetBox instance.
         secondary_nb_list: Optional list of secondary NetBox instances to use.
                           If not provided, uses utils.secondary_nb_list.
+
+    Returns:
+        bool: True if lock was acquired and operation succeeded, False if lock could not be acquired.
     """
     # Convert None to "n/a" for clearer user feedback
     if state is None:
@@ -317,8 +330,10 @@ def set_power_state(
                     )
         finally:
             lock.release()
+        return True
     else:
         logger.error(f"Could not acquire lock for node {device_name}")
+        return False
 
 
 @app.task(bind=True, name="osism.tasks.netbox.get_location_id")