Merge pull request #919 from fractal-analytics-platform/918_apply_registration_race_condition

Fix race condition handling in apply registration task

Author: jluethi

CHANGELOG.md

@@ -1,7 +1,9 @@
 **Note**: Numbers like (\#123) point to closed Pull Requests on the fractal-tasks-core repository.
 
-# Unreleased
+# 1.4.3
 
+* Tasks:
+    * Update apply_registration_to_image to handle race conditions better (\#919).
 * Testing
     * Add `spherical-harmonics` extra to scmultiplex, for testing manifest build (\#899).
     * Move manifest-creation CI to Python 3.11 (\#915).

fractal_tasks_core/tasks/apply_registration_to_image.py

@@ -205,23 +205,34 @@ def apply_registration_to_image(
             # Get the relevant metadata of the Zarr table & add it
             # See issue #516 for the need for this workaround
             max_retries = 20
-            sleep_time = 5
+            sleep_time = 10
             current_round = 0
             while current_round < max_retries:
                 try:
                     old_table_group = zarr.open_group(
                         table_dict[table], mode="r"
                     )
                     current_round = max_retries
-                except zarr.errors.GroupNotFoundError:
+                    curr_table = ad.read_zarr(table_dict[table])
+                    break  # Exit loop on success
+                except (
+                    zarr.errors.GroupNotFoundError,
+                    zarr.errors.PathNotFoundError,
+                ):
                     logger.debug(
                         f"Table {table} not found in attempt {current_round}. "
                         f"Waiting {sleep_time} seconds before trying again."
                     )
                     current_round += 1
                     time.sleep(sleep_time)
+            else:
+                # This runs only if the loop exits via exhaustion
+                raise RuntimeError(
+                    f"Table {table} not found after {max_retries} attempts."
+                    "Check whether this table actually exists. If it does, "
+                    "this may be a race condition issue."
+                )
             # Write the Zarr table
-            curr_table = ad.read_zarr(table_dict[table])
             write_table(
                 new_image_group,
                 table,