Make retries and backoff configurable (#1323)

* Make retries and backoff configurable

* Write changelog

Author: Tobias314

webknossos/Changelog.md

@@ -18,6 +18,7 @@ For upgrade instructions, please check the respective _Breaking Changes_ section
 - Added `layers_to_ignore` argument to `Dataset.copy_dataset`. [#1321](https://github.com/scalableminds/webknossos-libs/pull/1321)
 
 ### Changed
+- Make number of retries and backoff factor configurable (mainly for tensorstore reads/writes). See `DEFAULT_NUM_RETRIES` and `DEFAULT_BACKOFF_FACTOR` environment variables. [#1323](https://github.com/scalableminds/webknossos-libs/pull/1323)
 
 ### Fixed
 

webknossos/webknossos/dataset/_array.py

@@ -1,7 +1,6 @@
 import re
-import time
 from abc import ABC, abstractmethod
-from collections.abc import Callable, Iterable, Iterator
+from collections.abc import Iterable, Iterator
 from dataclasses import dataclass
 from functools import lru_cache
 from logging import getLogger
@@ -11,7 +10,6 @@
 from typing import (
     Any,
     Literal,
-    TypeVar,
 )
 from urllib.parse import urlparse
 
@@ -22,15 +20,13 @@
 from upath import UPath
 
 from ..geometry import BoundingBox, NDBoundingBox, Vec3Int, VecInt
-from ..utils import is_fs_path
+from ..utils import call_with_retries, is_fs_path
 from .data_format import DataFormat
 
 logger = getLogger(__name__)
 
 TS_CONTEXT = tensorstore.Context()
 
-DEFAULT_NUM_RETRIES = 5
-
 
 def _is_power_of_two(num: int) -> bool:
     return num & (num - 1) == 0
@@ -40,35 +36,6 @@ class ArrayException(Exception):
     pass
 
 
-ReturnType = TypeVar("ReturnType")
-
-
-def call_with_retries(
-    fn: Callable[[], ReturnType],
-    num_retries: int = DEFAULT_NUM_RETRIES,
-    description: str = "",
-) -> ReturnType:
-    """Call a function, retrying up to `num_retries` times on an exception during the call. Useful for retrying requests or network io."""
-    last_exception = None
-    for i in range(num_retries):
-        try:
-            return fn()
-        except Exception as e:  # noqa: PERF203 # allow try except in loop
-            logger.warning(
-                f"{description} attempt {i + 1}/{num_retries} failed, retrying..."
-                f"Error was: {e}"
-            )
-            # We introduce some randomness to avoid multiple processes retrying at the same time
-            random_factor = np.random.uniform(0.5, 1.5)
-            time.sleep(1 * (1.5**i) * random_factor)
-            last_exception = e
-    # If the last attempt fails, we log the error and raise it.
-    # This is important to avoid silent failures.
-    logger.error(f"{description} failed after {num_retries} attempts.")
-    assert last_exception is not None, "last_exception should never be None here"
-    raise last_exception
-
-
 @dataclass
 class ArrayInfo:
     data_format: DataFormat

webknossos/webknossos/dataset/defaults.py

@@ -59,3 +59,14 @@ def _rebuild_sslcontext(protocol: ssl._SSLMethod, cafile: str | None) -> ssl.SSL
 ZATTRS_FILE_NAME = ".zattrs"
 ZARR_JSON_FILE_NAME = "zarr.json"
 SSL_CONTEXT = _create_sslcontext()
+
+DEFAULT_NUM_RETRIES = (
+    int(os.environ["DEFAULT_NUM_RETRIES"])
+    if "DEFAULT_NUM_RETRIES" in os.environ
+    else 20
+)
+DEFAULT_BACKOFF_FACTOR = (
+    float(os.environ["DEFAULT_BACKOFF_FACTOR"])
+    if "DEFAULT_BACKOFF_FACTOR" in os.environ
+    else 1.75
+)

webknossos/webknossos/utils.py

@@ -31,8 +31,41 @@
 from rich.progress import Progress
 from upath import UPath
 
+from .dataset.defaults import DEFAULT_BACKOFF_FACTOR, DEFAULT_NUM_RETRIES
+
+logger = logging.getLogger(__name__)
+
 times = {}
 
+ReturnType = TypeVar("ReturnType")
+
+
+def call_with_retries(
+    fn: Callable[[], ReturnType],
+    num_retries: int = DEFAULT_NUM_RETRIES,
+    description: str = "",
+    backoff_factor: float = DEFAULT_BACKOFF_FACTOR,
+) -> ReturnType:
+    """Call a function, retrying up to `num_retries` times on an exception during the call. Useful for retrying requests or network io."""
+    last_exception = None
+    for i in range(num_retries):
+        try:
+            return fn()
+        except Exception as e:  # noqa: PERF203 # allow try except in loop
+            logger.warning(
+                f"{description} attempt {i + 1}/{num_retries} failed, retrying..."
+                f"Error was: {e}"
+            )
+            # We introduce some randomness to avoid multiple processes retrying at the same time
+            random_factor = np.random.uniform(0.66, 1.5)
+            time.sleep((backoff_factor**i) * random_factor)
+            last_exception = e
+    # If the last attempt fails, we log the error and raise it.
+    # This is important to avoid silent failures.
+    logger.error(f"{description} failed after {num_retries} attempts.")
+    assert last_exception is not None, "last_exception should never be None here"
+    raise last_exception
+
 
 def time_start(identifier: str) -> None:
     times[identifier] = time.time()