Increase the timeout of gRPC calls dynamically on none-successful attempts

Signed-off-by: camille-bouvy-frequenz <[email protected]>

Author: camille-bouvy-frequenz

src/frequenz/client/electricity_trading/_client.py

@@ -89,28 +89,54 @@ def validate_decimal_places(value: Decimal, decimal_places: int, name: str) -> N
 
 
 async def grpc_call_with_timeout(
-    call: Callable[..., Awaitable[Any]], *args: Any, timeout: float = 300, **kwargs: Any
+    call: Callable[..., Awaitable[Any]],
+    *args: Any,
+    initial_timeout: float = 60.0,
+    max_timeout: float = 300.0,
+    retries: int = 5,
+    **kwargs: Any,
 ) -> Any:
     """
-    Call a gRPC function with a timeout (in seconds).
+    Call a gRPC function with a timeout.
+
+    This timeout is increased dynamically on every none-successful reconnection attempts.
 
     Args:
         call: The gRPC method to be called.
         *args: Positional arguments for the gRPC call.
-        timeout: Timeout duration in seconds. Defaults to 300.
+        initial_timeout: Initial timeout duration in seconds.
+        max_timeout: Maximum timeout duration in seconds.
+        retries: Number of retry attempts for timeout errors.
         **kwargs: Keyword arguments for the gRPC call.
 
     Returns:
         The result of the gRPC call.
 
     Raises:
-        asyncio.TimeoutError: If the call exceeds the timeout.
+        asyncio.TimeoutError: If all retries are exhausted.
     """
-    try:
-        return await asyncio.wait_for(call(*args, **kwargs), timeout=timeout)
-    except asyncio.TimeoutError:
-        _logger.error("Timeout while calling %s", call)
-        raise
+    timeout_growth_factor = (max_timeout / initial_timeout) ** (1 / max(1, retries - 1))
+    timeout = initial_timeout
+
+    for attempt in range(1, retries + 1):
+        try:
+            return await asyncio.wait_for(call(*args, **kwargs), timeout=timeout)
+        except asyncio.TimeoutError:
+            if attempt == retries:
+                _logger.error(
+                    "Timeout after %d retries (timeout=%ds): %s",
+                    retries,
+                    timeout,
+                    call,
+                )
+                raise
+            _logger.warning(
+                "Timeout on attempt %d/%d (timeout=%ds). Retrying with increased timeout.",
+                attempt,
+                retries,
+                timeout,
+            )
+            timeout = min(max_timeout, timeout * timeout_growth_factor)
 
 
 class Client(BaseApiClient[ElectricityTradingServiceStub]):