feat: adapt health check timeout algorithm (#1559)

Co-authored-by: pyansys-ci-bot <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: 3 people

doc/changelog.d/1559.added.md

@@ -0,0 +1 @@
+adapt health check timeout algorithm

src/ansys/geometry/core/connection/client.py

@@ -73,8 +73,15 @@ def wait_until_healthy(channel: grpc.Channel, timeout: float):
     channel : ~grpc.Channel
         Channel that must be established and healthy.
     timeout : float
-        Timeout in seconds. An attempt is made every 100 milliseconds
-        until the timeout is exceeded.
+        Timeout in seconds. Attempts are made with the following backoff strategy:
+
+        * Starts with 0.1 seconds.
+        * If the attempt fails, double the timeout.
+        * This is repeated until the next timeoff exceeds the
+          value for the remaining time. In that case, a final attempt
+          is made with the remaining time.
+        * If the total elapsed time exceeds the value for the ``timeout`` parameter,
+          a ``TimeoutError`` is raised.
 
     Raises
     ------
@@ -84,12 +91,21 @@ def wait_until_healthy(channel: grpc.Channel, timeout: float):
     t_max = time.time() + timeout
     health_stub = health_pb2_grpc.HealthStub(channel)
     request = health_pb2.HealthCheckRequest(service="")
+
+    t_out = 0.1
     while time.time() < t_max:
         try:
-            out = health_stub.Check(request, timeout=0.1)
+            out = health_stub.Check(request, timeout=t_out)
             if out.status is health_pb2.HealthCheckResponse.SERVING:
                 break
         except _InactiveRpcError:
+            # Duplicate timeout and try again
+            t_now = time.time()
+            t_out *= 2
+            # If we have time to try again, continue.. but if we don't,
+            # just try for the remaining time
+            if t_now + t_out > t_max:
+                t_out = t_max - t_now
             continue
     else:
         target_str = channel._channel.target().decode()
@@ -171,7 +187,8 @@ def __init__(
             )
 
         # do not finish initialization until channel is healthy
-        wait_until_healthy(self._channel, timeout)
+        self._grpc_health_timeout = timeout
+        wait_until_healthy(self._channel, self._grpc_health_timeout)
 
         # once connection with the client is established, create a logger
         self._log = LOG.add_instance_logger(
@@ -275,12 +292,10 @@ def healthy(self) -> bool:
         """Flag indicating whether the client channel is healthy."""
         if self._closed:
             return False
-        health_stub = health_pb2_grpc.HealthStub(self._channel)
-        request = health_pb2.HealthCheckRequest(service="")
         try:
-            out = health_stub.Check(request, timeout=0.1)
-            return out.status is health_pb2.HealthCheckResponse.SERVING
-        except _InactiveRpcError:  # pragma: no cover
+            wait_until_healthy(self._channel, self._grpc_health_timeout)
+            return True
+        except TimeoutError:  # pragma: no cover
             return False
 
     def __repr__(self) -> str: