Support gRPC connection options for retry and timeout

Author: juergbi

doc/source/using_config.rst

@@ -154,6 +154,9 @@ toplevel of your configuration file, like so:
          access-token-reload-interval: 60
        connection-config:
          keepalive-time: 60
+         retry-limit: 4
+         retry-delay: 1000
+         request-timeout: 300
 
 
 Attributes
@@ -789,6 +792,9 @@ The ``connection-config`` block looks like this:
 
    connection-config:
      keepalive-time: 60
+     retry-limit: 4
+     retry-delay: 1000
+     request-timeout: 300
 
 
 Attributes
@@ -799,6 +805,20 @@ Attributes
   The interval in seconds between gRPC PING frames. Disabled by default.
   `gRPC keepalive guide <https://grpc.io/docs/guides/keepalive/>`_
 
+* ``retry-limit``
+
+  The maximum number of retries, not including the original request.
+
+* ``retry-delay``
+
+  The initial backoff in seconds for retries.
+  `gRPC retry guide <https://grpc.io/docs/guides/retry/>`_
+
+* ``request-timeout``
+
+  The timeout for gRPC requests in seconds. No timeout by default.
+  `gRPC deadline guide <https://grpc.io/docs/guides/deadlines/>`_
+
 
 .. _config_cache_servers:
 
@@ -841,6 +861,9 @@ Cache server configuration is declared in the following way:
        client-key: client.key
      connection-config:
        keepalive-time: 60
+       retry-limit: 4
+       retry-delay: 1000
+       request-timeout: 300
 
 
 Attributes
@@ -1167,6 +1190,9 @@ which looks like this:
      client-key: /keys/client.key
    connection-config:
      keepalive-time: 60
+     retry-limit: 4
+     retry-delay: 1000
+     request-timeout: 300
 
 **Attributes:**
 

src/buildstream/_cas/casdprocessmanager.py

@@ -121,6 +121,12 @@ def __init__(
                     )
             if remote_cache_spec.keepalive_time is not None:
                 casd_args.append("--cas-keepalive-time={}".format(remote_cache_spec.keepalive_time))
+            if remote_cache_spec.retry_limit is not None:
+                casd_args.append("--cas-retry-limit={}".format(remote_cache_spec.retry_limit))
+            if remote_cache_spec.retry_delay is not None:
+                casd_args.append("--cas-retry-delay={}".format(remote_cache_spec.retry_delay))
+            if remote_cache_spec.request_timeout is not None:
+                casd_args.append("--cas-request-timeout={}".format(remote_cache_spec.request_timeout))
 
         casd_args.append(path)
 

src/buildstream/_remotespec.py

@@ -140,6 +140,9 @@ def __hash__(self) -> int:
                 self.access_token_file,
                 self.access_token_reload_interval,
                 self.keepalive_time,
+                self.retry_limit,
+                self.retry_delay,
+                self.request_timeout,
             )
         )
 
@@ -195,6 +198,30 @@ def keepalive_time(self) -> Optional[int]:
             return self._connection_config.get_int("keepalive-time", None)
         return None
 
+    # grpc retry limit
+    #
+    @property
+    def retry_limit(self) -> Optional[int]:
+        if self._connection_config:
+            return self._connection_config.get_int("retry-limit", None)
+        return None
+
+    # grpc retry delay in milliseconds
+    #
+    @property
+    def retry_delay(self) -> Optional[int]:
+        if self._connection_config:
+            return self._connection_config.get_int("retry-delay", None)
+        return None
+
+    # grpc request timeout in seconds
+    #
+    @property
+    def request_timeout(self) -> Optional[int]:
+        if self._connection_config:
+            return self._connection_config.get_int("request-timeout", None)
+        return None
+
     # open_channel()
     #
     # Opens a gRPC channel based on this spec.
@@ -234,6 +261,12 @@ def to_localcas_remote(self, remote):
                 remote.access_token_reload_interval.FromSeconds(self.access_token_reload_interval * 60)
         if self.keepalive_time is not None:
             remote.keepalive_time.FromSeconds(self.keepalive_time)
+        if self.retry_limit is not None:
+            remote.retry_limit = self.retry_limit
+        if self.retry_delay is not None:
+            remote.retry_delay.FromMilliseconds(self.retry_delay)
+        if self.request_timeout is not None:
+            remote.request_timeout.FromSeconds(self.request_timeout)
 
     # new_from_node():
     #