remote: rework grpc timeout configuration

It's not really clear how keepalive_timeout_ms and the ping_timeout_ms
experiment should interact, so we set them both.

Signed-off-by: Jan Luebbe <[email protected]>

Author: jluebbe

labgrid/remote/client.py

@@ -86,7 +86,21 @@ def __attrs_post_init__(self):
         """Actions which are executed if a connection is successfully opened."""
         self.stopping = asyncio.Event()
 
-        self.channel = grpc.aio.insecure_channel(self.address)
+        # It seems since https://github.com/grpc/grpc/pull/34647, the
+        # ping_timeout_ms default of 60 seconds overrides keepalive_timeout_ms,
+        # so set it as well.
+        # Use GRPC_VERBOSITY=DEBUG GRPC_TRACE=http_keepalive for debugging.
+        channel_options = [
+            ("grpc.keepalive_time_ms", 7500),  # 7.5 seconds
+            ("grpc.keepalive_timeout_ms", 10000),  # 10 seconds
+            ("grpc.http2.ping_timeout_ms", 10000),  # 10 seconds
+            ("grpc.http2.max_pings_without_data", 0),  # no limit
+        ]
+
+        self.channel = grpc.aio.insecure_channel(
+            target=self.address,
+            options=channel_options,
+        )
         self.stub = labgrid_coordinator_pb2_grpc.CoordinatorStub(self.channel)
 
         self.out_queue = asyncio.Queue()

labgrid/remote/coordinator.py

@@ -959,17 +959,20 @@ async def GetReservations(self, request: labgrid_coordinator_pb2.GetReservations
 
 
 async def serve(listen, cleanup) -> None:
+    # It seems since https://github.com/grpc/grpc/pull/34647, the
+    # ping_timeout_ms default of 60 seconds overrides keepalive_timeout_ms,
+    # so set it as well.
+    # Use GRPC_VERBOSITY=DEBUG GRPC_TRACE=http_keepalive for debugging.
+    channel_options = [
+        ("grpc.keepalive_time_ms", 10000),  # 10 seconds
+        ("grpc.keepalive_timeout_ms", 10000),  # 10 seconds
+        ("grpc.http2.ping_timeout_ms", 15000),  # 15 seconds
+        ("grpc.http2.min_ping_interval_without_data_ms", 5000),
+        ("grpc.http2.max_pings_without_data", 0),  # no limit
+        ("grpc.keepalive_permit_without_calls", 1),  # allow keepalive pings even when there are no calls
+    ]
     server = grpc.aio.server(
-        options=[
-            ("grpc.keepalive_time_ms", 30000),  # Send keepalive ping every 30 seconds
-            (
-                "grpc.keepalive_timeout_ms",
-                10000,
-            ),  # Wait 10 seconds for ping ack before considering the connection dead
-            ("grpc.http2.min_time_between_pings_ms", 15000),  # Minimum amount of time between pings
-            ("grpc.http2.max_pings_without_data", 0),  # Allow pings even without active streams
-            ("grpc.keepalive_permit_without_calls", 1),  # Allow keepalive pings even when there are no calls
-        ],
+        options=channel_options,
     )
     coordinator = Coordinator()
     labgrid_coordinator_pb2_grpc.add_CoordinatorServicer_to_server(coordinator, server)

labgrid/remote/exporter.py

@@ -783,11 +783,25 @@ def __init__(self, config) -> None:
         self.hostname = config["hostname"]
         self.isolated = config["isolated"]
 
+        # It seems since https://github.com/grpc/grpc/pull/34647, the
+        # ping_timeout_ms default of 60 seconds overrides keepalive_timeout_ms,
+        # so set it as well.
+        # Use GRPC_VERBOSITY=DEBUG GRPC_TRACE=http_keepalive for debugging.
+        channel_options = [
+            ("grpc.keepalive_time_ms", 7500),  # 7.5 seconds
+            ("grpc.keepalive_timeout_ms", 10000),  # 10 seconds
+            ("grpc.http2.ping_timeout_ms", 10000),  # 10 seconds
+            ("grpc.http2.max_pings_without_data", 0),  # no limit
+        ]
+
         # default to port 20408 if not specified
         if urlsplit(f"//{config['coordinator']}").port is None:
             config["coordinator"] += ":20408"
 
-        self.channel = grpc.aio.insecure_channel(config["coordinator"])
+        self.channel = grpc.aio.insecure_channel(
+            target=config["coordinator"],
+            options=channel_options,
+        )
         self.stub = labgrid_coordinator_pb2_grpc.CoordinatorStub(self.channel)
         self.out_queue = asyncio.Queue()
         self.pump_task = None