chore: network keepalive

charts/kms-core/Chart.yaml

@@ -1,6 +1,6 @@
 name: kms-core
 description: A helm chart to distribute and deploy the Zama KMS core service.
-version: 1.5.0-beta.1
+version: 1.5.0-beta.2
 appVersion: 0.13.0   # Minimum kms version to run this chart
 apiVersion: v2
 keywords:

charts/kms-core/templates/kms-core-configmap.yaml

@@ -93,6 +93,9 @@ data:
     listen_port = {{ int .Values.kmsCore.ports.client }}
     timeout_secs = {{ int .Values.kmsCore.resources.limits.grpcTimeout }}
     grpc_max_message_size = {{ int .Values.kmsCore.resources.limits.grpcMaxMessageSize }}
+    http2_keep_alive_interval_secs = {{ int .Values.kmsCore.resources.keepalive.Http2KeepAliveIntervalSecs }}
+    http2_keep_alive_timeout_secs = {{ int .Values.kmsCore.resources.keepalive.Http2KeepAliveTimeoutSecs }}
+    tcp_keep_alive_secs = {{ int .Values.kmsCore.resources.keepalive.TcpKeepAliveSecs }}
 
     {{ if .Values.kmsCore.thresholdMode.enabled }}
     [threshold]
@@ -123,6 +126,9 @@ data:
     discard_inactive_sessions_interval = 900
     max_waiting_time_for_message_queue = 60
     max_opened_inactive_sessions_per_party = {{ int .Values.kmsCore.thresholdMode.maxOpenedInactiveSessionsPerParty }}
+    http2_keep_alive_interval_secs = {{ int .Values.kmsCore.thresholdMode.Http2KeepAliveIntervalSecs }}
+    http2_keep_alive_timeout_secs = {{ int .Values.kmsCore.thresholdMode.Http2KeepAliveTimeoutSecs }}
+    tcp_keep_alive_secs = {{ int .Values.kmsCore.thresholdMode.TcpKeepAliveSecs }}
 
       {{- if $.Values.kmsCore.thresholdMode.tls.enabled }}
         {{- if $.Values.kmsCore.nitroEnclave.enabled }}

charts/kms-core/values.yaml

@@ -101,6 +101,9 @@ kmsCore:
     refreshIntervalMs: 5000
     tokioWorkerThreads: 10
     rayonNumThreads: 40
+    Http2KeepAliveIntervalSecs: 30
+    Http2KeepAliveTimeoutSecs: 10
+    TcpKeepAliveSecs: 30
     # Threshold value is the number of corruptions that the protocol handles.
     # 1 for 4 parties, 4 for 13 parties
     thresholdValue: 4
@@ -201,6 +204,10 @@ kmsCore:
       grpcTimeout: 360
       # 100MB
       grpcMaxMessageSize: 104857600
+    keepalive:
+      Http2KeepAliveIntervalSecs: 30
+      Http2KeepAliveTimeoutSecs: 10
+      TcpKeepAliveSecs: 30
   nodeSelector:
   affinity:
   tolerations:

core/service/config/compose_1.toml

@@ -7,6 +7,9 @@ listen_address = "0.0.0.0"
 listen_port = 50100
 timeout_secs = 360
 grpc_max_message_size = 104857600 # 100 MiB
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 [aws]
 region = "us-east-1"
@@ -86,6 +89,9 @@ session_cleanup_interval_secs = 3600
 discard_inactive_sessions_interval = 900
 max_waiting_time_for_message_queue = 60
 max_opened_inactive_sessions_per_party = 100
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 
 # [threshold.preproc_redis]

core/service/config/compose_2.toml

@@ -7,6 +7,9 @@ listen_address = "0.0.0.0"
 listen_port = 50200
 timeout_secs = 360
 grpc_max_message_size = 104857600 # 100 MiB
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 [aws]
 region = "us-east-1"
@@ -86,6 +89,9 @@ session_cleanup_interval_secs = 3600
 discard_inactive_sessions_interval = 900
 max_waiting_time_for_message_queue = 60
 max_opened_inactive_sessions_per_party = 100
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 # [threshold.preproc_redis]
 # host = "redis://127.0.0.1"

core/service/config/compose_3.toml

@@ -7,6 +7,9 @@ listen_address = "0.0.0.0"
 listen_port = 50300
 timeout_secs = 360
 grpc_max_message_size = 104857600 # 100 MiB
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 [aws]
 region = "us-east-1"
@@ -86,6 +89,9 @@ session_cleanup_interval_secs = 3600
 discard_inactive_sessions_interval = 900
 max_waiting_time_for_message_queue = 60
 max_opened_inactive_sessions_per_party = 100
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 # [threshold.preproc_redis]
 # host = "redis://127.0.0.1"

core/service/config/compose_4.toml

@@ -7,6 +7,9 @@ listen_address = "0.0.0.0"
 listen_port = 50400
 timeout_secs = 360
 grpc_max_message_size = 104857600 # 100 MiB
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 [aws]
 region = "us-east-1"
@@ -86,6 +89,9 @@ session_cleanup_interval_secs = 3600
 discard_inactive_sessions_interval = 900
 max_waiting_time_for_message_queue = 60
 max_opened_inactive_sessions_per_party = 100
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 # [threshold.preproc_redis]
 # host = "redis://127.0.0.1"

core/service/config/compose_5.toml

@@ -7,6 +7,9 @@ listen_address = "0.0.0.0"
 listen_port = 50500
 timeout_secs = 360
 grpc_max_message_size = 104857600 # 100 MiB
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 [aws]
 region = "us-east-1"
@@ -86,6 +89,9 @@ session_cleanup_interval_secs = 3600
 discard_inactive_sessions_interval = 900
 max_waiting_time_for_message_queue = 60
 max_opened_inactive_sessions_per_party = 100
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 # [threshold.preproc_redis]
 # host = "redis://127.0.0.1"

core/service/config/compose_6.toml

@@ -7,6 +7,9 @@ listen_address = "0.0.0.0"
 listen_port = 50600
 timeout_secs = 360
 grpc_max_message_size = 104857600 # 100 MiB
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 [aws]
 region = "us-east-1"
@@ -86,6 +89,9 @@ session_cleanup_interval_secs = 3600
 discard_inactive_sessions_interval = 900
 max_waiting_time_for_message_queue = 60
 max_opened_inactive_sessions_per_party = 100
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 # [threshold.preproc_redis]
 # host = "redis://127.0.0.1"

core/service/config/compose_centralized.toml

@@ -5,6 +5,9 @@ listen_address = "0.0.0.0"
 listen_port = 50051
 timeout_secs = 360
 grpc_max_message_size = 104857600 # 100 MiB
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 [telemetry]
 tracing_service_name = "kms-centralized"

core/service/config/default_1.toml

@@ -16,6 +16,10 @@ timeout_secs = 360
 # Maximum gRPC message size in bytes.
 grpc_max_message_size = 104857600 # 100 MiB
 
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
+
 # Set the AWS region and endpoint used by [public_vault.storage]
 # and/or [private_vault.storage], if the storage URL starts with "s3://".
 # If the storage URL is file-based, this configuration is ignored.
@@ -193,6 +197,9 @@ session_cleanup_interval_secs = 3600
 discard_inactive_sessions_interval = 900
 max_waiting_time_for_message_queue = 60
 max_opened_inactive_sessions_per_party = 100
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 # Redis instance for preprocessed material (unset by default).
 # [threshold.preproc_redis]

core/service/config/default_2.toml

@@ -5,6 +5,9 @@ listen_address = "0.0.0.0"
 listen_port = 50200
 timeout_secs = 360
 grpc_max_message_size = 104857600 # 100 MiB
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 [aws]
 region = "us-east-1"
@@ -76,6 +79,9 @@ session_cleanup_interval_secs = 3600
 discard_inactive_sessions_interval = 900
 max_waiting_time_for_message_queue = 60
 max_opened_inactive_sessions_per_party = 100
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 # [threshold.preproc_redis]
 # host = "redis://127.0.0.1"

core/service/config/default_3.toml

@@ -5,6 +5,9 @@ listen_address = "0.0.0.0"
 listen_port = 50300
 timeout_secs = 360
 grpc_max_message_size = 104857600 # 100 MiB
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 [aws]
 region = "us-east-1"
@@ -76,6 +79,9 @@ session_cleanup_interval_secs = 3600
 discard_inactive_sessions_interval = 900
 max_waiting_time_for_message_queue = 60
 max_opened_inactive_sessions_per_party = 100
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 # [threshold.preproc_redis]
 # host = "redis://127.0.0.1"

core/service/config/default_4.toml

@@ -5,6 +5,9 @@ listen_address = "0.0.0.0"
 listen_port = 50400
 timeout_secs = 360
 grpc_max_message_size = 104857600 # 100 MiB
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 [aws]
 region = "us-east-1"
@@ -76,6 +79,9 @@ session_cleanup_interval_secs = 3600
 discard_inactive_sessions_interval = 900
 max_waiting_time_for_message_queue = 60
 max_opened_inactive_sessions_per_party = 100
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 # [threshold.preproc_redis]
 # host = "redis://127.0.0.1"

core/service/config/default_centralized.toml

@@ -5,6 +5,9 @@ listen_address = "0.0.0.0"
 listen_port = 50051
 timeout_secs = 360
 grpc_max_message_size = 104857600 # 100 MiB
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 [telemetry]
 tracing_service_name = "kms-centralized"

core/service/config/default_centralized_enclave.toml

@@ -5,6 +5,9 @@ listen_address = "0.0.0.0"
 listen_port = 50051
 timeout_secs = 360
 grpc_max_message_size = 104857600 # 100 MiB
+http2_keep_alive_interval_secs = 30
+http2_keep_alive_timeout_secs = 10
+tcp_keep_alive_secs = 30
 
 [telemetry]
 tracing_service_name = "kms-centralized"

core/service/src/client/test_tools.rs

@@ -107,6 +107,9 @@ pub async fn setup_threshold_no_client<
             listen_port: service_ports[i - 1],
             timeout_secs: 60u64,
             grpc_max_message_size: GRPC_MAX_MESSAGE_SIZE,
+            http2_keep_alive_interval_secs: Some(30),
+            http2_keep_alive_timeout_secs: Some(10),
+            tcp_keep_alive_secs: Some(30),
         };
         let mpc_conf = mpc_confs.clone();
 
@@ -439,6 +442,9 @@ pub async fn setup_centralized_no_client<
             listen_port,
             timeout_secs: 360,
             grpc_max_message_size: GRPC_MAX_MESSAGE_SIZE,
+            http2_keep_alive_interval_secs: Some(30),
+            http2_keep_alive_timeout_secs: Some(10),
+            tcp_keep_alive_secs: Some(30),
         };
 
         run_server(

core/service/src/conf/mod.rs

@@ -76,6 +76,12 @@ pub struct ServiceEndpoint {
     // maximum gRPC message size in bytes
     #[validate(range(min = 1, max = 2147483647))]
     pub grpc_max_message_size: usize,
+    #[validate(range(min = 1))]
+    pub http2_keep_alive_interval_secs: Option<u64>,
+    #[validate(range(min = 1))]
+    pub http2_keep_alive_timeout_secs: Option<u64>,
+    #[validate(range(min = 1))]
+    pub tcp_keep_alive_secs: Option<u64>,
 }
 
 pub trait ConfigTracing {

core/service/src/engine/server.rs

@@ -117,6 +117,17 @@ pub async fn run_server<
     );
     let server = Server::builder()
         .http2_adaptive_window(Some(true))
+        .http2_keepalive_interval(
+            config
+                .http2_keep_alive_interval_secs
+                .map(Duration::from_secs),
+        )
+        .http2_keepalive_timeout(
+            config
+                .http2_keep_alive_timeout_secs
+                .map(Duration::from_secs),
+        )
+        .tcp_keepalive(config.tcp_keep_alive_secs.map(Duration::from_secs))
         .layer(trace_request)
         // Make sure we never abort because we spent too much time on the blocking part of the get result
         // as we mean to do it.

core/service/src/engine/threshold/service/kms_impl.rs

@@ -28,7 +28,7 @@ use threshold_fhe::{
         zk::ceremony::SecureCeremony,
     },
     networking::{
-        grpc::{GrpcNetworkingManager, GrpcServer, TlsExtensionGetter},
+        grpc::{GrpcNetworkingManager, GrpcServer, OptionConfigWrapper, TlsExtensionGetter},
         tls::AttestedVerifier,
     },
 };
@@ -280,8 +280,12 @@ where
         .write()
         .await
         .new_server(TlsExtensionGetter::SslConnectInfo);
+    let p2p_config: OptionConfigWrapper = config.core_to_core_net.into();
     let router = Server::builder()
         .http2_adaptive_window(Some(true))
+        .http2_keepalive_interval(Some(p2p_config.get_http2_keep_alive_interval()))
+        .http2_keepalive_timeout(Some(p2p_config.get_http2_keep_alive_timeout()))
+        .tcp_keepalive(Some(p2p_config.get_tcp_keep_alive()))
         .add_service(networking_server)
         .add_service(threshold_health_service);
 

core/threshold/src/networking/constants.rs

@@ -25,6 +25,11 @@ pub(crate) const DISCARD_INACTIVE_SESSION_INTERVAL_SECS: u64 = 15 * 60;
 
 // The default maximum waiting time we wait for trying to push the message in the queue
 pub(crate) const MAX_WAITING_TIME_MESSAGE_QUEUE: u64 = 60;
+
+pub(crate) const HTTP2_KEEP_ALIVE_INTERVAL_SEC: u64 = 30;
+pub(crate) const HTTP2_KEEP_ALIVE_TIMEOUT_SEC: u64 = 10;
+pub(crate) const TCP_KEEP_ALIVE_SEC: u64 = 30;
+
 lazy_static! {
     /// The default maximum internal between retries (Cap at 60s intervals)
     pub static ref MAX_INTERVAL: Duration = Duration::from_secs(60);