Use pyroscope sample rate as config parameter (#3603)

## Motivation

Right now for local runs, 100 samples per second seems to be a bit too aggressive and causes some local issues (connections fail, etc)

## Proposal

Decrease local runs to have 10 samples per second, and make that part of the config

## Test Plan

Ran locally, was significantly seeing connection failures before, now they're gone

## Release Plan

- Nothing to do / These changes follow the usual release cycle.

Author: ndr-ds

configuration/compose/validator.toml

@@ -9,6 +9,7 @@ port = 19100
 metrics_port = 21100
 pyroscope_host = "docker-pyroscope"
 pyroscope_port = 4040
+pyroscope_sample_rate = 100
 internal_host = "proxy"
 internal_port = 20100
 [external_protocol]
@@ -22,24 +23,28 @@ port = 19100
 metrics_port = 21100
 pyroscope_host = "docker-pyroscope"
 pyroscope_port = 4040
+pyroscope_sample_rate = 100
 
 [[shards]]
 host = "docker-shard-2"
 port = 19100
 metrics_port = 21100
 pyroscope_host = "docker-pyroscope"
 pyroscope_port = 4040
+pyroscope_sample_rate = 100
 
 [[shards]]
 host = "docker-shard-3"
 port = 19100
 metrics_port = 21100
 pyroscope_host = "docker-pyroscope"
 pyroscope_port = 4040
+pyroscope_sample_rate = 100
 
 [[shards]]
 host = "docker-shard-4"
 port = 19100
 metrics_port = 21100
 pyroscope_host = "docker-pyroscope"
 pyroscope_port = 4040
+pyroscope_sample_rate = 100

linera-rpc/src/config.rs

@@ -56,6 +56,8 @@ pub struct ShardConfig {
     pub pyroscope_host: String,
     /// The port on which pyroscope is served.
     pub pyroscope_port: Option<u16>,
+    /// The sample rate for pyroscope.
+    pub pyroscope_sample_rate: u32,
 }
 
 impl ShardConfig {
@@ -121,6 +123,8 @@ pub struct ValidatorInternalNetworkPreConfig<P> {
     pub pyroscope_host: String,
     /// The port of the pyroscope server.
     pub pyroscope_port: u16,
+    /// The sample rate for pyroscope.
+    pub pyroscope_sample_rate: u32,
 }
 
 impl<P> ValidatorInternalNetworkPreConfig<P> {
@@ -134,6 +138,7 @@ impl<P> ValidatorInternalNetworkPreConfig<P> {
             metrics_port: self.metrics_port,
             pyroscope_host: self.pyroscope_host.clone(),
             pyroscope_port: self.pyroscope_port,
+            pyroscope_sample_rate: self.pyroscope_sample_rate,
         }
     }
 }

linera-service/src/cli_wrappers/local_kubernetes_net.rs

@@ -342,6 +342,7 @@ impl LocalKubernetesNet {
         let internal_port = 20100;
         let metrics_port = 21100;
         let pyroscope_port = 4040;
+        let pyroscope_sample_rate = 10;
         let mut content = format!(
             r#"
                 server_config_path = "server_{n}.json"
@@ -352,6 +353,7 @@ impl LocalKubernetesNet {
                 metrics_port = {metrics_port}
                 pyroscope_host = "linera-core-pyroscope.default.svc.cluster.local"
                 pyroscope_port = {pyroscope_port}
+                pyroscope_sample_rate = {pyroscope_sample_rate}
                 [external_protocol]
                 Grpc = "ClearText"
                 [internal_protocol]
@@ -371,6 +373,7 @@ impl LocalKubernetesNet {
                 metrics_port = {shard_metrics_port}
                 pyroscope_host = "linera-core-pyroscope.default.svc.cluster.local"
                 pyroscope_port = {shard_pyroscope_port}
+                pyroscope_sample_rate = {pyroscope_sample_rate}
                 "#
             ));
         }

linera-service/src/cli_wrappers/local_net.rs

@@ -432,6 +432,7 @@ impl LocalNet {
         let internal_port = Self::internal_port(n);
         let metrics_port = Self::proxy_metrics_port(n);
         let pyroscope_port = Self::proxy_pyroscope_port(n);
+        let pyroscope_sample_rate = 100;
         let external_protocol = self.network.external.toml();
         let internal_protocol = self.network.internal.toml();
         let external_host = self.network.external.localhost();
@@ -447,6 +448,7 @@ impl LocalNet {
                 metrics_port = {metrics_port}
                 pyroscope_port = {pyroscope_port}
                 pyroscope_host = "{pyroscope_host}"
+                pyroscope_sample_rate = {pyroscope_sample_rate}
                 external_protocol = {external_protocol}
                 internal_protocol = {internal_protocol}
             "#
@@ -464,6 +466,7 @@ impl LocalNet {
                 metrics_port = {shard_metrics_port}
                 pyroscope_host = "{pyroscope_host}"
                 pyroscope_port = {shard_pyroscope_port}
+                pyroscope_sample_rate = {pyroscope_sample_rate}
                 "#
             ));
         }

linera-service/src/proxy/grpc.rs

@@ -261,6 +261,7 @@ where
                 self.pyroscope_address(),
                 "proxy".to_string(),
                 shutdown_signal.clone(),
+                self.0.internal_config.pyroscope_sample_rate,
             )?;
         }
 

linera-service/src/proxy/main.rs

@@ -257,6 +257,7 @@ where
                 self.get_pyroscope_address(),
                 "proxy".to_string(),
                 shutdown_signal.clone(),
+                self.internal_config.pyroscope_sample_rate,
             )?;
         }
 

linera-service/src/pyroscope_server.rs

@@ -10,9 +10,10 @@ pub fn start_pyroscope(
     address: String,
     application_name: String,
     shutdown_signal: CancellationToken,
+    sample_rate: u32,
 ) -> Result<(), PyroscopeError> {
     info!("Starting to push pyroscope data to {:?}", address);
-    let pprof_config = PprofConfig::new().sample_rate(100);
+    let pprof_config = PprofConfig::new().sample_rate(sample_rate);
     let backend_impl = pprof_backend(pprof_config);
 
     let agent = PyroscopeAgent::builder(address, application_name)

linera-service/src/server.rs

@@ -120,6 +120,7 @@ impl ServerContext {
                     self.pyroscope_address(shard.pyroscope_host, port),
                     "server".to_string(),
                     shutdown_signal.clone(),
+                    shard.pyroscope_sample_rate,
                 )?;
             }
 
@@ -175,6 +176,7 @@ impl ServerContext {
                     self.pyroscope_address(shard.pyroscope_host, port),
                     "server".to_string(),
                     shutdown_signal.clone(),
+                    shard.pyroscope_sample_rate,
                 )?;
             }
 
@@ -303,6 +305,9 @@ struct ValidatorOptions {
     /// The port for the pyroscope endpoint
     pyroscope_port: u16,
 
+    /// The sample rate for pyroscope.
+    pyroscope_sample_rate: u32,
+
     /// The host of the proxy in the internal network.
     internal_host: String,
 
@@ -341,6 +346,7 @@ fn make_server_config<R: CryptoRng>(
         metrics_port: options.metrics_port,
         pyroscope_host: options.pyroscope_host,
         pyroscope_port: options.pyroscope_port,
+        pyroscope_sample_rate: options.pyroscope_sample_rate,
     };
     let validator = ValidatorConfig {
         network,
@@ -491,6 +497,10 @@ enum ServerCommand {
         /// shard number.
         #[arg(long)]
         pyroscope_port: Option<String>,
+
+        /// The sample rate for pyroscope.
+        #[arg(long, default_value = "100")]
+        pyroscope_sample_rate: u32,
     },
 }
 
@@ -670,6 +680,7 @@ async fn run(options: ServerOptions) {
             metrics_port,
             pyroscope_host,
             pyroscope_port,
+            pyroscope_sample_rate,
         } => {
             let mut server_config =
                 persistent::File::<ValidatorServerConfig>::read(&server_config_path)
@@ -681,6 +692,7 @@ async fn run(options: ServerOptions) {
                 metrics_port,
                 pyroscope_host,
                 pyroscope_port,
+                pyroscope_sample_rate,
             )
             .expect("Failed to generate shard configs");
             server_config.internal_network.shards = shards;
@@ -698,6 +710,7 @@ fn generate_shard_configs(
     metrics_port: Option<String>,
     pyroscope_host: String,
     pyroscope_port: Option<String>,
+    pyroscope_sample_rate: u32,
 ) -> anyhow::Result<Vec<ShardConfig>> {
     let mut shards = Vec::new();
     let len = num_shards.len();
@@ -736,6 +749,7 @@ fn generate_shard_configs(
             metrics_port,
             pyroscope_host,
             pyroscope_port,
+            pyroscope_sample_rate,
         };
         shards.push(shard);
     }
@@ -759,6 +773,7 @@ mod test {
             metrics_port = 5000
             pyroscope_host = "pyroscope_host"
             pyroscope_port = 4000
+            pyroscope_sample_rate = 100
             external_protocol = { Simple = "Tcp" }
             internal_protocol = { Simple = "Udp" }
 
@@ -768,13 +783,15 @@ mod test {
             metrics_port = 5001
             pyroscope_host = "pyroscope_host"
             pyroscope_port = 4001
+            pyroscope_sample_rate = 100
 
             [[shards]]
             host = "host2"
             port = 9002
             metrics_port = 5002
             pyroscope_host = "pyroscope_host"
             pyroscope_port = 4002
+            pyroscope_sample_rate = 100
         "#;
         let options: ValidatorOptions = toml::from_str(toml_str).unwrap();
         assert_eq!(
@@ -790,20 +807,23 @@ mod test {
                 metrics_port: 5000,
                 pyroscope_host: "pyroscope_host".into(),
                 pyroscope_port: 4000,
+                pyroscope_sample_rate: 100,
                 shards: vec![
                     ShardConfig {
                         host: "host1".into(),
                         port: 9001,
                         metrics_port: Some(5001),
                         pyroscope_host: "pyroscope_host".into(),
                         pyroscope_port: Some(4001),
+                        pyroscope_sample_rate: 100,
                     },
                     ShardConfig {
                         host: "host2".into(),
                         port: 9002,
                         metrics_port: Some(5002),
                         pyroscope_host: "pyroscope_host".into(),
                         pyroscope_port: Some(4002),
+                        pyroscope_sample_rate: 100,
                     },
                 ],
             }
@@ -820,6 +840,7 @@ mod test {
                 Some("11%%".into()),
                 "pyroscope_host".into(),
                 Some("40%%".into()),
+                100,
             )
             .unwrap(),
             vec![
@@ -829,13 +850,15 @@ mod test {
                     metrics_port: Some(1101),
                     pyroscope_host: "pyroscope_host".into(),
                     pyroscope_port: Some(4001),
+                    pyroscope_sample_rate: 100,
                 },
                 ShardConfig {
                     host: "host02".into(),
                     port: 1002,
                     metrics_port: Some(1102),
                     pyroscope_host: "pyroscope_host".into(),
                     pyroscope_port: Some(4002),
+                    pyroscope_sample_rate: 100,
                 },
             ],
         );
@@ -847,6 +870,7 @@ mod test {
             Some("11%%".into()),
             "pyroscope_host".into(),
             Some("40%%".into()),
+            100,
         )
         .is_err());
     }