otel support

Author: ltitanb

Cargo.lock

@@ -44,7 +44,11 @@ eyre = "0.6.12"
 futures = "0.3.30"
 headers = "0.4.0"
 indexmap = "2.2.6"
+jsonwebtoken = { version = "9.3.1", default-features = false }
 lazy_static = "1.5.0"
+opentelemetry = { version = "0.29", features = ["trace"] }
+opentelemetry-otlp = { version = "0.29", features = ["grpc-tonic", "http-proto", "trace"] }
+opentelemetry_sdk = { version = "0.29", features = ["rt-tokio", "trace"] }
 parking_lot = "0.12.3"
 pbkdf2 = "0.12.2"
 prometheus = "0.13.4"
@@ -63,11 +67,11 @@ tonic = { version = "0.12.3", features = ["channel", "prost", "tls"] }
 tonic-build = "0.12.3"
 tracing = "0.1.40"
 tracing-appender = "0.2.3"
+tracing-opentelemetry = "0.30"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter", "json"] }
 tree_hash = "0.9"
 tree_hash_derive = "0.9"
 typenum = "1.17.0"
 unicode-normalization = "0.1.24"
 url = { version = "2.5.0", features = ["serde"] }
 uuid = { version = "1.8.0", features = ["fast-rng", "serde", "v4"] }
-jsonwebtoken = { version = "9.3.1", default-features = false }

Cargo.toml

@@ -129,8 +129,8 @@ loader = "./tests/data/mux_keys.example.json"
 # loader = { url = "http://localhost:8000/keys" }
 # loader = { registry = "lido", node_operator_id = 8 }
 # loader = { registry = "ssv", node_operator_id = 8 }
-timeout_get_header_ms = 900
 late_in_slot_time_ms = 1500
+timeout_get_header_ms = 900
 # For each mux, one or more [[mux.relays]] can be defined, which will be used for the matching validator pubkeys
 # Only the relays defined here will be used, and the relays defined in the main [[relays]] config will be ignored
 # The fields specified here are the same as in [[relays]] (headers, get_params, enable_timing_games, target_first_request_ms, frequency_get_header_ms)
@@ -254,6 +254,12 @@ host = "127.0.0.1"
 # OPTIONAL, DEFAULT: 10000
 start_port = 10000
 
+[logs]
+# Whether to export OpenTelemtry traces. OTEL is configured via env variables, set those in the [env] section or in your docker compose.
+# For more info see: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md
+# OPTIONAL, DEFAULT: false
+export_traces = false
+
 # Configuration stdout logs
 # OPTIONAL, DEFAULT: enabled
 [logs.stdout]
@@ -288,3 +294,8 @@ dir_path = "./logs"
 # Maximum number of log files to keep
 # OPTIONAL
 max_files = 30
+
+# Env variables set for all services.
+# OPTIONAL
+[env]
+A_COMMON_ENV = "a_common_value"

config.example.toml

@@ -72,6 +72,12 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re
     // targets to pass to prometheus
     let mut targets = Vec::new();
 
+    let mut service_common_envs = IndexMap::new();
+    for (key, val) in cb_config.env {
+        let (key, val) = get_env_val(key.as_str(), val.as_str());
+        service_common_envs.insert(key, val);
+    }
+
     // address for signer API communication
     let signer_port = 20000;
     let signer_server =
@@ -112,6 +118,7 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re
                         get_env_interp(MODULE_JWT_ENV, &jwt_name),
                         get_env_val(SIGNER_URL_ENV, &signer_server),
                     ]);
+                    module_envs.extend(service_common_envs.clone());
 
                     // Pass on the env variables
                     if let Some(envs) = module.env {
@@ -198,6 +205,7 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re
                         get_env_val(CONFIG_ENV, CONFIG_DEFAULT),
                         get_env_uval(BUILDER_PORT_ENV, builder_events_port),
                     ]);
+                    module_envs.extend(service_common_envs.clone());
 
                     if let Some((key, val)) = chain_spec_env.clone() {
                         module_envs.insert(key, val);
@@ -247,6 +255,7 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re
     // setup pbs service
 
     let mut pbs_envs = IndexMap::from([get_env_val(CONFIG_ENV, CONFIG_DEFAULT)]);
+    pbs_envs.extend(service_common_envs.clone());
     let mut pbs_volumes = vec![config_volume.clone()];
 
     // ports
@@ -336,6 +345,7 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re
                     get_env_same(JWTS_ENV),
                     get_env_uval(SIGNER_PORT_ENV, signer_port as u64),
                 ]);
+                signer_envs.extend(service_common_envs.clone());
 
                 let mut ports = vec![];
 
@@ -464,6 +474,7 @@ pub async fn handle_docker_init(config_path: PathBuf, output_dir: PathBuf) -> Re
                     get_env_val(DIRK_KEY_ENV, DIRK_KEY_DEFAULT),
                     get_env_val(DIRK_DIR_SECRETS_ENV, DIRK_DIR_SECRETS_DEFAULT),
                 ]);
+                signer_envs.extend(service_common_envs.clone());
 
                 let mut ports = vec![];
 

crates/cli/src/docker_init.rs

@@ -21,6 +21,10 @@ ethereum_serde_utils.workspace = true
 ethereum_ssz.workspace = true
 ethereum_ssz_derive.workspace = true
 eyre.workspace = true
+jsonwebtoken.workspace = true
+opentelemetry.workspace = true
+opentelemetry-otlp.workspace = true
+opentelemetry_sdk.workspace = true
 pbkdf2.workspace = true
 rand.workspace = true
 reqwest.workspace = true
@@ -35,9 +39,9 @@ toml.workspace = true
 tonic.workspace = true
 tracing.workspace = true
 tracing-appender.workspace = true
+tracing-opentelemetry.workspace = true
 tracing-subscriber.workspace = true
 tree_hash.workspace = true
 tree_hash_derive.workspace = true
 unicode-normalization.workspace = true
 url.workspace = true
-jsonwebtoken.workspace = true

crates/common/Cargo.toml

@@ -8,7 +8,12 @@ use crate::utils::default_bool;
 
 #[derive(Clone, Default, Debug, Deserialize, Serialize)]
 pub struct LogsSettings {
+    /// Whether to export OpenTelemetry traces
+    #[serde(default = "default_bool::<false>")]
+    pub export_traces: bool,
+    #[serde(default)]
     pub stdout: StdoutLogSettings,
+    #[serde(default)]
     pub file: FileLogSettings,
 }
 

crates/common/src/config/log.rs

@@ -1,4 +1,4 @@
-use std::path::PathBuf;
+use std::{collections::HashMap, path::PathBuf};
 
 use eyre::Result;
 use serde::{Deserialize, Serialize};
@@ -35,6 +35,8 @@ pub struct CommitBoostConfig {
     pub metrics: Option<MetricsConfig>,
     #[serde(default)]
     pub logs: LogsSettings,
+    #[serde(default)]
+    pub env: HashMap<String, String>,
 }
 
 impl CommitBoostConfig {
@@ -81,6 +83,7 @@ impl CommitBoostConfig {
             signer: helper_config.signer,
             metrics: helper_config.metrics,
             logs: helper_config.logs,
+            env: helper_config.env,
         };
 
         Ok(config)
@@ -120,4 +123,6 @@ struct HelperConfig {
     metrics: Option<MetricsConfig>,
     #[serde(default)]
     logs: LogsSettings,
+    #[serde(default)]
+    env: HashMap<String, String>,
 }

crates/common/src/config/mod.rs

@@ -9,13 +9,16 @@ use alloy::{
 };
 use axum::http::HeaderValue;
 use blst::min_pk::{PublicKey, Signature};
+use opentelemetry::trace::TracerProvider as _;
+use opentelemetry_otlp::OTEL_EXPORTER_OTLP_PROTOCOL;
 use rand::{distr::Alphanumeric, Rng};
 use reqwest::header::HeaderMap;
 use serde::{de::DeserializeOwned, Serialize};
 use serde_json::Value;
 use ssz::{Decode, Encode};
 use tracing::Level;
 use tracing_appender::{non_blocking::WorkerGuard, rolling::Rotation};
+use tracing_opentelemetry::OpenTelemetryLayer;
 use tracing_subscriber::{
     fmt::{format::Format, Layer},
     prelude::*,
@@ -235,6 +238,32 @@ pub fn initialize_tracing_log(
         }
     };
 
+    if settings.export_traces {
+        // grpc by default
+        let exporter = if let Ok(protocol) = std::env::var(OTEL_EXPORTER_OTLP_PROTOCOL) {
+            if protocol.contains("http") {
+                opentelemetry_otlp::SpanExporter::builder().with_http().build()?
+            } else {
+                opentelemetry_otlp::SpanExporter::builder().with_tonic().build()?
+            }
+        } else {
+            opentelemetry_otlp::SpanExporter::builder().with_tonic().build()?
+        };
+
+        let tracer = opentelemetry_sdk::trace::SdkTracerProvider::builder()
+            .with_batch_exporter(exporter)
+            .build()
+            .tracer("commit_boost");
+
+        let layer = OpenTelemetryLayer::new(tracer)
+            .with_tracked_inactivity(false)
+            .with_threads(false)
+            .with_filter(format_crates_filter("info", "trace"))
+            .boxed();
+
+        layers.push(layer);
+    }
+
     tracing_subscriber::registry().with(layers).init();
 
     Ok((stdout_guard, file_guard))

crates/common/src/utils.rs

@@ -42,7 +42,7 @@ pub fn create_app_router<S: BuilderApiState, A: BuilderApi<S>>(state: PbsStateGu
 }
 
 #[tracing::instrument(
-    name = "", 
+    name = "request", 
     skip_all,
     fields(
         method = %req.extensions().get::<MatchedPath>().map(|m| m.as_str()).unwrap_or("unknown"),