add error handling, following instructions: open-telemetry/opentelemetry-rust#2260

Author: meryl-c

Cargo.lock

@@ -352,7 +352,8 @@ dependencies = [
  "num-traits 0.2.19",
  "oci-client",
  "once_cell",
- "opentelemetry",
+ "opentelemetry 0.30.0",
+ "opentelemetry-appender-tracing",
  "opentelemetry-aws",
  "opentelemetry-datadog",
  "opentelemetry-http",
@@ -4519,6 +4520,20 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e"
 
+[[package]]
+name = "opentelemetry"
+version = "0.27.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab70038c28ed37b97d8ed414b6429d343a8bbf44c9f79ec854f3a643029ba6d7"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+ "js-sys",
+ "pin-project-lite",
+ "thiserror 1.0.69",
+ "tracing",
+]
+
 [[package]]
 name = "opentelemetry"
 version = "0.30.0"
@@ -4533,13 +4548,25 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "opentelemetry-appender-tracing"
+version = "0.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab5feffc321035ad94088a7e5333abb4d84a8726e54a802e736ce9dd7237e85b"
+dependencies = [
+ "opentelemetry 0.27.1",
+ "tracing",
+ "tracing-core",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "opentelemetry-aws"
 version = "0.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6f3632d7494b282abac9039ef96e8619d86e8a3d0650dc6b9072920d25e2310a"
 dependencies = [
- "opentelemetry",
+ "opentelemetry 0.30.0",
  "opentelemetry_sdk",
  "tracing",
 ]
@@ -4554,7 +4581,7 @@ dependencies = [
  "http 1.3.1",
  "indexmap 2.11.0",
  "itoa",
- "opentelemetry",
+ "opentelemetry 0.30.0",
  "opentelemetry-http",
  "opentelemetry-semantic-conventions",
  "opentelemetry_sdk",
@@ -4574,7 +4601,7 @@ dependencies = [
  "async-trait",
  "bytes",
  "http 1.3.1",
- "opentelemetry",
+ "opentelemetry 0.30.0",
  "reqwest",
 ]
 
@@ -4584,7 +4611,7 @@ version = "0.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "090b8ec07bb2e304b529581aa1fe530d7861298c9ef549ebbf44a4a56472c539"
 dependencies = [
- "opentelemetry",
+ "opentelemetry 0.30.0",
 ]
 
 [[package]]
@@ -4594,7 +4621,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dbee664a43e07615731afc539ca60c6d9f1a9425e25ca09c57bc36c87c55852b"
 dependencies = [
  "http 1.3.1",
- "opentelemetry",
+ "opentelemetry 0.30.0",
  "opentelemetry-http",
  "opentelemetry-proto",
  "opentelemetry_sdk",
@@ -4611,7 +4638,7 @@ version = "0.29.1"
 source = "git+https://github.com/sandhose/opentelemetry-rust.git?branch=otel-prometheus-0.30#a60122ce4d534631b70222d84a19bfcb8e9d69a2"
 dependencies = [
  "once_cell",
- "opentelemetry",
+ "opentelemetry 0.30.0",
  "opentelemetry_sdk",
  "prometheus",
  "tracing",
@@ -4625,7 +4652,7 @@ checksum = "2e046fd7660710fe5a05e8748e70d9058dc15c94ba914e7c4faa7c728f0e8ddc"
 dependencies = [
  "base64 0.22.1",
  "hex",
- "opentelemetry",
+ "opentelemetry 0.30.0",
  "opentelemetry_sdk",
  "prost",
  "serde",
@@ -4645,7 +4672,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "447191061af41c3943e082ea359ab8b64ff27d6d34d30d327df309ddef1eef6f"
 dependencies = [
  "chrono",
- "opentelemetry",
+ "opentelemetry 0.30.0",
  "opentelemetry_sdk",
 ]
 
@@ -4657,7 +4684,7 @@ checksum = "8823568a9abd4003d1acc564bcfec8e3406a8c73203f8f50bdfc88b44d7ecfb6"
 dependencies = [
  "http 1.3.1",
  "once_cell",
- "opentelemetry",
+ "opentelemetry 0.30.0",
  "opentelemetry-http",
  "opentelemetry_sdk",
  "reqwest",
@@ -4676,7 +4703,7 @@ dependencies = [
  "futures-channel",
  "futures-executor",
  "futures-util",
- "opentelemetry",
+ "opentelemetry 0.30.0",
  "percent-encoding",
  "rand 0.9.2",
  "serde_json",
@@ -7264,7 +7291,7 @@ checksum = "ddcf5959f39507d0d04d6413119c04f33b623f4f951ebcbdddddfad2d0623a9c"
 dependencies = [
  "js-sys",
  "once_cell",
- "opentelemetry",
+ "opentelemetry 0.30.0",
  "opentelemetry_sdk",
  "smallvec",
  "tracing",

apollo-router/Cargo.toml

@@ -160,6 +160,7 @@ opentelemetry_sdk = { version = "0.30.0", default-features = false, features = [
     "experimental_metrics_custom_reader",
     "experimental_metrics_periodicreader_with_async_runtime",
 ] }
+opentelemetry-appender-tracing = "0.27.0"
 opentelemetry-aws = "0.18.0"
 opentelemetry-datadog = { version = "0.18.0", features = ["agent-sampling", "reqwest-client"] }
 rmp = "0.8"

apollo-router/src/plugins/telemetry/mod.rs

@@ -26,6 +26,7 @@ use http::header;
 use metrics::apollo::studio::SingleLimitsStats;
 use metrics::local_type_stats::LocalTypeStatRecorder;
 use multimap::MultiMap;
+use opentelemetry::InstrumentationScope;
 use opentelemetry::Key;
 use opentelemetry::KeyValue;
 use opentelemetry::global::GlobalTracerProvider;
@@ -56,6 +57,9 @@ use tokio::runtime::Handle;
 use tower::BoxError;
 use tower::ServiceBuilder;
 use tower::ServiceExt;
+use tracing_subscriber::filter::filter_fn;
+use tracing_subscriber::fmt::Layer;
+use tracing_subscriber::prelude::*;
 use uuid::Uuid;
 
 use self::apollo::ForwardValues;
@@ -365,7 +369,6 @@ impl PluginPrivate for Telemetry {
     type Config = config::Conf;
 
     async fn new(init: PluginInit<Self::Config>) -> Result<Self, BoxError> {
-
         let mut config = init.config;
         config.instrumentation.spans.update_defaults();
         config.instrumentation.instruments.update_defaults();
@@ -380,6 +383,19 @@ impl PluginPrivate for Telemetry {
         let metrics_builder = Self::create_metrics_builder(&config)?;
         let tracer_provider = Self::create_tracer_provider(&config)?;
 
+        // handle OpenTelemetry internal logs and stop them from propagating to tracing bridge/appender
+        // capture logs with the opentelemetry prefix and print them to eprintln!
+        let opentelemetry_layer = Layer::new()
+            .with_writer(std::io::stderr)
+            .with_filter(filter_fn(|metadata| {
+                metadata.target().starts_with("opentelemetry")
+            }));
+
+        // register layer in tracing registry
+        tracing_subscriber::registry()
+            .with(opentelemetry_layer) // OpenTelemetry internal logs (filtered out)
+            .init();
+
         if config.instrumentation.spans.mode == SpanMode::Deprecated {
             ::tracing::warn!(
                 "telemetry.instrumentation.spans.mode is currently set to 'deprecated', either explicitly or via defaulting. Set telemetry.instrumentation.spans.mode explicitly in your router.yaml to 'spec_compliant' for log and span attributes that follow OpenTelemetry semantic conventions. This option will be defaulted to 'spec_compliant' in a future release and eventually removed altogether"
@@ -1171,8 +1187,10 @@ impl PluginPrivate for Telemetry {
                 .take()
                 .expect("must have new tracer_provider");
 
-            let tracer = tracer_provider
-                .tracer(GLOBAL_TRACER_NAME);
+            let scope = InstrumentationScope::builder(GLOBAL_TRACER_NAME)
+                .with_version(env!("CARGO_PKG_VERSION"))
+                .build();
+            let tracer = tracer_provider.tracer_with_scope(scope);
             hot_tracer.reload(tracer);
 
             let last_provider = opentelemetry::global::set_tracer_provider(tracer_provider);