Improve OTel error logging

Signed-off-by: Caleb Schoepp <[email protected]>

Author: calebschoepp

crates/telemetry/Cargo.toml

@@ -8,7 +8,7 @@ edition = { workspace = true }
 anyhow = { workspace = true }
 http0 = { version = "0.2.9", package = "http" }
 http1 = { version = "1.0.0", package = "http" }
-opentelemetry = { version = "0.22.0", features = [ "metrics", "trace"] }
+opentelemetry = { version = "0.22.0", features = [ "metrics", "trace", "logs"] }
 opentelemetry_sdk = { version = "0.22.1", features = ["rt-tokio", "logs_level_enabled"] }
 opentelemetry-otlp = { version = "0.15.0", default-features=false, features = ["http-proto", "trace", "http", "reqwest-client", "metrics", "grpc-tonic", "logs"] }
 opentelemetry-semantic-conventions = "0.14.0"

crates/telemetry/src/lib.rs

@@ -1,4 +1,7 @@
+use std::cell::Cell;
 use std::io::IsTerminal;
+use std::time::Duration;
+use std::time::Instant;
 
 use env::otel_logs_enabled;
 use env::otel_metrics_enabled;
@@ -98,12 +101,34 @@ pub fn init(spin_version: String) -> anyhow::Result<ShutdownGuard> {
 }
 
 fn otel_error_handler(err: opentelemetry::global::Error) {
+    // Track the error count
+    let signal = match err {
+        opentelemetry::global::Error::Metric(_) => "metrics",
+        opentelemetry::global::Error::Trace(_) => "traces",
+        opentelemetry::global::Error::Log(_) => "logs",
+        _ => "unknown",
+    };
+    metrics::monotonic_counter!(spin.otel_error_count = 1, signal = signal);
+
+    // Only log the first error at ERROR level, subsequent errors will be logged at higher levels and rate limited
     static FIRST_OTEL_ERROR: std::sync::Once = std::sync::Once::new();
     FIRST_OTEL_ERROR.call_once(|| {
-        tracing::error!("There has been an error with the OpenTelemetry system, traces and metrics are likely failing to export");
-        tracing::error!("Further OpenTelemetry errors will be logged at DEBUG level")
+        tracing::error!(?err, "OpenTelemetry error");
+        tracing::error!("There has been an error with the OpenTelemetry system. Traces, logs or metrics are likely failing to export.");
+        tracing::error!("Further OpenTelemetry errors will be available at WARN level (rate limited) or at TRACE level.");
     });
-    tracing::debug!(?err, "OpenTelemetry error");
+
+    // Rate limit the logging of the OTel errors to not occur more frequently on each thread than OTEL_ERROR_INTERVAL
+    const OTEL_ERROR_INTERVAL: Duration = Duration::from_millis(5000);
+    thread_local! {
+        static LAST_OTEL_ERROR: Cell<Instant> = Cell::new(Instant::now() - OTEL_ERROR_INTERVAL);
+    }
+    if LAST_OTEL_ERROR.get().elapsed() > OTEL_ERROR_INTERVAL {
+        LAST_OTEL_ERROR.set(Instant::now());
+        tracing::warn!(?err, "OpenTelemetry error");
+    } else {
+        tracing::trace!(?err, "OpenTelemetry error");
+    }
 }
 
 /// An RAII implementation for connection to open telemetry services.