Skip to content

Commit 16d3541

Browse files
itowlsonitowlson
committed
Upgrade OTel to stop Tokio being pinned
Signed-off-by: itowlson <[email protected]>
1 parent 2256bef commit 16d3541

File tree

7 files changed

+110
-82
lines changed

7 files changed

+110
-82
lines changed

‎Cargo.lock

Lines changed: 27 additions & 12 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -127,8 +127,8 @@ http = "1"
127127
http-body-util = "0.1"
128128
hyper = { version = "1", features = ["full"] }
129129
itertools = "0.13"
130-
opentelemetry = { version = "0.25", features = ["metrics", "trace", "logs"] }
131-
opentelemetry_sdk = { version = "0.25", features = ["rt-tokio", "logs_level_enabled", "metrics"] }
130+
opentelemetry = { version = "0.27", features = ["metrics", "trace", "logs"] }
131+
opentelemetry_sdk = { version = "0.27", features = ["rt-tokio", "spec_unstable_logs_enabled", "metrics"] }
132132
rand = "0.8"
133133
regex = "1"
134134
reqwest = { version = "0.12", features = ["stream", "blocking"] }
@@ -144,7 +144,7 @@ thiserror = "1"
144144
tokio = "1"
145145
toml = "0.8"
146146
tracing = { version = "0.1", features = ["log"] }
147-
tracing-opentelemetry = { version = "0.26", default-features = false, features = ["metrics"] }
147+
tracing-opentelemetry = { version = "0.28", default-features = false, features = ["metrics"] }
148148
url = "2"
149149
wasi-common-preview1 = { version = "25.0.0", package = "wasi-common", features = [
150150
"tokio",

‎crates/telemetry/Cargo.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ anyhow = { workspace = true }
99
http0 = { version = "0.2.9", package = "http" }
1010
http1 = { version = "1.0.0", package = "http" }
1111
opentelemetry = { workspace = true }
12-
opentelemetry-otlp = { version = "0.25", features = ["http-proto", "http", "reqwest-client"] }
12+
opentelemetry-appender-tracing = "0.27"
13+
opentelemetry-otlp = { version = "0.27", features = ["http-proto", "http", "reqwest-client"] }
1314
opentelemetry_sdk = { workspace = true }
1415
terminal = { path = "../terminal" }
1516
tracing = { workspace = true }

‎crates/telemetry/src/lib.rs

Lines changed: 43 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
1-
use std::cell::Cell;
21
use std::io::IsTerminal;
3-
use std::time::Duration;
4-
use std::time::Instant;
52

63
use env::otel_logs_enabled;
74
use env::otel_metrics_enabled;
85
use env::otel_tracing_enabled;
96
use opentelemetry_sdk::propagation::TraceContextPropagator;
7+
use tracing_subscriber::filter::filter_fn;
8+
use tracing_subscriber::fmt::Layer as OtelFmtLayer;
109
use tracing_subscriber::{fmt, prelude::*, registry, EnvFilter, Layer};
1110

1211
pub mod detector;
@@ -67,8 +66,15 @@ pub fn init(spin_version: String) -> anyhow::Result<ShutdownGuard> {
6766
.add_directive("[{app_log_non_utf8}]=off".parse()?),
6867
);
6968

70-
// Even if metrics or tracing aren't enabled we're okay to turn on the global error handler
71-
opentelemetry::global::set_error_handler(otel_error_handler)?;
69+
let opentelemetry_layer = OtelFmtLayer::new()
70+
.with_writer(std::io::stderr)
71+
.with_filter(filter_fn(|metadata| {
72+
metadata.target().starts_with("opentelemetry")
73+
}));
74+
75+
// let non_opentelemetry_filter = filter_fn(|metadata| !metadata.target().starts_with("opentelemetry"));
76+
// let otel_bridge_layer = layer::OpenTelemetryTracingBridge::new(&cloned_provider)
77+
// .with_filter(non_opentelemetry_filter);
7278

7379
let otel_tracing_layer = if otel_tracing_enabled() {
7480
Some(traces::otel_tracing_layer(spin_version.clone())?)
@@ -87,6 +93,8 @@ pub fn init(spin_version: String) -> anyhow::Result<ShutdownGuard> {
8793
.with(otel_tracing_layer)
8894
.with(otel_metrics_layer)
8995
.with(fmt_layer)
96+
.with(opentelemetry_layer)
97+
// .with(otel_bridge_layer)
9098
.init();
9199

92100
// Used to propagate trace information in the standard W3C TraceContext format. Even if the otel
@@ -100,36 +108,36 @@ pub fn init(spin_version: String) -> anyhow::Result<ShutdownGuard> {
100108
Ok(ShutdownGuard)
101109
}
102110

103-
fn otel_error_handler(err: opentelemetry::global::Error) {
104-
// Track the error count
105-
let signal = match err {
106-
opentelemetry::global::Error::Metric(_) => "metrics",
107-
opentelemetry::global::Error::Trace(_) => "traces",
108-
opentelemetry::global::Error::Log(_) => "logs",
109-
_ => "unknown",
110-
};
111-
metrics::monotonic_counter!(spin.otel_error_count = 1, signal = signal);
112-
113-
// Only log the first error at ERROR level, subsequent errors will be logged at higher levels and rate limited
114-
static FIRST_OTEL_ERROR: std::sync::Once = std::sync::Once::new();
115-
FIRST_OTEL_ERROR.call_once(|| {
116-
tracing::error!(?err, "OpenTelemetry error");
117-
tracing::error!("There has been an error with the OpenTelemetry system. Traces, logs or metrics are likely failing to export.");
118-
tracing::error!("Further OpenTelemetry errors will be available at WARN level (rate limited) or at TRACE level.");
119-
});
120-
121-
// Rate limit the logging of the OTel errors to not occur more frequently on each thread than OTEL_ERROR_INTERVAL
122-
const OTEL_ERROR_INTERVAL: Duration = Duration::from_millis(5000);
123-
thread_local! {
124-
static LAST_OTEL_ERROR: Cell<Instant> = Cell::new(Instant::now() - OTEL_ERROR_INTERVAL);
125-
}
126-
if LAST_OTEL_ERROR.get().elapsed() > OTEL_ERROR_INTERVAL {
127-
LAST_OTEL_ERROR.set(Instant::now());
128-
tracing::warn!(?err, "OpenTelemetry error");
129-
} else {
130-
tracing::trace!(?err, "OpenTelemetry error");
131-
}
132-
}
111+
// fn otel_error_handler(err: opentelemetry::global::Error) {
112+
// // Track the error count
113+
// let signal = match err {
114+
// opentelemetry::global::Error::Metric(_) => "metrics",
115+
// opentelemetry::global::Error::Trace(_) => "traces",
116+
// opentelemetry::global::Error::Log(_) => "logs",
117+
// _ => "unknown",
118+
// };
119+
// metrics::monotonic_counter!(spin.otel_error_count = 1, signal = signal);
120+
121+
// // Only log the first error at ERROR level, subsequent errors will be logged at higher levels and rate limited
122+
// static FIRST_OTEL_ERROR: std::sync::Once = std::sync::Once::new();
123+
// FIRST_OTEL_ERROR.call_once(|| {
124+
// tracing::error!(?err, "OpenTelemetry error");
125+
// tracing::error!("There has been an error with the OpenTelemetry system. Traces, logs or metrics are likely failing to export.");
126+
// tracing::error!("Further OpenTelemetry errors will be available at WARN level (rate limited) or at TRACE level.");
127+
// });
128+
129+
// // Rate limit the logging of the OTel errors to not occur more frequently on each thread than OTEL_ERROR_INTERVAL
130+
// const OTEL_ERROR_INTERVAL: Duration = Duration::from_millis(5000);
131+
// thread_local! {
132+
// static LAST_OTEL_ERROR: Cell<Instant> = Cell::new(Instant::now() - OTEL_ERROR_INTERVAL);
133+
// }
134+
// if LAST_OTEL_ERROR.get().elapsed() > OTEL_ERROR_INTERVAL {
135+
// LAST_OTEL_ERROR.set(Instant::now());
136+
// tracing::warn!(?err, "OpenTelemetry error");
137+
// } else {
138+
// tracing::trace!(?err, "OpenTelemetry error");
139+
// }
140+
// }
133141

134142
/// An RAII implementation for connection to open telemetry services.
135143
///

‎crates/telemetry/src/logs.rs

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ use std::{ascii::escape_default, sync::OnceLock, time::Duration};
22

33
use anyhow::bail;
44
use opentelemetry::logs::{LogRecord, Logger, LoggerProvider};
5-
use opentelemetry_otlp::LogExporterBuilder;
65
use opentelemetry_sdk::{
76
logs::{BatchConfigBuilder, BatchLogProcessor, Logger as SdkLogger},
87
resource::{EnvResourceDetector, TelemetryResourceDetector},
@@ -86,21 +85,22 @@ pub(crate) fn init_otel_logging_backend(spin_version: String) -> anyhow::Result<
8685
// currently default to using the HTTP exporter but in the future we could select off of the
8786
// combination of OTEL_EXPORTER_OTLP_PROTOCOL and OTEL_EXPORTER_OTLP_LOGS_PROTOCOL to
8887
// determine whether we should use http/protobuf or grpc.
89-
let exporter_builder: LogExporterBuilder = match OtlpProtocol::logs_protocol_from_env() {
90-
OtlpProtocol::Grpc => opentelemetry_otlp::new_exporter().tonic().into(),
91-
OtlpProtocol::HttpProtobuf => opentelemetry_otlp::new_exporter().http().into(),
88+
let exporter = match OtlpProtocol::logs_protocol_from_env() {
89+
OtlpProtocol::Grpc => opentelemetry_otlp::LogExporter::builder()
90+
.with_tonic()
91+
.build()?,
92+
OtlpProtocol::HttpProtobuf => opentelemetry_otlp::LogExporter::builder()
93+
.with_http()
94+
.build()?,
9295
OtlpProtocol::HttpJson => bail!("http/json OTLP protocol is not supported"),
9396
};
9497

9598
let provider = opentelemetry_sdk::logs::LoggerProvider::builder()
9699
.with_resource(resource)
97100
.with_log_processor(
98-
BatchLogProcessor::builder(
99-
exporter_builder.build_log_exporter()?,
100-
opentelemetry_sdk::runtime::Tokio,
101-
)
102-
.with_batch_config(BatchConfigBuilder::default().build())
103-
.build(),
101+
BatchLogProcessor::builder(exporter, opentelemetry_sdk::runtime::Tokio)
102+
.with_batch_config(BatchConfigBuilder::default().build())
103+
.build(),
104104
)
105105
.build();
106106

‎crates/telemetry/src/metrics.rs

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,8 @@ use std::time::Duration;
22

33
use anyhow::{bail, Result};
44
use opentelemetry::global;
5-
use opentelemetry_otlp::MetricsExporterBuilder;
65
use opentelemetry_sdk::{
7-
metrics::{
8-
reader::{DefaultAggregationSelector, DefaultTemporalitySelector},
9-
PeriodicReader, SdkMeterProvider,
10-
},
6+
metrics::{PeriodicReader, SdkMeterProvider},
117
resource::{EnvResourceDetector, TelemetryResourceDetector},
128
runtime, Resource,
139
};
@@ -42,15 +38,15 @@ pub(crate) fn otel_metrics_layer<S: Subscriber + for<'span> LookupSpan<'span>>(
4238
// currently default to using the HTTP exporter but in the future we could select off of the
4339
// combination of OTEL_EXPORTER_OTLP_PROTOCOL and OTEL_EXPORTER_OTLP_TRACES_PROTOCOL to
4440
// determine whether we should use http/protobuf or grpc.
45-
let exporter_builder: MetricsExporterBuilder = match OtlpProtocol::metrics_protocol_from_env() {
46-
OtlpProtocol::Grpc => opentelemetry_otlp::new_exporter().tonic().into(),
47-
OtlpProtocol::HttpProtobuf => opentelemetry_otlp::new_exporter().http().into(),
41+
let exporter = match OtlpProtocol::metrics_protocol_from_env() {
42+
OtlpProtocol::Grpc => opentelemetry_otlp::MetricExporter::builder()
43+
.with_tonic()
44+
.build()?,
45+
OtlpProtocol::HttpProtobuf => opentelemetry_otlp::MetricExporter::builder()
46+
.with_http()
47+
.build()?,
4848
OtlpProtocol::HttpJson => bail!("http/json OTLP protocol is not supported"),
4949
};
50-
let exporter = exporter_builder.build_metrics_exporter(
51-
Box::new(DefaultTemporalitySelector::new()),
52-
Box::new(DefaultAggregationSelector::new()),
53-
)?;
5450

5551
let reader = PeriodicReader::builder(exporter, runtime::Tokio).build();
5652
let meter_provider = SdkMeterProvider::builder()

‎crates/telemetry/src/traces.rs

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ use std::time::Duration;
22

33
use anyhow::bail;
44
use opentelemetry::{global, trace::TracerProvider};
5-
use opentelemetry_otlp::SpanExporterBuilder;
65
use opentelemetry_sdk::{
76
resource::{EnvResourceDetector, TelemetryResourceDetector},
87
Resource,
@@ -35,17 +34,26 @@ pub(crate) fn otel_tracing_layer<S: Subscriber + for<'span> LookupSpan<'span>>(
3534
);
3635

3736
// This will configure the exporter based on the OTEL_EXPORTER_* environment variables.
38-
let exporter_builder: SpanExporterBuilder = match OtlpProtocol::traces_protocol_from_env() {
39-
OtlpProtocol::Grpc => opentelemetry_otlp::new_exporter().tonic().into(),
40-
OtlpProtocol::HttpProtobuf => opentelemetry_otlp::new_exporter().http().into(),
37+
let exporter = match OtlpProtocol::traces_protocol_from_env() {
38+
OtlpProtocol::Grpc => opentelemetry_otlp::SpanExporter::builder()
39+
.with_tonic()
40+
.build()?,
41+
OtlpProtocol::HttpProtobuf => opentelemetry_otlp::SpanExporter::builder()
42+
.with_http()
43+
.build()?,
4144
OtlpProtocol::HttpJson => bail!("http/json OTLP protocol is not supported"),
4245
};
4346

44-
let tracer_provider = opentelemetry_otlp::new_pipeline()
45-
.tracing()
46-
.with_exporter(exporter_builder)
47-
.with_trace_config(opentelemetry_sdk::trace::Config::default().with_resource(resource))
48-
.install_batch(opentelemetry_sdk::runtime::Tokio)?;
47+
let span_processor = opentelemetry_sdk::trace::BatchSpanProcessor::builder(
48+
exporter,
49+
opentelemetry_sdk::runtime::Tokio,
50+
)
51+
.build();
52+
53+
let tracer_provider = opentelemetry_sdk::trace::TracerProvider::builder()
54+
.with_config(opentelemetry_sdk::trace::Config::default().with_resource(resource))
55+
.with_span_processor(span_processor)
56+
.build();
4957

5058
global::set_tracer_provider(tracer_provider.clone());
5159

0 commit comments

Comments
 (0)