Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
use opentelemetry::trace::TraceError;
use std::time::SystemTime;

// leaky bucket based rate limit
Expand Down Expand Up @@ -53,10 +52,12 @@ impl LeakyBucket {
false
}
}
Err(_) => {
opentelemetry::global::handle_error(TraceError::Other(
"jaeger remote sampler gets rewinded timestamp".into(),
));
Err(err) => {
opentelemetry::otel_debug!(
name: "JaegerRemoteSampler.LeakyBucket.ClockAdjustment",
message = "jaeger remote sampler detected a rewind in system clock",
reason = format!("{:?}", err),
);
true
}
}
Expand Down
10 changes: 8 additions & 2 deletions opentelemetry-sdk/src/trace/sampler/jaeger_remote/sampler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use crate::trace::{Sampler, ShouldSample};
use futures_util::{stream, StreamExt as _};
use http::Uri;
use opentelemetry::trace::{Link, SamplingResult, SpanKind, TraceError, TraceId};
use opentelemetry::{global, Context, KeyValue};
use opentelemetry::{otel_error, Context, KeyValue};
use opentelemetry_http::HttpClient;
use std::str::FromStr;
use std::sync::Arc;
Expand Down Expand Up @@ -203,7 +203,13 @@ impl JaegerRemoteSampler {
// send request
match Self::request_new_strategy(&client, endpoint.clone()).await {
Ok(remote_strategy_resp) => strategy.update(remote_strategy_resp),
Err(err_msg) => global::handle_error(TraceError::Other(err_msg.into())),
Err(err_msg) => {
otel_warn!(
name: "JaegerRemoteSampler.UpdateStrategy.RequestFailed",
message = "Failed to fetch the new sampling strategy from remote endpoint. This may cause the sampler to use stale configuration until the next successful update.",
reason = format!("{}", err_msg),
);
}
};
} else {
// shutdown
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ use crate::trace::sampler::jaeger_remote::remote::{
};
use crate::trace::sampler::sample_based_on_probability;
use opentelemetry::trace::{
SamplingDecision, SamplingResult, TraceContextExt, TraceError, TraceId, TraceState,
SamplingDecision, SamplingResult, TraceContextExt, TraceId, TraceState,
};
use opentelemetry::{global, Context};
use opentelemetry::{otel_debug, otel_warn, Context};
use std::collections::HashMap;
use std::fmt::{Debug, Formatter};
use std::sync::Mutex;
Expand Down Expand Up @@ -107,9 +107,10 @@ impl Inner {
}
})
.unwrap_or_else(|_err| {
global::handle_error(TraceError::Other(
"jaeger remote sampler mutex poisoned".into(),
))
otel_debug!(
name: "JaegerRemoteSampler.MutexPoisoned",
message = "Unable to update Jaeger Remote sampling strategy: the sampler's internal mutex is poisoned, likely due to a panic in another thread holding the lock. This may result in using stale configuration until the remote sampling client is restarted.",
);
});
}

Expand Down Expand Up @@ -137,7 +138,15 @@ impl Inner {
(_, _, Some(probabilistic)) => {
Some(Strategy::Probabilistic(probabilistic.sampling_rate))
}
_ => None,
_ => {
otel_warn!(
name: "Sampler.JaegerRemote.InvalidStrategy",
message = "Received invalid sampling strategy from Jaeger remote endpoint. Expected one of: OperationSampling, RateLimitingSampling (max traces per second), or ProbabilisticSampling (0.0-1.0 sampling probability). No valid strategy was found in the response. Using previous strategy if available.",
received_operation_sampling = operation_sampling.is_some(),
received_rate_limiting = rate_limiting_sampling.is_some(),
received_probabilistic = probabilistic_sampling.is_some()
);
}
}
}

Expand Down
Loading