diff --git a/sentry_sdk/integrations/opentelemetry/potel_span_processor.py b/sentry_sdk/integrations/opentelemetry/potel_span_processor.py index fb20c7abfe..383234cba4 100644 --- a/sentry_sdk/integrations/opentelemetry/potel_span_processor.py +++ b/sentry_sdk/integrations/opentelemetry/potel_span_processor.py @@ -62,11 +62,12 @@ def on_end(self, span): if is_sentry_span(span): return - if span.parent and not span.parent.is_remote: - self._children_spans[span.parent.span_id].append(span) - else: + is_root_span = not span.parent or span.parent.is_remote + if is_root_span: # if have a root span ending, we build a transaction and send it self._flush_root_span(span) + else: + self._children_spans[span.parent.span_id].append(span) # TODO-neel-potel not sure we need a clear like JS def shutdown(self): diff --git a/sentry_sdk/integrations/opentelemetry/sampler.py b/sentry_sdk/integrations/opentelemetry/sampler.py index 8ffad41b86..b5c500b3f3 100644 --- a/sentry_sdk/integrations/opentelemetry/sampler.py +++ b/sentry_sdk/integrations/opentelemetry/sampler.py @@ -1,5 +1,5 @@ +import random from typing import cast -from random import random from opentelemetry import trace @@ -46,10 +46,10 @@ def get_parent_sampled(parent_context, trace_id): return None -def dropped_result(span_context, attributes, sample_rate=None): +def dropped_result(parent_span_context, attributes, sample_rate=None): # type: (SpanContext, Attributes, Optional[float]) -> SamplingResult # these will only be added the first time in a root span sampling decision - trace_state = span_context.trace_state + trace_state = parent_span_context.trace_state if TRACESTATE_SAMPLED_KEY not in trace_state: trace_state = trace_state.add(TRACESTATE_SAMPLED_KEY, "false") @@ -57,6 +57,22 @@ def dropped_result(span_context, attributes, sample_rate=None): if sample_rate and TRACESTATE_SAMPLE_RATE_KEY not in trace_state: trace_state = trace_state.add(TRACESTATE_SAMPLE_RATE_KEY, str(sample_rate)) + is_root_span = not ( + parent_span_context.is_valid and not parent_span_context.is_remote + ) + if is_root_span: + # Tell Sentry why we dropped the transaction/root-span + client = sentry_sdk.get_client() + if client.monitor and client.monitor.downsample_factor > 0: + reason = "backpressure" + else: + reason = "sample_rate" + + client.transport.record_lost_event(reason, data_category="transaction") + + # Only one span (the transaction itself) is discarded, since we did not record any spans here. + client.transport.record_lost_event(reason, data_category="span") + return SamplingResult( Decision.DROP, attributes=attributes, @@ -136,9 +152,14 @@ def should_sample( ) return dropped_result(parent_span_context, attributes) + # Down-sample in case of back pressure monitor says so + # TODO: this should only be done for transactions (aka root spans) + if client.monitor: + sample_rate /= 2**client.monitor.downsample_factor + # Roll the dice on sample rate sample_rate = float(cast("Union[bool, float, int]", sample_rate)) - sampled = random() < sample_rate + sampled = random.random() < sample_rate if sampled: return sampled_result(parent_span_context, attributes, sample_rate) diff --git a/tests/test_monitor.py b/tests/test_monitor.py index 03e415b5cc..041169d515 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -55,14 +55,16 @@ def test_monitor_unhealthy(sentry_init): assert monitor.downsample_factor == (i + 1 if i < 10 else 10) -def test_transaction_uses_downsampled_rate( - sentry_init, capture_record_lost_event_calls, monkeypatch +def test_transaction_uses_downsample_rate( + sentry_init, capture_envelopes, capture_record_lost_event_calls, monkeypatch ): sentry_init( traces_sample_rate=1.0, transport=UnhealthyTestTransport(), ) + envelopes = capture_envelopes() + record_lost_event_calls = capture_record_lost_event_calls() monitor = sentry_sdk.get_client().monitor @@ -77,13 +79,32 @@ def test_transaction_uses_downsampled_rate( assert monitor.downsample_factor == 1 with sentry_sdk.start_transaction(name="foobar") as transaction: + with sentry_sdk.start_span(name="foospan"): + with sentry_sdk.start_span(name="foospan2"): + with sentry_sdk.start_span(name="foospan3"): + ... + assert transaction.sampled is False - assert transaction.sample_rate == 0.5 + assert ( + transaction.sample_rate == 0.5 + ) # TODO: this fails until we put the sample_rate in the POTelSpan + + assert len(envelopes) == 0 assert Counter(record_lost_event_calls) == Counter( [ - ("backpressure", "transaction", None, 1), - ("backpressure", "span", None, 1), + ( + "backpressure", + "transaction", + None, + 1, + ), + ( + "backpressure", + "span", + None, + 1, + ), # Only one span (the transaction itself) is counted, since we did not record any spans in the first place. ] )