feat(tracing): Port sample_rand to POTel (#4106)

Port `sample_rand` to `potel-base`. See
[spec](https://develop.sentry.dev/sdk/telemetry/traces/#propagated-random-value).

There are now two places where a `sample_rand` might be generated:
- If we're explicitly propagating with `continue_trace`, we'll
[backfill](https://github.com/getsentry/sentry-python/pull/4106/files#diff-7c64294459f5053c93d44e0e33e4e73ffcef0adefcd77ba91f4031aa461a8c42R396-R397)
`sample_rand` on the propagation context like on master, either using
the incoming one or generating a new one from the incoming
`sampled`/`sample_rate`.
- Otherwise, we generate a new `sample_rand` [in the
Sampler](https://github.com/getsentry/sentry-python/pull/4106/files#diff-59aa7195d955e153b5cdd730f888994996a72eaf5e9ea174335ce961841584a9R194-R213).

The generated `sample_rand` is then saved on the trace state.

This change fixes most of the failures in the Common test suite.

Closes #4027

---------

Co-authored-by: Daniel Szoke <[email protected]>

Author: sentrivana

MIGRATION_GUIDE.md

@@ -157,6 +157,7 @@ Looking to upgrade from Sentry SDK 2.x to 3.x? Here's a comprehensive list of wh
 - `profiles_sample_rate` and `profiler_mode` were removed from options available via `_experiments`. Use the top-level `profiles_sample_rate` and `profiler_mode` options instead.
 - `Transport.capture_event` has been removed. Use `Transport.capture_envelope` instead.
 - Function transports are no longer supported. Subclass the `Transport` instead.
+- `start_transaction` (`start_span`) no longer takes a `baggage` argument. Use the `continue_trace()` context manager instead to propagate baggage.
 
 ### Deprecated
 

sentry_sdk/integrations/opentelemetry/consts.py

@@ -12,9 +12,12 @@
 SENTRY_USE_CURRENT_SCOPE_KEY = create_key("sentry_use_current_scope")
 SENTRY_USE_ISOLATION_SCOPE_KEY = create_key("sentry_use_isolation_scope")
 
+# trace state keys
 TRACESTATE_SAMPLED_KEY = Baggage.SENTRY_PREFIX + "sampled"
 TRACESTATE_SAMPLE_RATE_KEY = Baggage.SENTRY_PREFIX + "sample_rate"
+TRACESTATE_SAMPLE_RAND_KEY = Baggage.SENTRY_PREFIX + "sample_rand"
 
+# misc
 OTEL_SENTRY_CONTEXT = "otel"
 SPAN_ORIGIN = "auto.otel"
 

sentry_sdk/integrations/opentelemetry/sampler.py

@@ -1,15 +1,19 @@
-import random
+from decimal import Decimal
 from typing import cast
 
 from opentelemetry import trace
 from opentelemetry.sdk.trace.sampling import Sampler, SamplingResult, Decision
 from opentelemetry.trace.span import TraceState
 
 import sentry_sdk
-from sentry_sdk.tracing_utils import has_tracing_enabled
+from sentry_sdk.tracing_utils import (
+    _generate_sample_rand,
+    has_tracing_enabled,
+)
 from sentry_sdk.utils import is_valid_sample_rate, logger
 from sentry_sdk.integrations.opentelemetry.consts import (
     TRACESTATE_SAMPLED_KEY,
+    TRACESTATE_SAMPLE_RAND_KEY,
     TRACESTATE_SAMPLE_RATE_KEY,
     SentrySpanAttribute,
 )
@@ -70,23 +74,40 @@ def get_parent_sample_rate(parent_context, trace_id):
     return None
 
 
-def dropped_result(parent_span_context, attributes, sample_rate=None):
-    # type: (SpanContext, Attributes, Optional[float]) -> SamplingResult
-    # these will only be added the first time in a root span sampling decision
-    # if sample_rate is provided, it'll be updated in trace state
-    trace_state = parent_span_context.trace_state
+def get_parent_sample_rand(parent_context, trace_id):
+    # type: (Optional[SpanContext], int) -> Optional[Decimal]
+    if parent_context is None:
+        return None
 
-    if TRACESTATE_SAMPLED_KEY not in trace_state:
-        trace_state = trace_state.add(TRACESTATE_SAMPLED_KEY, "false")
-    elif trace_state.get(TRACESTATE_SAMPLED_KEY) == "deferred":
-        trace_state = trace_state.update(TRACESTATE_SAMPLED_KEY, "false")
+    is_span_context_valid = parent_context is not None and parent_context.is_valid
 
-    if sample_rate is not None:
-        trace_state = trace_state.update(TRACESTATE_SAMPLE_RATE_KEY, str(sample_rate))
+    if is_span_context_valid and parent_context.trace_id == trace_id:
+        parent_sample_rand = parent_context.trace_state.get(TRACESTATE_SAMPLE_RAND_KEY)
+        if parent_sample_rand is None:
+            return None
 
-    is_root_span = not (
-        parent_span_context.is_valid and not parent_span_context.is_remote
+        return Decimal(parent_sample_rand)
+
+    return None
+
+
+def dropped_result(span_context, attributes, sample_rate=None, sample_rand=None):
+    # type: (SpanContext, Attributes, Optional[float], Optional[Decimal]) -> SamplingResult
+    """
+    React to a span getting unsampled and return a DROP SamplingResult.
+
+    Update the trace_state with the effective sampled, sample_rate and sample_rand,
+    record that we dropped the event for client report purposes, and return
+    an OTel SamplingResult with Decision.DROP.
+
+    See for more info about OTel sampling:
+    https://opentelemetry-python.readthedocs.io/en/latest/sdk/trace.sampling.html
+    """
+    trace_state = _update_trace_state(
+        span_context, sampled=False, sample_rate=sample_rate, sample_rand=sample_rand
     )
+
+    is_root_span = not (span_context.is_valid and not span_context.is_remote)
     if is_root_span:
         # Tell Sentry why we dropped the transaction/root-span
         client = sentry_sdk.get_client()
@@ -108,19 +129,20 @@ def dropped_result(parent_span_context, attributes, sample_rate=None):
     )
 
 
-def sampled_result(span_context, attributes, sample_rate):
-    # type: (SpanContext, Attributes, Optional[float]) -> SamplingResult
-    # these will only be added the first time in a root span sampling decision
-    # if sample_rate is provided, it'll be updated in trace state
-    trace_state = span_context.trace_state
+def sampled_result(span_context, attributes, sample_rate=None, sample_rand=None):
+    # type: (SpanContext, Attributes, Optional[float], Optional[Decimal]) -> SamplingResult
+    """
+    React to a span being sampled and return a sampled SamplingResult.
 
-    if TRACESTATE_SAMPLED_KEY not in trace_state:
-        trace_state = trace_state.add(TRACESTATE_SAMPLED_KEY, "true")
-    elif trace_state.get(TRACESTATE_SAMPLED_KEY) == "deferred":
-        trace_state = trace_state.update(TRACESTATE_SAMPLED_KEY, "true")
+    Update the trace_state with the effective sampled, sample_rate and sample_rand,
+    and return an OTel SamplingResult with Decision.RECORD_AND_SAMPLE.
 
-    if sample_rate is not None:
-        trace_state = trace_state.update(TRACESTATE_SAMPLE_RATE_KEY, str(sample_rate))
+    See for more info about OTel sampling:
+    https://opentelemetry-python.readthedocs.io/en/latest/sdk/trace.sampling.html
+    """
+    trace_state = _update_trace_state(
+        span_context, sampled=True, sample_rate=sample_rate, sample_rand=sample_rand
+    )
 
     return SamplingResult(
         Decision.RECORD_AND_SAMPLE,
@@ -129,6 +151,27 @@ def sampled_result(span_context, attributes, sample_rate):
     )
 
 
+def _update_trace_state(span_context, sampled, sample_rate=None, sample_rand=None):
+    # type: (SpanContext, bool, Optional[float], Optional[Decimal]) -> TraceState
+    trace_state = span_context.trace_state
+
+    sampled = "true" if sampled else "false"
+    if TRACESTATE_SAMPLED_KEY not in trace_state:
+        trace_state = trace_state.add(TRACESTATE_SAMPLED_KEY, sampled)
+    elif trace_state.get(TRACESTATE_SAMPLED_KEY) == "deferred":
+        trace_state = trace_state.update(TRACESTATE_SAMPLED_KEY, sampled)
+
+    if sample_rate is not None:
+        trace_state = trace_state.update(TRACESTATE_SAMPLE_RATE_KEY, str(sample_rate))
+
+    if sample_rand is not None:
+        trace_state = trace_state.update(
+            TRACESTATE_SAMPLE_RAND_KEY, f"{sample_rand:.6f}"  # noqa: E231
+        )
+
+    return trace_state
+
+
 class SentrySampler(Sampler):
     def should_sample(
         self,
@@ -156,6 +199,18 @@ def should_sample(
 
         sample_rate = None
 
+        parent_sampled = get_parent_sampled(parent_span_context, trace_id)
+        parent_sample_rate = get_parent_sample_rate(parent_span_context, trace_id)
+        parent_sample_rand = get_parent_sample_rand(parent_span_context, trace_id)
+
+        if parent_sample_rand is not None:
+            # We have a sample_rand on the incoming trace or we already backfilled
+            # it in PropagationContext
+            sample_rand = parent_sample_rand
+        else:
+            # We are the head SDK and we need to generate a new sample_rand
+            sample_rand = cast(Decimal, _generate_sample_rand(str(trace_id), (0, 1)))
+
         # Explicit sampled value provided at start_span
         custom_sampled = cast(
             "Optional[bool]", attributes.get(SentrySpanAttribute.CUSTOM_SAMPLED)
@@ -165,11 +220,17 @@ def should_sample(
                 sample_rate = float(custom_sampled)
                 if sample_rate > 0:
                     return sampled_result(
-                        parent_span_context, attributes, sample_rate=sample_rate
+                        parent_span_context,
+                        attributes,
+                        sample_rate=sample_rate,
+                        sample_rand=sample_rand,
                     )
                 else:
                     return dropped_result(
-                        parent_span_context, attributes, sample_rate=sample_rate
+                        parent_span_context,
+                        attributes,
+                        sample_rate=sample_rate,
+                        sample_rand=sample_rand,
                     )
             else:
                 logger.debug(
@@ -190,8 +251,6 @@ def should_sample(
             sample_rate_to_propagate = sample_rate
         else:
             # Check if there is a parent with a sampling decision
-            parent_sampled = get_parent_sampled(parent_span_context, trace_id)
-            parent_sample_rate = get_parent_sample_rate(parent_span_context, trace_id)
             if parent_sampled is not None:
                 sample_rate = bool(parent_sampled)
                 sample_rate_to_propagate = (
@@ -215,17 +274,23 @@ def should_sample(
             if client.monitor.downsample_factor > 0:
                 sample_rate_to_propagate = sample_rate
 
-        # Roll the dice on sample rate
+        # Compare sample_rand to sample_rate to make the final sampling decision
         sample_rate = float(cast("Union[bool, float, int]", sample_rate))
-        sampled = random.random() < sample_rate
+        sampled = sample_rand < sample_rate
 
         if sampled:
             return sampled_result(
-                parent_span_context, attributes, sample_rate=sample_rate_to_propagate
+                parent_span_context,
+                attributes,
+                sample_rate=sample_rate_to_propagate,
+                sample_rand=None if sample_rand == parent_sample_rand else sample_rand,
             )
         else:
             return dropped_result(
-                parent_span_context, attributes, sample_rate=sample_rate_to_propagate
+                parent_span_context,
+                attributes,
+                sample_rate=sample_rate_to_propagate,
+                sample_rand=None if sample_rand == parent_sample_rand else sample_rand,
             )
 
     def get_description(self) -> str:

sentry_sdk/integrations/opentelemetry/scope.py

@@ -30,7 +30,6 @@
 from sentry_sdk.integrations.opentelemetry.utils import trace_state_from_baggage
 from sentry_sdk.scope import Scope, ScopeType
 from sentry_sdk.tracing import Span
-from sentry_sdk.utils import logger
 from sentry_sdk._types import TYPE_CHECKING
 
 if TYPE_CHECKING:

sentry_sdk/integrations/stdlib.py

@@ -73,7 +73,7 @@ def putrequest(self, method, url, *args, **kwargs):
 
         client = sentry_sdk.get_client()
         if client.get_integration(StdlibIntegration) is None or is_sentry_url(
-            client, f"{host}:{port}"
+            client, f"{host}:{port}"  # noqa: E231
         ):
             return real_putrequest(self, method, url, *args, **kwargs)
 

sentry_sdk/tracing.py

@@ -20,7 +20,6 @@
 from sentry_sdk.utils import (
     _serialize_span_attribute,
     get_current_thread_meta,
-    logger,
     should_be_treated_as_error,
 )
 

sentry_sdk/tracing_utils.py

@@ -1,4 +1,5 @@
 import contextlib
+import decimal
 import inspect
 import os
 import re
@@ -392,6 +393,9 @@ def from_incoming_data(cls, incoming_data):
                     propagation_context = PropagationContext()
                 propagation_context.update(sentrytrace_data)
 
+        if propagation_context is not None:
+            propagation_context._fill_sample_rand()
+
         return propagation_context
 
     @property
@@ -433,6 +437,78 @@ def update(self, other_dict):
             except AttributeError:
                 pass
 
+    def _fill_sample_rand(self):
+        # type: () -> None
+        """
+        Ensure that there is a valid sample_rand value in the baggage.
+
+        If there is a valid sample_rand value in the baggage, we keep it.
+        Otherwise, we generate a sample_rand value according to the following:
+
+          - If we have a parent_sampled value and a sample_rate in the DSC, we compute
+            a sample_rand value randomly in the range:
+                - [0, sample_rate) if parent_sampled is True,
+                - or, in the range [sample_rate, 1) if parent_sampled is False.
+
+          - If either parent_sampled or sample_rate is missing, we generate a random
+            value in the range [0, 1).
+
+        The sample_rand is deterministically generated from the trace_id, if present.
+
+        This function does nothing if there is no dynamic_sampling_context.
+        """
+        if self.dynamic_sampling_context is None or self.baggage is None:
+            return
+
+        sentry_baggage = self.baggage.sentry_items
+
+        sample_rand = None
+        if sentry_baggage.get("sample_rand"):
+            try:
+                sample_rand = Decimal(sentry_baggage["sample_rand"])
+            except Exception:
+                logger.debug(
+                    f"Failed to convert incoming sample_rand to Decimal: {sample_rand}"
+                )
+
+        if sample_rand is not None and 0 <= sample_rand < 1:
+            # sample_rand is present and valid, so don't overwrite it
+            return
+
+        sample_rate = None
+        if sentry_baggage.get("sample_rate"):
+            try:
+                sample_rate = float(sentry_baggage["sample_rate"])
+            except Exception:
+                logger.debug(
+                    f"Failed to convert incoming sample_rate to float: {sample_rate}"
+                )
+
+        lower, upper = _sample_rand_range(self.parent_sampled, sample_rate)
+
+        try:
+            sample_rand = _generate_sample_rand(self.trace_id, interval=(lower, upper))
+        except ValueError:
+            # ValueError is raised if the interval is invalid, i.e. lower >= upper.
+            # lower >= upper might happen if the incoming trace's sampled flag
+            # and sample_rate are inconsistent, e.g. sample_rate=0.0 but sampled=True.
+            # We cannot generate a sensible sample_rand value in this case.
+            logger.debug(
+                f"Could not backfill sample_rand, since parent_sampled={self.parent_sampled} "
+                f"and sample_rate={sample_rate}."
+            )
+            return
+
+        self.baggage.sentry_items["sample_rand"] = f"{sample_rand:.6f}"  # noqa: E231
+
+    def _sample_rand(self):
+        # type: () -> Optional[str]
+        """Convenience method to get the sample_rand value from the baggage."""
+        if self.baggage is None:
+            return None
+
+        return self.baggage.sentry_items.get("sample_rand")
+
     def __repr__(self):
         # type: (...) -> str
         return "<PropagationContext _trace_id={} _span_id={} parent_span_id={} parent_sampled={} baggage={} dynamic_sampling_context={}>".format(
@@ -684,13 +760,11 @@ def get_current_span(scope=None):
     return current_span
 
 
-# XXX-potel-ivana: use this
 def _generate_sample_rand(
     trace_id,  # type: Optional[str]
-    *,
     interval=(0.0, 1.0),  # type: tuple[float, float]
 ):
-    # type: (...) -> Any
+    # type: (...) -> Optional[decimal.Decimal]
     """Generate a sample_rand value from a trace ID.
 
     The generated value will be pseudorandomly chosen from the provided
@@ -709,15 +783,11 @@ def _generate_sample_rand(
     while sample_rand >= upper:
         sample_rand = rng.uniform(lower, upper)
 
-    # Round down to exactly six decimal-digit precision.
-    # Setting the context is needed to avoid an InvalidOperation exception
-    # in case the user has changed the default precision.
     return Decimal(sample_rand).quantize(
         Decimal("0.000001"), rounding=ROUND_DOWN, context=Context(prec=6)
     )
 
 
-# XXX-potel-ivana: use this
 def _sample_rand_range(parent_sampled, sample_rate):
     # type: (Optional[bool], Optional[float]) -> tuple[float, float]
     """