inc: Add killswitch for trace id partitioning (#4706)

ref INC-1143

Some projects send such heavy abuse into single trace IDs that this
causes problems in production today. A better solution is needed
long-term as disabling this partition will lead to product
degradation once span buffer is rolled out.

Why a global option instead of a projectconfig? I think global options
are easier to reason about wrt how they propagate, and it's easier to
globally disable trace id partitioning this way.

Author: untitaker

CHANGELOG.md

@@ -9,6 +9,7 @@
 - Add OTA Updates Event Context for Expo and other applications. ([#4690](https://github.com/getsentry/relay/pull/4690))
 - Add data categories for Seer. ([#4692](https://github.com/getsentry/relay/pull/4692))
 - Allow pii scrubbing of all span `sentry_tags` fields. ([#4698](https://github.com/getsentry/relay/pull/4698))
+- Add killswitch for trace id partitioning. ([#4706](https://github.com/getsentry/relay/pull/4706))
 
 **Bug Fixes**:
 

relay-dynamic-config/src/global.rs

@@ -225,6 +225,17 @@ pub struct Options {
     )]
     pub deprecated2: f32,
 
+    /// Disable semantic partitioning of spans by trace ID. Use this in case there is partition
+    /// imbalance on the spans topic produced to by Relay (either snuba-spans or ingest-spans).
+    /// This will break the span buffer, and anything that depends on segments being assembled by
+    /// it (performance issue, etc). As of 2025-05-06, the span buffer is not yet rolled out to
+    /// most regions though.
+    #[serde(
+        rename = "relay.spans-ignore-trace-id-partitioning.projects",
+        skip_serializing_if = "is_default"
+    )]
+    pub spans_ignore_trace_id_partitioning: bool,
+
     /// All other unknown options.
     #[serde(flatten)]
     other: HashMap<String, Value>,

relay-server/src/services/store.rs

@@ -506,10 +506,20 @@ impl StoreService {
         &self,
         topic: KafkaTopic,
         // Takes message by value to ensure it is not being produced twice.
-        message: KafkaMessage,
+        mut message: KafkaMessage,
     ) -> Result<(), StoreError> {
         relay_log::trace!("Sending kafka message of type {}", message.variant());
 
+        if let KafkaMessage::Span {
+            ref mut ignore_trace_id_partitioning,
+            ..
+        } = message
+        {
+            let global_config = self.global_config.current();
+            *ignore_trace_id_partitioning =
+                global_config.options.spans_ignore_trace_id_partitioning;
+        }
+
         let topic_name = self.producer.client.send_message(topic, &message)?;
 
         match &message {
@@ -932,6 +942,7 @@ impl StoreService {
                     scoping.project_id.to_string(),
                 )]),
                 message: span,
+                ignore_trace_id_partitioning: false,
             },
         )?;
 
@@ -1588,6 +1599,8 @@ enum KafkaMessage<'a> {
         headers: BTreeMap<String, String>,
         #[serde(flatten)]
         message: SpanKafkaMessage<'a>,
+        #[serde(skip)]
+        ignore_trace_id_partitioning: bool,
     },
     Log {
         #[serde(skip)]
@@ -1631,7 +1644,17 @@ impl Message for KafkaMessage<'_> {
             Self::AttachmentChunk(message) => message.event_id.0,
             Self::UserReport(message) => message.event_id.0,
             Self::ReplayEvent(message) => message.replay_id.0,
-            Self::Span { message, .. } => message.trace_id.0,
+            Self::Span {
+                message,
+                ignore_trace_id_partitioning,
+                ..
+            } => {
+                if *ignore_trace_id_partitioning {
+                    Uuid::nil()
+                } else {
+                    message.trace_id.0
+                }
+            }
 
             // Monitor check-ins use the hinted UUID passed through from the Envelope.
             //