fix(tracemetrics): Switch experimental backend to kwarg rate (#101520)

### Summary
Based on some digging, last time we tried to use an option with metrics
it caused INC-1101 and INC-1130 with spikes hitting db in production as
some metrics are module/import level. This will be slower (as it needs a
deploy to change volume) but we can still kill switch via
tracemetrics-ingestion in relay.

Author: k-fish

src/sentry/metrics/sentry_sdk.py

@@ -1,20 +1,16 @@
+from random import random
 from typing import Any
 
 from sentry_sdk import _metrics as metrics
 
-from sentry import options
-from sentry.options.rollout import in_random_rollout
-
 from .base import MetricsBackend, Tags
 
 __all__ = ["SentrySDKMetricsBackend"]
 
 
 class SentrySDKMetricsBackend(MetricsBackend):
     def __init__(self, **kwargs: Any) -> None:
-        self._rollout_option = kwargs.pop(
-            "rollout_option", "tracemetrics.sentry_sdk_metrics_backend_rate"
-        )
+        self._experimental_sample_rate = kwargs.pop("experimental_sample_rate", 0.0)
         self._deny_list = tuple(kwargs.pop("deny_list", []))
         super().__init__(**kwargs)
 
@@ -24,10 +20,11 @@ def _is_denied(self, key: str) -> bool:
     def _should_send(self, key: str) -> bool:
         if self._is_denied(key):
             return False
-        # We don't want to send metrics before the cache exists to avoid excessive cache misses.
-        if not options.default_store.cache:
-            return False
-        return in_random_rollout(self._rollout_option)
+        return self._should_sample_experimental()
+
+    def _should_sample_experimental(self) -> bool:
+        """Sample based on passed in sample rate, can't use options as they hit the db too much."""
+        return self._experimental_sample_rate >= 1.0 or random() < self._experimental_sample_rate
 
     def incr(
         self,

src/sentry/options/defaults.py

@@ -2767,12 +2767,6 @@
     flags=FLAG_PRIORITIZE_DISK | FLAG_AUTOMATOR_MODIFIABLE,
 )
 
-register(
-    "tracemetrics.sentry_sdk_metrics_backend_rate",
-    type=Float,
-    default=0.0,
-    flags=FLAG_AUTOMATOR_MODIFIABLE,
-)
 
 # TODO: For now, only a small number of projects are going through a grouping config transition at
 # any given time, so we're sampling at 100% in order to be able to get good signal. Once we've fully

tests/sentry/metrics/test_dualwrite.py

@@ -1,7 +1,6 @@
 from unittest import mock
 
 from sentry.metrics.dualwrite import DualWriteMetricsBackend
-from sentry.testutils.helpers.options import override_options
 from sentry.testutils.thread_leaks.pytest import thread_leak_allowlist
 
 
@@ -36,15 +35,14 @@ def test_dualwrite_distribution(distribution, timing):
     distribution.assert_not_called()
 
 
-@override_options({"tracemetrics.sentry_sdk_metrics_backend_rate": 1.0})
 @mock.patch("sentry_sdk._metrics.count")
 @mock.patch("datadog.dogstatsd.base.DogStatsd.increment")
 @thread_leak_allowlist(reason="datadog dualwrite metrics", issue=98803)
 def test_dualwrite_experimental_backend(dogstatsd_incr, sentry_sdk_incr):
     backend = DualWriteMetricsBackend(
         primary_backend="sentry.metrics.dogstatsd.DogStatsdMetricsBackend",
         experimental_backend="sentry.metrics.sentry_sdk.SentrySDKMetricsBackend",
-        experimental_args={"deny_list": ["denied"]},
+        experimental_args={"deny_list": ["denied"], "experimental_sample_rate": 1.0},
     )
 
     backend.incr("allowed", tags={"test": "tag"}, unit="none")
@@ -59,15 +57,14 @@ def test_dualwrite_experimental_backend(dogstatsd_incr, sentry_sdk_incr):
     sentry_sdk_incr.assert_not_called()
 
 
-@override_options({"tracemetrics.sentry_sdk_metrics_backend_rate": 0.0})
 @mock.patch("sentry_sdk._metrics.gauge")
 @mock.patch("datadog.dogstatsd.base.DogStatsd.gauge")
 @thread_leak_allowlist(reason="datadog dualwrite metrics", issue=98803)
 def test_dualwrite_experimental_backend_rollout_disabled(dogstatsd_gauge, sentry_sdk_gauge):
     backend = DualWriteMetricsBackend(
         primary_backend="sentry.metrics.dogstatsd.DogStatsdMetricsBackend",
         experimental_backend="sentry.metrics.sentry_sdk.SentrySDKMetricsBackend",
-        experimental_args={"deny_list": []},
+        experimental_args={"deny_list": [], "experimental_sample_rate": 0.0},
     )
 
     backend.gauge("metric", 42, tags={"test": "tag"}, unit="none")

tests/sentry/metrics/test_sentry_sdk.py

@@ -8,7 +8,7 @@
 class TestSentrySDKMetricsBackend:
     @pytest.fixture
     def backend(self):
-        return SentrySDKMetricsBackend(prefix="test.")
+        return SentrySDKMetricsBackend(prefix="test.", experimental_sample_rate=1.0)
 
     @mock.patch("sentry_sdk._metrics.count")
     def test_incr(self, mock_count, backend):
@@ -77,9 +77,18 @@ def test_event_noop(self, backend):
         backend.event("title", "message")
 
     @mock.patch("sentry_sdk._metrics.count")
-    def test_incr_sampling(self, mock_count, backend):
-        with mock.patch.object(backend, "_should_send", return_value=True):
-            with mock.patch.object(backend, "_should_sample", return_value=False):
-                backend.incr("foo", amount=1)
+    def test_incr_sampling(self, mock_count):
+        backend = SentrySDKMetricsBackend(prefix="test.", experimental_sample_rate=0.0)
+        backend.incr("foo", amount=1)
+        mock_count.assert_not_called()
 
+    @mock.patch("sentry_sdk._metrics.count")
+    def test_incr_deny_list(self, mock_count):
+        backend = SentrySDKMetricsBackend(
+            prefix="test.", experimental_sample_rate=1.0, deny_list=["denied"]
+        )
+        backend.incr("denied.metric", amount=1)
         mock_count.assert_not_called()
+
+        backend.incr("allowed.metric", amount=1)
+        mock_count.assert_called_once()