Allow double-sending metrics to another "precise" backend (#97473)

We are using an in-between service doing pre-aggregation of
distribution-like (distribution and timer) metrics. This preaggregation
is bad, and we would like to get rid both of the preaggregation, and the
service doing it.

As a first step, this allows configuring a second "precise" metrics
backend, and opting into sending "precise" metrics to that second
backend if it is configured.

See also getsentry/getsentry#18133

Author: Swatinem

src/sentry/conf/server.py

@@ -2206,6 +2206,8 @@ def custom_parameter_sort(parameter: dict) -> tuple[str, int]:
 # Internal metrics
 SENTRY_METRICS_BACKEND = "sentry.metrics.dummy.DummyMetricsBackend"
 SENTRY_METRICS_OPTIONS: dict[str, Any] = {}
+SENTRY_METRICS_PRECISE_BACKEND: str | None = None
+SENTRY_METRICS_PRECISE_OPTIONS: dict[str, Any] = {}
 SENTRY_METRICS_SAMPLE_RATE = 1.0
 SENTRY_METRICS_PREFIX = "sentry."
 SENTRY_METRICS_SKIP_INTERNAL_PREFIXES: list[str] = []  # Order this by most frequent prefixes.

src/sentry/metrics/base.py

@@ -76,6 +76,7 @@ def distribution(
         sample_rate: float = 1,
         unit: str | None = None,
         stacklevel: int = 0,
+        precise: bool = False,
     ) -> None:
         raise NotImplementedError
 

src/sentry/metrics/datadog.py

@@ -108,9 +108,23 @@ def distribution(
         sample_rate: float = 1,
         unit: str | None = None,
         stacklevel: int = 0,
+        precise: bool = False,
     ) -> None:
-        # We keep the same implementation for Datadog.
-        self.timing(key, value, instance, tags, sample_rate)
+        if not precise:
+            # We keep the same implementation for Datadog.
+            return self.timing(key, value, instance, tags, sample_rate)
+
+        tags = dict(tags or ())
+
+        if self.tags:
+            tags.update(self.tags)
+        if instance:
+            tags["instance"] = instance
+
+        tags_list = [f"{k}:{v}" for k, v in tags.items()]
+        self.stats.distribution(
+            self._get_key(key), value, sample_rate=sample_rate, tags=tags_list, host=self.host
+        )
 
     def event(
         self,

src/sentry/metrics/dogstatsd.py

@@ -110,9 +110,21 @@ def distribution(
         sample_rate: float = 1,
         unit: str | None = None,
         stacklevel: int = 0,
+        precise: bool = False,
     ) -> None:
-        # We keep the same implementation for Datadog.
-        self.timing(key, value, instance, tags, sample_rate)
+        if not precise:
+            # We keep the same implementation for Datadog.
+            return self.timing(key, value, instance, tags, sample_rate)
+
+        tags = dict(tags or ())
+
+        if self.tags:
+            tags.update(self.tags)
+        if instance:
+            tags["instance"] = instance
+
+        tags_list = [f"{k}:{v}" for k, v in tags.items()]
+        statsd.distribution(self._get_key(key), value, sample_rate=sample_rate, tags=tags_list)
 
     def event(
         self,

src/sentry/metrics/dummy.py

@@ -48,6 +48,7 @@ def distribution(
         sample_rate: float = 1,
         unit: str | None = None,
         stacklevel: int = 0,
+        precise: bool = False,
     ) -> None:
         pass
 

src/sentry/metrics/logging.py

@@ -52,6 +52,7 @@ def distribution(
         sample_rate: float = 1,
         unit: str | None = None,
         stacklevel: int = 0,
+        precise: bool = False,
     ) -> None:
         logger.debug("%r: %+g", key, value, extra={"instance": instance, "tags": tags or {}})
 

src/sentry/metrics/middleware.py

@@ -180,14 +180,15 @@ def distribution(
         sample_rate: float = 1,
         unit: str | None = None,
         stacklevel: int = 0,
+        precise: bool = False,
     ) -> None:
         current_tags = get_current_global_tags()
         if tags is not None:
             current_tags.update(tags)
         current_tags = _filter_tags(key, current_tags)
 
         return self.inner.distribution(
-            key, value, instance, current_tags, sample_rate, unit, stacklevel + 1
+            key, value, instance, current_tags, sample_rate, unit, stacklevel + 1, precise
         )
 
     def event(

src/sentry/metrics/statsd.py

@@ -61,7 +61,9 @@ def distribution(
         sample_rate: float = 1,
         unit: str | None = None,
         stacklevel: int = 0,
+        precise: bool = False,
     ) -> None:
+        # NOTE: the statsd client does not have a `distribution` method
         self.timing(key, value, instance, tags, sample_rate)
 
     def event(

src/sentry/utils/metrics.py

@@ -31,6 +31,7 @@
     "timing",
     "gauge",
     "backend",
+    "precise_backend",  # just for mocking in tests
     "MutableTags",
     "ensure_crash_rate_in_bounds",
 ]
@@ -40,22 +41,21 @@
 F = TypeVar("F", bound=Callable[..., Any])
 
 
-def get_default_backend() -> MetricsBackend:
+def get_default_backend() -> tuple[MetricsBackend, MetricsBackend | None]:
     from sentry.utils.imports import import_string
 
-    cls: type[MetricsBackend] = import_string(settings.SENTRY_METRICS_BACKEND)
+    default_cls: type[MetricsBackend] = import_string(settings.SENTRY_METRICS_BACKEND)
+    default_backend = MiddlewareWrapper(default_cls(**settings.SENTRY_METRICS_OPTIONS))
 
-    return MiddlewareWrapper(cls(**settings.SENTRY_METRICS_OPTIONS))
+    precise_backend = None
+    if precise_import := settings.SENTRY_METRICS_PRECISE_BACKEND:
+        precise_cls: type[MetricsBackend] = import_string(precise_import)
+        precise_backend = MiddlewareWrapper(precise_cls(**settings.SENTRY_METRICS_PRECISE_OPTIONS))
 
+    return default_backend, precise_backend
 
-backend = get_default_backend()
 
-
-def _get_key(key: str) -> str:
-    prefix = settings.SENTRY_METRICS_PREFIX
-    if prefix:
-        return f"{prefix}{key}"
-    return key
+backend, precise_backend = get_default_backend()
 
 
 def _should_sample(sample_rate: float) -> bool:
@@ -168,13 +168,21 @@ def timing(
     tags: Tags | None = None,
     sample_rate: float = settings.SENTRY_METRICS_SAMPLE_RATE,
     stacklevel: int = 0,
+    precise: bool = False,
 ) -> None:
     try:
         backend.timing(key, value, instance, tags, sample_rate, stacklevel + 1)
     except Exception:
         logger = logging.getLogger("sentry.errors")
         logger.exception("Unable to record backend metric")
 
+    if precise and precise_backend:
+        try:
+            precise_backend.timing(key, value, instance, tags, sample_rate, stacklevel + 1)
+        except Exception:
+            logger = logging.getLogger("sentry.errors")
+            logger.exception("Unable to record precise metric")
+
 
 def distribution(
     key: str,
@@ -184,13 +192,23 @@ def distribution(
     sample_rate: float = settings.SENTRY_METRICS_SAMPLE_RATE,
     unit: str | None = None,
     stacklevel: int = 0,
+    precise: bool = False,
 ) -> None:
     try:
         backend.distribution(key, value, instance, tags, sample_rate, unit, stacklevel + 1)
     except Exception:
         logger = logging.getLogger("sentry.errors")
         logger.exception("Unable to record backend metric")
 
+    if precise and precise_backend:
+        try:
+            precise_backend.distribution(
+                key, value, instance, tags, sample_rate, unit, stacklevel + 1, precise=True
+            )
+        except Exception:
+            logger = logging.getLogger("sentry.errors")
+            logger.exception("Unable to record precise metric")
+
 
 @contextmanager
 def timer(
@@ -199,6 +217,7 @@ def timer(
     tags: Tags | None = None,
     sample_rate: float = settings.SENTRY_METRICS_SAMPLE_RATE,
     stacklevel: int = 0,
+    precise: bool = False,
 ) -> Generator[MutableTags]:
     start = time.monotonic()
     current_tags: MutableTags = dict(tags or ())
@@ -211,7 +230,15 @@ def timer(
         current_tags["result"] = "success"
     finally:
         # stacklevel must be increased by 2 because of the contextmanager indirection
-        timing(key, time.monotonic() - start, instance, current_tags, sample_rate, stacklevel + 2)
+        timing(
+            key,
+            time.monotonic() - start,
+            instance,
+            current_tags,
+            sample_rate,
+            stacklevel + 2,
+            precise,
+        )
 
 
 def wraps(
@@ -220,6 +247,7 @@ def wraps(
     tags: Tags | None = None,
     sample_rate: float = settings.SENTRY_METRICS_SAMPLE_RATE,
     stacklevel: int = 0,
+    precise: bool = False,
 ) -> Callable[[F], F]:
     def wrapper(f: F) -> F:
         @functools.wraps(f)
@@ -230,6 +258,7 @@ def inner(*args: Any, **kwargs: Any) -> Any:
                 tags=tags,
                 sample_rate=sample_rate,
                 stacklevel=stacklevel + 1,
+                precise=precise,
             ):
                 return f(*args, **kwargs)
 

tests/sentry/metrics/test_precise.py

@@ -0,0 +1,18 @@
+from unittest import mock
+
+from sentry.utils import metrics
+
+
+@mock.patch("sentry.utils.metrics.precise_backend")
+@mock.patch("sentry.utils.metrics.backend")
+def test_precise_distribution(backend, precise):
+    metrics.distribution("foo", 100, tags={"some": "stuff"}, unit="byte")
+
+    backend.distribution.assert_called_once()
+    precise.distribution.assert_not_called()
+    backend.reset_mock()
+
+    metrics.distribution("foo", 100, tags={"some": "stuff"}, unit="byte", precise=True)
+
+    backend.distribution.assert_called_once()
+    precise.distribution.assert_called_once()