diff --git a/sentry_sdk/_types.py b/sentry_sdk/_types.py index b28c7260ce..d057f215e4 100644 --- a/sentry_sdk/_types.py +++ b/sentry_sdk/_types.py @@ -210,7 +210,6 @@ class SDKInfo(TypedDict): "type": Literal["check_in", "transaction"], "user": dict[str, object], "_dropped_spans": int, - "_metrics_summary": dict[str, object], }, total=False, ) @@ -266,7 +265,6 @@ class SDKInfo(TypedDict): "internal", "profile", "profile_chunk", - "metric_bucket", "monitor", "span", "log_item", @@ -276,26 +274,6 @@ class SDKInfo(TypedDict): ContinuousProfilerMode = Literal["thread", "gevent", "unknown"] ProfilerMode = Union[ContinuousProfilerMode, Literal["sleep"]] - # Type of the metric. - MetricType = Literal["d", "s", "g", "c"] - - # Value of the metric. - MetricValue = Union[int, float, str] - - # Internal representation of tags as a tuple of tuples (this is done in order to allow for the same key to exist - # multiple times). - MetricTagsInternal = Tuple[Tuple[str, str], ...] - - # External representation of tags as a dictionary. - MetricTagValue = Union[str, int, float, None] - MetricTags = Mapping[str, MetricTagValue] - - # Value inside the generator for the metric value. - FlushedMetricValue = Union[int, float] - - BucketKey = Tuple[MetricType, str, MeasurementUnit, MetricTagsInternal] - MetricMetaKey = Tuple[MetricType, str, MeasurementUnit] - MonitorConfigScheduleType = Literal["crontab", "interval"] MonitorConfigScheduleUnit = Literal[ "year", diff --git a/sentry_sdk/client.py b/sentry_sdk/client.py index c06043ebe2..59a5013783 100644 --- a/sentry_sdk/client.py +++ b/sentry_sdk/client.py @@ -61,7 +61,6 @@ from sentry_sdk._types import Event, Hint, SDKInfo, Log from sentry_sdk.integrations import Integration - from sentry_sdk.metrics import MetricsAggregator from sentry_sdk.scope import Scope from sentry_sdk.session import Session from sentry_sdk.spotlight import SpotlightClient @@ -182,7 +181,6 @@ def __init__(self, options=None): self.transport = None # type: Optional[Transport] self.monitor = None # type: Optional[Monitor] - self.metrics_aggregator = None # type: Optional[MetricsAggregator] self.log_batcher = None # type: Optional[LogBatcher] def __getstate__(self, *args, **kwargs): @@ -361,26 +359,6 @@ def _capture_envelope(envelope): self.session_flusher = SessionFlusher(capture_func=_capture_envelope) - self.metrics_aggregator = None # type: Optional[MetricsAggregator] - experiments = self.options.get("_experiments", {}) - if experiments.get("enable_metrics", True): - # Context vars are not working correctly on Python <=3.6 - # with gevent. - metrics_supported = not is_gevent() or PY37 - if metrics_supported: - from sentry_sdk.metrics import MetricsAggregator - - self.metrics_aggregator = MetricsAggregator( - capture_func=_capture_envelope, - enable_code_locations=bool( - experiments.get("metric_code_locations", True) - ), - ) - else: - logger.info( - "Metrics not supported on Python 3.6 and lower with gevent." - ) - self.log_batcher = None if has_logs_enabled(self.options): @@ -467,7 +445,6 @@ def _capture_envelope(envelope): if ( self.monitor - or self.metrics_aggregator or self.log_batcher or has_profiling_enabled(self.options) or isinstance(self.transport, BaseHttpTransport) @@ -1019,8 +996,6 @@ def close( if self.transport is not None: self.flush(timeout=timeout, callback=callback) self.session_flusher.kill() - if self.metrics_aggregator is not None: - self.metrics_aggregator.kill() if self.log_batcher is not None: self.log_batcher.kill() if self.monitor: @@ -1045,8 +1020,6 @@ def flush( if timeout is None: timeout = self.options["shutdown_timeout"] self.session_flusher.flush() - if self.metrics_aggregator is not None: - self.metrics_aggregator.flush() if self.log_batcher is not None: self.log_batcher.flush() self.transport.flush(timeout=timeout, callback=callback) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 43c7e857ac..0f71a0d460 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -55,8 +55,6 @@ class CompressionAlgo(Enum): ProfilerMode, TracesSampler, TransactionProcessor, - MetricTags, - MetricValue, ) # Experiments are feature flags to enable and disable certain unstable SDK @@ -77,11 +75,6 @@ class CompressionAlgo(Enum): "transport_compression_algo": Optional[CompressionAlgo], "transport_num_pools": Optional[int], "transport_http2": Optional[bool], - "enable_metrics": Optional[bool], - "before_emit_metric": Optional[ - Callable[[str, MetricValue, MeasurementUnit, MetricTags], bool] - ], - "metric_code_locations": Optional[bool], "enable_logs": Optional[bool], "before_send_log": Optional[Callable[[Log, Hint], Optional[Log]]], }, diff --git a/sentry_sdk/envelope.py b/sentry_sdk/envelope.py index d9b2c1629a..b26c458d41 100644 --- a/sentry_sdk/envelope.py +++ b/sentry_sdk/envelope.py @@ -291,8 +291,6 @@ def data_category(self): return "profile" elif ty == "profile_chunk": return "profile_chunk" - elif ty == "statsd": - return "metric_bucket" elif ty == "check_in": return "monitor" else: @@ -354,7 +352,7 @@ def deserialize_from( # if no length was specified we need to read up to the end of line # and remove it (if it is present, i.e. not the very last char in an eof terminated envelope) payload = f.readline().rstrip(b"\n") - if headers.get("type") in ("event", "transaction", "metric_buckets"): + if headers.get("type") in ("event", "transaction"): rv = cls(headers=headers, payload=PayloadRef(json=parse_json(payload))) else: rv = cls(headers=headers, payload=payload) diff --git a/sentry_sdk/metrics.py b/sentry_sdk/metrics.py deleted file mode 100644 index d0041114ce..0000000000 --- a/sentry_sdk/metrics.py +++ /dev/null @@ -1,971 +0,0 @@ -import io -import os -import random -import re -import sys -import threading -import time -import warnings -import zlib -from abc import ABC, abstractmethod -from contextlib import contextmanager -from datetime import datetime, timezone -from functools import wraps, partial - -import sentry_sdk -from sentry_sdk.utils import ( - ContextVar, - now, - nanosecond_time, - to_timestamp, - serialize_frame, - json_dumps, -) -from sentry_sdk.envelope import Envelope, Item -from sentry_sdk.tracing import TransactionSource - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from typing import Any - from typing import Callable - from typing import Dict - from typing import Generator - from typing import Iterable - from typing import List - from typing import Optional - from typing import Set - from typing import Tuple - from typing import Union - - from sentry_sdk._types import BucketKey - from sentry_sdk._types import DurationUnit - from sentry_sdk._types import FlushedMetricValue - from sentry_sdk._types import MeasurementUnit - from sentry_sdk._types import MetricMetaKey - from sentry_sdk._types import MetricTagValue - from sentry_sdk._types import MetricTags - from sentry_sdk._types import MetricTagsInternal - from sentry_sdk._types import MetricType - from sentry_sdk._types import MetricValue - - -warnings.warn( - "The sentry_sdk.metrics module is deprecated and will be removed in the next major release. " - "Sentry will reject all metrics sent after October 7, 2024. " - "Learn more: https://sentry.zendesk.com/hc/en-us/articles/26369339769883-Upcoming-API-Changes-to-Metrics", - DeprecationWarning, - stacklevel=2, -) - -_in_metrics = ContextVar("in_metrics", default=False) -_set = set # set is shadowed below - -GOOD_TRANSACTION_SOURCES = frozenset( - [ - TransactionSource.ROUTE, - TransactionSource.VIEW, - TransactionSource.COMPONENT, - TransactionSource.TASK, - ] -) - -_sanitize_unit = partial(re.compile(r"[^a-zA-Z0-9_]+").sub, "") -_sanitize_metric_key = partial(re.compile(r"[^a-zA-Z0-9_\-.]+").sub, "_") -_sanitize_tag_key = partial(re.compile(r"[^a-zA-Z0-9_\-.\/]+").sub, "") - - -def _sanitize_tag_value(value): - # type: (str) -> str - table = str.maketrans( - { - "\n": "\\n", - "\r": "\\r", - "\t": "\\t", - "\\": "\\\\", - "|": "\\u{7c}", - ",": "\\u{2c}", - } - ) - return value.translate(table) - - -def get_code_location(stacklevel): - # type: (int) -> Optional[Dict[str, Any]] - try: - frm = sys._getframe(stacklevel) - except Exception: - return None - - return serialize_frame( - frm, include_local_variables=False, include_source_context=True - ) - - -@contextmanager -def recursion_protection(): - # type: () -> Generator[bool, None, None] - """Enters recursion protection and returns the old flag.""" - old_in_metrics = _in_metrics.get() - _in_metrics.set(True) - try: - yield old_in_metrics - finally: - _in_metrics.set(old_in_metrics) - - -def metrics_noop(func): - # type: (Any) -> Any - """Convenient decorator that uses `recursion_protection` to - make a function a noop. - """ - - @wraps(func) - def new_func(*args, **kwargs): - # type: (*Any, **Any) -> Any - with recursion_protection() as in_metrics: - if not in_metrics: - return func(*args, **kwargs) - - return new_func - - -class Metric(ABC): - __slots__ = () - - @abstractmethod - def __init__(self, first): - # type: (MetricValue) -> None - pass - - @property - @abstractmethod - def weight(self): - # type: () -> int - pass - - @abstractmethod - def add(self, value): - # type: (MetricValue) -> None - pass - - @abstractmethod - def serialize_value(self): - # type: () -> Iterable[FlushedMetricValue] - pass - - -class CounterMetric(Metric): - __slots__ = ("value",) - - def __init__( - self, - first, # type: MetricValue - ): - # type: (...) -> None - self.value = float(first) - - @property - def weight(self): - # type: (...) -> int - return 1 - - def add( - self, - value, # type: MetricValue - ): - # type: (...) -> None - self.value += float(value) - - def serialize_value(self): - # type: (...) -> Iterable[FlushedMetricValue] - return (self.value,) - - -class GaugeMetric(Metric): - __slots__ = ( - "last", - "min", - "max", - "sum", - "count", - ) - - def __init__( - self, - first, # type: MetricValue - ): - # type: (...) -> None - first = float(first) - self.last = first - self.min = first - self.max = first - self.sum = first - self.count = 1 - - @property - def weight(self): - # type: (...) -> int - # Number of elements. - return 5 - - def add( - self, - value, # type: MetricValue - ): - # type: (...) -> None - value = float(value) - self.last = value - self.min = min(self.min, value) - self.max = max(self.max, value) - self.sum += value - self.count += 1 - - def serialize_value(self): - # type: (...) -> Iterable[FlushedMetricValue] - return ( - self.last, - self.min, - self.max, - self.sum, - self.count, - ) - - -class DistributionMetric(Metric): - __slots__ = ("value",) - - def __init__( - self, - first, # type: MetricValue - ): - # type(...) -> None - self.value = [float(first)] - - @property - def weight(self): - # type: (...) -> int - return len(self.value) - - def add( - self, - value, # type: MetricValue - ): - # type: (...) -> None - self.value.append(float(value)) - - def serialize_value(self): - # type: (...) -> Iterable[FlushedMetricValue] - return self.value - - -class SetMetric(Metric): - __slots__ = ("value",) - - def __init__( - self, - first, # type: MetricValue - ): - # type: (...) -> None - self.value = {first} - - @property - def weight(self): - # type: (...) -> int - return len(self.value) - - def add( - self, - value, # type: MetricValue - ): - # type: (...) -> None - self.value.add(value) - - def serialize_value(self): - # type: (...) -> Iterable[FlushedMetricValue] - def _hash(x): - # type: (MetricValue) -> int - if isinstance(x, str): - return zlib.crc32(x.encode("utf-8")) & 0xFFFFFFFF - return int(x) - - return (_hash(value) for value in self.value) - - -def _encode_metrics(flushable_buckets): - # type: (Iterable[Tuple[int, Dict[BucketKey, Metric]]]) -> bytes - out = io.BytesIO() - _write = out.write - - # Note on sanitization: we intentionally sanitize in emission (serialization) - # and not during aggregation for performance reasons. This means that the - # envelope can in fact have duplicate buckets stored. This is acceptable for - # relay side emission and should not happen commonly. - - for timestamp, buckets in flushable_buckets: - for bucket_key, metric in buckets.items(): - metric_type, metric_name, metric_unit, metric_tags = bucket_key - metric_name = _sanitize_metric_key(metric_name) - metric_unit = _sanitize_unit(metric_unit) - _write(metric_name.encode("utf-8")) - _write(b"@") - _write(metric_unit.encode("utf-8")) - - for serialized_value in metric.serialize_value(): - _write(b":") - _write(str(serialized_value).encode("utf-8")) - - _write(b"|") - _write(metric_type.encode("ascii")) - - if metric_tags: - _write(b"|#") - first = True - for tag_key, tag_value in metric_tags: - tag_key = _sanitize_tag_key(tag_key) - if not tag_key: - continue - if first: - first = False - else: - _write(b",") - _write(tag_key.encode("utf-8")) - _write(b":") - _write(_sanitize_tag_value(tag_value).encode("utf-8")) - - _write(b"|T") - _write(str(timestamp).encode("ascii")) - _write(b"\n") - - return out.getvalue() - - -def _encode_locations(timestamp, code_locations): - # type: (int, Iterable[Tuple[MetricMetaKey, Dict[str, Any]]]) -> bytes - mapping = {} # type: Dict[str, List[Any]] - - for key, loc in code_locations: - metric_type, name, unit = key - mri = "{}:{}@{}".format( - metric_type, _sanitize_metric_key(name), _sanitize_unit(unit) - ) - - loc["type"] = "location" - mapping.setdefault(mri, []).append(loc) - - return json_dumps({"timestamp": timestamp, "mapping": mapping}) - - -METRIC_TYPES = { - "c": CounterMetric, - "g": GaugeMetric, - "d": DistributionMetric, - "s": SetMetric, -} # type: dict[MetricType, type[Metric]] - -# some of these are dumb -TIMING_FUNCTIONS = { - "nanosecond": nanosecond_time, - "microsecond": lambda: nanosecond_time() / 1000.0, - "millisecond": lambda: nanosecond_time() / 1000000.0, - "second": now, - "minute": lambda: now() / 60.0, - "hour": lambda: now() / 3600.0, - "day": lambda: now() / 3600.0 / 24.0, - "week": lambda: now() / 3600.0 / 24.0 / 7.0, -} - - -class LocalAggregator: - __slots__ = ("_measurements",) - - def __init__(self): - # type: (...) -> None - self._measurements = {} # type: Dict[Tuple[str, MetricTagsInternal], Tuple[float, float, int, float]] - - def add( - self, - ty, # type: MetricType - key, # type: str - value, # type: float - unit, # type: MeasurementUnit - tags, # type: MetricTagsInternal - ): - # type: (...) -> None - export_key = "%s:%s@%s" % (ty, key, unit) - bucket_key = (export_key, tags) - - old = self._measurements.get(bucket_key) - if old is not None: - v_min, v_max, v_count, v_sum = old - v_min = min(v_min, value) - v_max = max(v_max, value) - v_count += 1 - v_sum += value - else: - v_min = v_max = v_sum = value - v_count = 1 - self._measurements[bucket_key] = (v_min, v_max, v_count, v_sum) - - def to_json(self): - # type: (...) -> Dict[str, Any] - rv = {} # type: Any - for (export_key, tags), ( - v_min, - v_max, - v_count, - v_sum, - ) in self._measurements.items(): - rv.setdefault(export_key, []).append( - { - "tags": _tags_to_dict(tags), - "min": v_min, - "max": v_max, - "count": v_count, - "sum": v_sum, - } - ) - return rv - - -class MetricsAggregator: - ROLLUP_IN_SECONDS = 10.0 - MAX_WEIGHT = 100000 - FLUSHER_SLEEP_TIME = 5.0 - - def __init__( - self, - capture_func, # type: Callable[[Envelope], None] - enable_code_locations=False, # type: bool - ): - # type: (...) -> None - self.buckets = {} # type: Dict[int, Any] - self._enable_code_locations = enable_code_locations - self._seen_locations = _set() # type: Set[Tuple[int, MetricMetaKey]] - self._pending_locations = {} # type: Dict[int, List[Tuple[MetricMetaKey, Any]]] - self._buckets_total_weight = 0 - self._capture_func = capture_func - self._running = True - self._lock = threading.Lock() - - self._flush_event = threading.Event() # type: threading.Event - self._force_flush = False - - # The aggregator shifts its flushing by up to an entire rollup window to - # avoid multiple clients trampling on end of a 10 second window as all the - # buckets are anchored to multiples of ROLLUP seconds. We randomize this - # number once per aggregator boot to achieve some level of offsetting - # across a fleet of deployed SDKs. Relay itself will also apply independent - # jittering. - self._flush_shift = random.random() * self.ROLLUP_IN_SECONDS - - self._flusher = None # type: Optional[threading.Thread] - self._flusher_pid = None # type: Optional[int] - - def _ensure_thread(self): - # type: (...) -> bool - """For forking processes we might need to restart this thread. - This ensures that our process actually has that thread running. - """ - if not self._running: - return False - - pid = os.getpid() - if self._flusher_pid == pid: - return True - - with self._lock: - # Recheck to make sure another thread didn't get here and start the - # the flusher in the meantime - if self._flusher_pid == pid: - return True - - self._flusher_pid = pid - - self._flusher = threading.Thread(target=self._flush_loop) - self._flusher.daemon = True - - try: - self._flusher.start() - except RuntimeError: - # Unfortunately at this point the interpreter is in a state that no - # longer allows us to spawn a thread and we have to bail. - self._running = False - return False - - return True - - def _flush_loop(self): - # type: (...) -> None - _in_metrics.set(True) - while self._running or self._force_flush: - if self._running: - self._flush_event.wait(self.FLUSHER_SLEEP_TIME) - self._flush() - - def _flush(self): - # type: (...) -> None - self._emit(self._flushable_buckets(), self._flushable_locations()) - - def _flushable_buckets(self): - # type: (...) -> (Iterable[Tuple[int, Dict[BucketKey, Metric]]]) - with self._lock: - force_flush = self._force_flush - cutoff = time.time() - self.ROLLUP_IN_SECONDS - self._flush_shift - flushable_buckets = () # type: Iterable[Tuple[int, Dict[BucketKey, Metric]]] - weight_to_remove = 0 - - if force_flush: - flushable_buckets = self.buckets.items() - self.buckets = {} - self._buckets_total_weight = 0 - self._force_flush = False - else: - flushable_buckets = [] - for buckets_timestamp, buckets in self.buckets.items(): - # If the timestamp of the bucket is newer that the rollup we want to skip it. - if buckets_timestamp <= cutoff: - flushable_buckets.append((buckets_timestamp, buckets)) - - # We will clear the elements while holding the lock, in order to avoid requesting it downstream again. - for buckets_timestamp, buckets in flushable_buckets: - for metric in buckets.values(): - weight_to_remove += metric.weight - del self.buckets[buckets_timestamp] - - self._buckets_total_weight -= weight_to_remove - - return flushable_buckets - - def _flushable_locations(self): - # type: (...) -> Dict[int, List[Tuple[MetricMetaKey, Dict[str, Any]]]] - with self._lock: - locations = self._pending_locations - self._pending_locations = {} - return locations - - @metrics_noop - def add( - self, - ty, # type: MetricType - key, # type: str - value, # type: MetricValue - unit, # type: MeasurementUnit - tags, # type: Optional[MetricTags] - timestamp=None, # type: Optional[Union[float, datetime]] - local_aggregator=None, # type: Optional[LocalAggregator] - stacklevel=0, # type: Optional[int] - ): - # type: (...) -> None - if not self._ensure_thread() or self._flusher is None: - return None - - if timestamp is None: - timestamp = time.time() - elif isinstance(timestamp, datetime): - timestamp = to_timestamp(timestamp) - - bucket_timestamp = int( - (timestamp // self.ROLLUP_IN_SECONDS) * self.ROLLUP_IN_SECONDS - ) - serialized_tags = _serialize_tags(tags) - bucket_key = ( - ty, - key, - unit, - serialized_tags, - ) - - with self._lock: - local_buckets = self.buckets.setdefault(bucket_timestamp, {}) - metric = local_buckets.get(bucket_key) - if metric is not None: - previous_weight = metric.weight - metric.add(value) - else: - metric = local_buckets[bucket_key] = METRIC_TYPES[ty](value) - previous_weight = 0 - - added = metric.weight - previous_weight - - if stacklevel is not None: - self.record_code_location(ty, key, unit, stacklevel + 2, timestamp) - - # Given the new weight we consider whether we want to force flush. - self._consider_force_flush() - - # For sets, we only record that a value has been added to the set but not which one. - # See develop docs: https://develop.sentry.dev/sdk/metrics/#sets - if local_aggregator is not None: - local_value = float(added if ty == "s" else value) - local_aggregator.add(ty, key, local_value, unit, serialized_tags) - - def record_code_location( - self, - ty, # type: MetricType - key, # type: str - unit, # type: MeasurementUnit - stacklevel, # type: int - timestamp=None, # type: Optional[float] - ): - # type: (...) -> None - if not self._enable_code_locations: - return - if timestamp is None: - timestamp = time.time() - meta_key = (ty, key, unit) - start_of_day = datetime.fromtimestamp(timestamp, timezone.utc).replace( - hour=0, minute=0, second=0, microsecond=0, tzinfo=None - ) - start_of_day = int(to_timestamp(start_of_day)) - - if (start_of_day, meta_key) not in self._seen_locations: - self._seen_locations.add((start_of_day, meta_key)) - loc = get_code_location(stacklevel + 3) - if loc is not None: - # Group metadata by day to make flushing more efficient. - # There needs to be one envelope item per timestamp. - self._pending_locations.setdefault(start_of_day, []).append( - (meta_key, loc) - ) - - @metrics_noop - def need_code_location( - self, - ty, # type: MetricType - key, # type: str - unit, # type: MeasurementUnit - timestamp, # type: float - ): - # type: (...) -> bool - if self._enable_code_locations: - return False - meta_key = (ty, key, unit) - start_of_day = datetime.fromtimestamp(timestamp, timezone.utc).replace( - hour=0, minute=0, second=0, microsecond=0, tzinfo=None - ) - start_of_day = int(to_timestamp(start_of_day)) - return (start_of_day, meta_key) not in self._seen_locations - - def kill(self): - # type: (...) -> None - if self._flusher is None: - return - - self._running = False - self._flush_event.set() - self._flusher = None - - @metrics_noop - def flush(self): - # type: (...) -> None - self._force_flush = True - self._flush() - - def _consider_force_flush(self): - # type: (...) -> None - # It's important to acquire a lock around this method, since it will touch shared data structures. - total_weight = len(self.buckets) + self._buckets_total_weight - if total_weight >= self.MAX_WEIGHT: - self._force_flush = True - self._flush_event.set() - - def _emit( - self, - flushable_buckets, # type: (Iterable[Tuple[int, Dict[BucketKey, Metric]]]) - code_locations, # type: Dict[int, List[Tuple[MetricMetaKey, Dict[str, Any]]]] - ): - # type: (...) -> Optional[Envelope] - envelope = Envelope() - - if flushable_buckets: - encoded_metrics = _encode_metrics(flushable_buckets) - envelope.add_item(Item(payload=encoded_metrics, type="statsd")) - - for timestamp, locations in code_locations.items(): - encoded_locations = _encode_locations(timestamp, locations) - envelope.add_item(Item(payload=encoded_locations, type="metric_meta")) - - if envelope.items: - self._capture_func(envelope) - return envelope - return None - - -def _serialize_tags( - tags, # type: Optional[MetricTags] -): - # type: (...) -> MetricTagsInternal - if not tags: - return () - - rv = [] - for key, value in tags.items(): - # If the value is a collection, we want to flatten it. - if isinstance(value, (list, tuple)): - for inner_value in value: - if inner_value is not None: - rv.append((key, str(inner_value))) - elif value is not None: - rv.append((key, str(value))) - - # It's very important to sort the tags in order to obtain the - # same bucket key. - return tuple(sorted(rv)) - - -def _tags_to_dict(tags): - # type: (MetricTagsInternal) -> Dict[str, Any] - rv = {} # type: Dict[str, Any] - for tag_name, tag_value in tags: - old_value = rv.get(tag_name) - if old_value is not None: - if isinstance(old_value, list): - old_value.append(tag_value) - else: - rv[tag_name] = [old_value, tag_value] - else: - rv[tag_name] = tag_value - return rv - - -def _get_aggregator(): - # type: () -> Optional[MetricsAggregator] - client = sentry_sdk.get_client() - return ( - client.metrics_aggregator - if client.is_active() and client.metrics_aggregator is not None - else None - ) - - -def _get_aggregator_and_update_tags(key, value, unit, tags): - # type: (str, Optional[MetricValue], MeasurementUnit, Optional[MetricTags]) -> Tuple[Optional[MetricsAggregator], Optional[LocalAggregator], Optional[MetricTags]] - client = sentry_sdk.get_client() - if not client.is_active() or client.metrics_aggregator is None: - return None, None, tags - - updated_tags = dict(tags or ()) # type: Dict[str, MetricTagValue] - updated_tags.setdefault("release", client.options["release"]) - updated_tags.setdefault("environment", client.options["environment"]) - - scope = sentry_sdk.get_current_scope() - local_aggregator = None - - # We go with the low-level API here to access transaction information as - # this one is the same between just errors and errors + performance - transaction_source = scope._transaction_info.get("source") - if transaction_source in GOOD_TRANSACTION_SOURCES: - transaction_name = scope._transaction - if transaction_name: - updated_tags.setdefault("transaction", transaction_name) - if scope._span is not None: - local_aggregator = scope._span._get_local_aggregator() - - experiments = client.options.get("_experiments", {}) - before_emit_callback = experiments.get("before_emit_metric") - if before_emit_callback is not None: - with recursion_protection() as in_metrics: - if not in_metrics: - if not before_emit_callback(key, value, unit, updated_tags): - return None, None, updated_tags - - return client.metrics_aggregator, local_aggregator, updated_tags - - -def increment( - key, # type: str - value=1.0, # type: float - unit="none", # type: MeasurementUnit - tags=None, # type: Optional[MetricTags] - timestamp=None, # type: Optional[Union[float, datetime]] - stacklevel=0, # type: int -): - # type: (...) -> None - """Increments a counter.""" - aggregator, local_aggregator, tags = _get_aggregator_and_update_tags( - key, value, unit, tags - ) - if aggregator is not None: - aggregator.add( - "c", key, value, unit, tags, timestamp, local_aggregator, stacklevel - ) - - -# alias as incr is relatively common in python -incr = increment - - -class _Timing: - def __init__( - self, - key, # type: str - tags, # type: Optional[MetricTags] - timestamp, # type: Optional[Union[float, datetime]] - value, # type: Optional[float] - unit, # type: DurationUnit - stacklevel, # type: int - ): - # type: (...) -> None - self.key = key - self.tags = tags - self.timestamp = timestamp - self.value = value - self.unit = unit - self.entered = None # type: Optional[float] - self._span = None # type: Optional[sentry_sdk.tracing.Span] - self.stacklevel = stacklevel - - def _validate_invocation(self, context): - # type: (str) -> None - if self.value is not None: - raise TypeError( - "cannot use timing as %s when a value is provided" % context - ) - - def __enter__(self): - # type: (...) -> _Timing - self.entered = TIMING_FUNCTIONS[self.unit]() - self._validate_invocation("context-manager") - self._span = sentry_sdk.start_span(op="metric.timing", name=self.key) - if self.tags: - for key, value in self.tags.items(): - if isinstance(value, (tuple, list)): - value = ",".join(sorted(map(str, value))) - self._span.set_tag(key, value) - self._span.__enter__() - - # report code locations here for better accuracy - aggregator = _get_aggregator() - if aggregator is not None: - aggregator.record_code_location("d", self.key, self.unit, self.stacklevel) - - return self - - def __exit__(self, exc_type, exc_value, tb): - # type: (Any, Any, Any) -> None - assert self._span, "did not enter" - aggregator, local_aggregator, tags = _get_aggregator_and_update_tags( - self.key, - self.value, - self.unit, - self.tags, - ) - if aggregator is not None: - elapsed = TIMING_FUNCTIONS[self.unit]() - self.entered # type: ignore - aggregator.add( - "d", - self.key, - elapsed, - self.unit, - tags, - self.timestamp, - local_aggregator, - None, # code locations are reported in __enter__ - ) - - self._span.__exit__(exc_type, exc_value, tb) - self._span = None - - def __call__(self, f): - # type: (Any) -> Any - self._validate_invocation("decorator") - - @wraps(f) - def timed_func(*args, **kwargs): - # type: (*Any, **Any) -> Any - with timing( - key=self.key, - tags=self.tags, - timestamp=self.timestamp, - unit=self.unit, - stacklevel=self.stacklevel + 1, - ): - return f(*args, **kwargs) - - return timed_func - - -def timing( - key, # type: str - value=None, # type: Optional[float] - unit="second", # type: DurationUnit - tags=None, # type: Optional[MetricTags] - timestamp=None, # type: Optional[Union[float, datetime]] - stacklevel=0, # type: int -): - # type: (...) -> _Timing - """Emits a distribution with the time it takes to run the given code block. - - This method supports three forms of invocation: - - - when a `value` is provided, it functions similar to `distribution` but with - - it can be used as a context manager - - it can be used as a decorator - """ - if value is not None: - aggregator, local_aggregator, tags = _get_aggregator_and_update_tags( - key, value, unit, tags - ) - if aggregator is not None: - aggregator.add( - "d", key, value, unit, tags, timestamp, local_aggregator, stacklevel - ) - return _Timing(key, tags, timestamp, value, unit, stacklevel) - - -def distribution( - key, # type: str - value, # type: float - unit="none", # type: MeasurementUnit - tags=None, # type: Optional[MetricTags] - timestamp=None, # type: Optional[Union[float, datetime]] - stacklevel=0, # type: int -): - # type: (...) -> None - """Emits a distribution.""" - aggregator, local_aggregator, tags = _get_aggregator_and_update_tags( - key, value, unit, tags - ) - if aggregator is not None: - aggregator.add( - "d", key, value, unit, tags, timestamp, local_aggregator, stacklevel - ) - - -def set( - key, # type: str - value, # type: Union[int, str] - unit="none", # type: MeasurementUnit - tags=None, # type: Optional[MetricTags] - timestamp=None, # type: Optional[Union[float, datetime]] - stacklevel=0, # type: int -): - # type: (...) -> None - """Emits a set.""" - aggregator, local_aggregator, tags = _get_aggregator_and_update_tags( - key, value, unit, tags - ) - if aggregator is not None: - aggregator.add( - "s", key, value, unit, tags, timestamp, local_aggregator, stacklevel - ) - - -def gauge( - key, # type: str - value, # type: float - unit="none", # type: MeasurementUnit - tags=None, # type: Optional[MetricTags] - timestamp=None, # type: Optional[Union[float, datetime]] - stacklevel=0, # type: int -): - # type: (...) -> None - """Emits a gauge.""" - aggregator, local_aggregator, tags = _get_aggregator_and_update_tags( - key, value, unit, tags - ) - if aggregator is not None: - aggregator.add( - "g", key, value, unit, tags, timestamp, local_aggregator, stacklevel - ) diff --git a/sentry_sdk/tracing.py b/sentry_sdk/tracing.py index 1697df1f22..0d652e490a 100644 --- a/sentry_sdk/tracing.py +++ b/sentry_sdk/tracing.py @@ -276,7 +276,6 @@ class Span: "hub", "_context_manager_state", "_containing_transaction", - "_local_aggregator", "scope", "origin", "name", @@ -345,7 +344,6 @@ def __init__( self.timestamp = None # type: Optional[datetime] self._span_recorder = None # type: Optional[_SpanRecorder] - self._local_aggregator = None # type: Optional[LocalAggregator] self.update_active_thread() self.set_profiler_id(get_profiler_id()) @@ -383,13 +381,6 @@ def span_id(self, value): # type: (str) -> None self._span_id = value - def _get_local_aggregator(self): - # type: (...) -> LocalAggregator - rv = self._local_aggregator - if rv is None: - rv = self._local_aggregator = LocalAggregator() - return rv - def __repr__(self): # type: () -> str return ( @@ -741,11 +732,6 @@ def to_json(self): if self.status: self._tags["status"] = self.status - if self._local_aggregator is not None: - metrics_summary = self._local_aggregator.to_json() - if metrics_summary: - rv["_metrics_summary"] = metrics_summary - if len(self._measurements) > 0: rv["measurements"] = self._measurements @@ -1122,13 +1108,6 @@ def finish( event["measurements"] = self._measurements - # This is here since `to_json` is not invoked. This really should - # be gone when we switch to onlyspans. - if self._local_aggregator is not None: - metrics_summary = self._local_aggregator.to_json() - if metrics_summary: - event["_metrics_summary"] = metrics_summary - return scope.capture_event(event) def set_measurement(self, name, value, unit=""): @@ -1505,8 +1484,3 @@ def calculate_interest_rate(amount, rate, years): has_tracing_enabled, maybe_create_breadcrumbs_from_span, ) - -with warnings.catch_warnings(): - # The code in this file which uses `LocalAggregator` is only called from the deprecated `metrics` module. - warnings.simplefilter("ignore", DeprecationWarning) - from sentry_sdk.metrics import LocalAggregator diff --git a/sentry_sdk/transport.py b/sentry_sdk/transport.py index 75384519e9..645bfead19 100644 --- a/sentry_sdk/transport.py +++ b/sentry_sdk/transport.py @@ -171,17 +171,7 @@ def _parse_rate_limits(header, now=None): retry_after = now + timedelta(seconds=int(retry_after_val)) for category in categories and categories.split(";") or (None,): - if category == "metric_bucket": - try: - namespaces = parameters[4].split(";") - except IndexError: - namespaces = [] - - if not namespaces or "custom" in namespaces: - yield category, retry_after # type: ignore - - else: - yield category, retry_after # type: ignore + yield category, retry_after # type: ignore except (LookupError, ValueError): continue @@ -417,12 +407,6 @@ def _check_disabled(self, category): # type: (str) -> bool def _disabled(bucket): # type: (Any) -> bool - - # The envelope item type used for metrics is statsd - # whereas the rate limit category is metric_bucket - if bucket == "statsd": - bucket = "metric_bucket" - ts = self._disabled_until.get(bucket) return ts is not None and ts > datetime.now(timezone.utc) diff --git a/tests/test_envelope.py b/tests/test_envelope.py index 06f8971dc3..d66cd9460a 100644 --- a/tests/test_envelope.py +++ b/tests/test_envelope.py @@ -252,7 +252,6 @@ def test_envelope_item_data_category_mapping(): ("client_report", "internal"), ("profile", "profile"), ("profile_chunk", "profile_chunk"), - ("statsd", "metric_bucket"), ("check_in", "monitor"), ("unknown_type", "default"), ] diff --git a/tests/test_metrics.py b/tests/test_metrics.py deleted file mode 100644 index c02f075288..0000000000 --- a/tests/test_metrics.py +++ /dev/null @@ -1,971 +0,0 @@ -import sys -import time -import linecache -from unittest import mock - -import pytest - -import sentry_sdk -from sentry_sdk import metrics -from sentry_sdk.tracing import TransactionSource -from sentry_sdk.envelope import parse_json - -try: - import gevent -except ImportError: - gevent = None - - -minimum_python_37_with_gevent = pytest.mark.skipif( - gevent and sys.version_info < (3, 7), - reason="Require Python 3.7 or higher with gevent", -) - - -def parse_metrics(bytes): - rv = [] - for line in bytes.splitlines(): - pieces = line.decode("utf-8").split("|") - payload = pieces[0].split(":") - name = payload[0] - values = payload[1:] - ty = pieces[1] - ts = None - tags = {} - for piece in pieces[2:]: - if piece[0] == "#": - for pair in piece[1:].split(","): - k, v = pair.split(":", 1) - old = tags.get(k) - if old is not None: - if isinstance(old, list): - old.append(v) - else: - tags[k] = [old, v] - else: - tags[k] = v - elif piece[0] == "T": - ts = int(piece[1:]) - else: - raise ValueError("unknown piece %r" % (piece,)) - rv.append((ts, name, ty, values, tags)) - rv.sort(key=lambda x: (x[0], x[1], tuple(sorted(tags.items())))) - return rv - - -@minimum_python_37_with_gevent -@pytest.mark.forked -def test_increment(sentry_init, capture_envelopes, maybe_monkeypatched_threading): - sentry_init( - release="fun-release", - environment="not-fun-env", - _experiments={"enable_metrics": True, "metric_code_locations": True}, - ) - ts = time.time() - envelopes = capture_envelopes() - - metrics.increment("foobar", 1.0, tags={"foo": "bar", "blub": "blah"}, timestamp=ts) - # python specific alias - metrics.incr("foobar", 2.0, tags={"foo": "bar", "blub": "blah"}, timestamp=ts) - sentry_sdk.flush() - - (envelope,) = envelopes - statsd_item, meta_item = envelope.items - - assert statsd_item.headers["type"] == "statsd" - m = parse_metrics(statsd_item.payload.get_bytes()) - - assert len(m) == 1 - assert m[0][1] == "foobar@none" - assert m[0][2] == "c" - assert m[0][3] == ["3.0"] - assert m[0][4] == { - "blub": "blah", - "foo": "bar", - "release": "fun-release", - "environment": "not-fun-env", - } - - assert meta_item.headers["type"] == "metric_meta" - assert parse_json(meta_item.payload.get_bytes()) == { - "timestamp": mock.ANY, - "mapping": { - "c:foobar@none": [ - { - "type": "location", - "filename": "tests/test_metrics.py", - "abs_path": __file__, - "function": sys._getframe().f_code.co_name, - "module": __name__, - "lineno": mock.ANY, - "pre_context": mock.ANY, - "context_line": mock.ANY, - "post_context": mock.ANY, - } - ] - }, - } - - -@minimum_python_37_with_gevent -@pytest.mark.forked -def test_timing(sentry_init, capture_envelopes, maybe_monkeypatched_threading): - sentry_init( - release="fun-release@1.0.0", - environment="not-fun-env", - _experiments={"enable_metrics": True, "metric_code_locations": True}, - ) - ts = time.time() - envelopes = capture_envelopes() - - with metrics.timing("whatever", tags={"blub": "blah"}, timestamp=ts): - time.sleep(0.1) - sentry_sdk.flush() - - (envelope,) = envelopes - statsd_item, meta_item = envelope.items - - assert statsd_item.headers["type"] == "statsd" - m = parse_metrics(statsd_item.payload.get_bytes()) - - assert len(m) == 1 - assert m[0][1] == "whatever@second" - assert m[0][2] == "d" - assert len(m[0][3]) == 1 - assert float(m[0][3][0]) >= 0.1 - assert m[0][4] == { - "blub": "blah", - "release": "fun-release@1.0.0", - "environment": "not-fun-env", - } - - assert meta_item.headers["type"] == "metric_meta" - json = parse_json(meta_item.payload.get_bytes()) - assert json == { - "timestamp": mock.ANY, - "mapping": { - "d:whatever@second": [ - { - "type": "location", - "filename": "tests/test_metrics.py", - "abs_path": __file__, - "function": sys._getframe().f_code.co_name, - "module": __name__, - "lineno": mock.ANY, - "pre_context": mock.ANY, - "context_line": mock.ANY, - "post_context": mock.ANY, - } - ] - }, - } - - loc = json["mapping"]["d:whatever@second"][0] - line = linecache.getline(loc["abs_path"], loc["lineno"]) - assert ( - line.strip() - == 'with metrics.timing("whatever", tags={"blub": "blah"}, timestamp=ts):' - ) - - -@minimum_python_37_with_gevent -@pytest.mark.forked -def test_timing_decorator( - sentry_init, capture_envelopes, maybe_monkeypatched_threading -): - sentry_init( - release="fun-release@1.0.0", - environment="not-fun-env", - _experiments={"enable_metrics": True, "metric_code_locations": True}, - ) - envelopes = capture_envelopes() - - @metrics.timing("whatever-1", tags={"x": "y"}) - def amazing(): - time.sleep(0.1) - return 42 - - @metrics.timing("whatever-2", tags={"x": "y"}, unit="nanosecond") - def amazing_nano(): - time.sleep(0.01) - return 23 - - assert amazing() == 42 - assert amazing_nano() == 23 - sentry_sdk.flush() - - (envelope,) = envelopes - statsd_item, meta_item = envelope.items - - assert statsd_item.headers["type"] == "statsd" - m = parse_metrics(statsd_item.payload.get_bytes()) - - assert len(m) == 2 - assert m[0][1] == "whatever-1@second" - assert m[0][2] == "d" - assert len(m[0][3]) == 1 - assert float(m[0][3][0]) >= 0.1 - assert m[0][4] == { - "x": "y", - "release": "fun-release@1.0.0", - "environment": "not-fun-env", - } - - assert m[1][1] == "whatever-2@nanosecond" - assert m[1][2] == "d" - assert len(m[1][3]) == 1 - assert float(m[1][3][0]) >= 10000000.0 - assert m[1][4] == { - "x": "y", - "release": "fun-release@1.0.0", - "environment": "not-fun-env", - } - - assert meta_item.headers["type"] == "metric_meta" - json = parse_json(meta_item.payload.get_bytes()) - assert json == { - "timestamp": mock.ANY, - "mapping": { - "d:whatever-1@second": [ - { - "type": "location", - "filename": "tests/test_metrics.py", - "abs_path": __file__, - "function": sys._getframe().f_code.co_name, - "module": __name__, - "lineno": mock.ANY, - "pre_context": mock.ANY, - "context_line": mock.ANY, - "post_context": mock.ANY, - } - ], - "d:whatever-2@nanosecond": [ - { - "type": "location", - "filename": "tests/test_metrics.py", - "abs_path": __file__, - "function": sys._getframe().f_code.co_name, - "module": __name__, - "lineno": mock.ANY, - "pre_context": mock.ANY, - "context_line": mock.ANY, - "post_context": mock.ANY, - } - ], - }, - } - - # XXX: this is not the best location. It would probably be better to - # report the location in the function, however that is quite a bit - # tricker to do since we report from outside the function so we really - # only see the callsite. - loc = json["mapping"]["d:whatever-1@second"][0] - line = linecache.getline(loc["abs_path"], loc["lineno"]) - assert line.strip() == "assert amazing() == 42" - - -@minimum_python_37_with_gevent -@pytest.mark.forked -def test_timing_basic(sentry_init, capture_envelopes, maybe_monkeypatched_threading): - sentry_init( - release="fun-release@1.0.0", - environment="not-fun-env", - _experiments={"enable_metrics": True, "metric_code_locations": True}, - ) - ts = time.time() - envelopes = capture_envelopes() - - metrics.timing("timing", 1.0, tags={"a": "b"}, timestamp=ts) - metrics.timing("timing", 2.0, tags={"a": "b"}, timestamp=ts) - metrics.timing("timing", 2.0, tags={"a": "b"}, timestamp=ts) - metrics.timing("timing", 3.0, tags={"a": "b"}, timestamp=ts) - sentry_sdk.flush() - - (envelope,) = envelopes - statsd_item, meta_item = envelope.items - - assert statsd_item.headers["type"] == "statsd" - m = parse_metrics(statsd_item.payload.get_bytes()) - - assert len(m) == 1 - assert m[0][1] == "timing@second" - assert m[0][2] == "d" - assert len(m[0][3]) == 4 - assert sorted(map(float, m[0][3])) == [1.0, 2.0, 2.0, 3.0] - assert m[0][4] == { - "a": "b", - "release": "fun-release@1.0.0", - "environment": "not-fun-env", - } - - assert meta_item.headers["type"] == "metric_meta" - assert parse_json(meta_item.payload.get_bytes()) == { - "timestamp": mock.ANY, - "mapping": { - "d:timing@second": [ - { - "type": "location", - "filename": "tests/test_metrics.py", - "abs_path": __file__, - "function": sys._getframe().f_code.co_name, - "module": __name__, - "lineno": mock.ANY, - "pre_context": mock.ANY, - "context_line": mock.ANY, - "post_context": mock.ANY, - } - ] - }, - } - - -@minimum_python_37_with_gevent -@pytest.mark.forked -def test_distribution(sentry_init, capture_envelopes, maybe_monkeypatched_threading): - sentry_init( - release="fun-release@1.0.0", - environment="not-fun-env", - _experiments={"enable_metrics": True, "metric_code_locations": True}, - ) - ts = time.time() - envelopes = capture_envelopes() - - metrics.distribution("dist", 1.0, tags={"a": "b"}, timestamp=ts) - metrics.distribution("dist", 2.0, tags={"a": "b"}, timestamp=ts) - metrics.distribution("dist", 2.0, tags={"a": "b"}, timestamp=ts) - metrics.distribution("dist", 3.0, tags={"a": "b"}, timestamp=ts) - sentry_sdk.flush() - - (envelope,) = envelopes - statsd_item, meta_item = envelope.items - - assert statsd_item.headers["type"] == "statsd" - m = parse_metrics(statsd_item.payload.get_bytes()) - - assert len(m) == 1 - assert m[0][1] == "dist@none" - assert m[0][2] == "d" - assert len(m[0][3]) == 4 - assert sorted(map(float, m[0][3])) == [1.0, 2.0, 2.0, 3.0] - assert m[0][4] == { - "a": "b", - "release": "fun-release@1.0.0", - "environment": "not-fun-env", - } - - assert meta_item.headers["type"] == "metric_meta" - json = parse_json(meta_item.payload.get_bytes()) - assert json == { - "timestamp": mock.ANY, - "mapping": { - "d:dist@none": [ - { - "type": "location", - "filename": "tests/test_metrics.py", - "abs_path": __file__, - "function": sys._getframe().f_code.co_name, - "module": __name__, - "lineno": mock.ANY, - "pre_context": mock.ANY, - "context_line": mock.ANY, - "post_context": mock.ANY, - } - ] - }, - } - - loc = json["mapping"]["d:dist@none"][0] - line = linecache.getline(loc["abs_path"], loc["lineno"]) - assert ( - line.strip() - == 'metrics.distribution("dist", 1.0, tags={"a": "b"}, timestamp=ts)' - ) - - -@minimum_python_37_with_gevent -@pytest.mark.forked -def test_set(sentry_init, capture_envelopes, maybe_monkeypatched_threading): - sentry_init( - release="fun-release@1.0.0", - environment="not-fun-env", - _experiments={"enable_metrics": True, "metric_code_locations": True}, - ) - ts = time.time() - envelopes = capture_envelopes() - - metrics.set("my-set", "peter", tags={"magic": "puff"}, timestamp=ts) - metrics.set("my-set", "paul", tags={"magic": "puff"}, timestamp=ts) - metrics.set("my-set", "mary", tags={"magic": "puff"}, timestamp=ts) - sentry_sdk.flush() - - (envelope,) = envelopes - statsd_item, meta_item = envelope.items - - assert statsd_item.headers["type"] == "statsd" - m = parse_metrics(statsd_item.payload.get_bytes()) - - assert len(m) == 1 - assert m[0][1] == "my-set@none" - assert m[0][2] == "s" - assert len(m[0][3]) == 3 - assert sorted(map(int, m[0][3])) == [354582103, 2513273657, 3329318813] - assert m[0][4] == { - "magic": "puff", - "release": "fun-release@1.0.0", - "environment": "not-fun-env", - } - - assert meta_item.headers["type"] == "metric_meta" - assert parse_json(meta_item.payload.get_bytes()) == { - "timestamp": mock.ANY, - "mapping": { - "s:my-set@none": [ - { - "type": "location", - "filename": "tests/test_metrics.py", - "abs_path": __file__, - "function": sys._getframe().f_code.co_name, - "module": __name__, - "lineno": mock.ANY, - "pre_context": mock.ANY, - "context_line": mock.ANY, - "post_context": mock.ANY, - } - ] - }, - } - - -@minimum_python_37_with_gevent -@pytest.mark.forked -def test_gauge(sentry_init, capture_envelopes, maybe_monkeypatched_threading): - sentry_init( - release="fun-release@1.0.0", - environment="not-fun-env", - _experiments={"enable_metrics": True, "metric_code_locations": False}, - ) - ts = time.time() - envelopes = capture_envelopes() - - metrics.gauge("my-gauge", 10.0, tags={"x": "y"}, timestamp=ts) - metrics.gauge("my-gauge", 20.0, tags={"x": "y"}, timestamp=ts) - metrics.gauge("my-gauge", 30.0, tags={"x": "y"}, timestamp=ts) - sentry_sdk.flush() - - (envelope,) = envelopes - - assert len(envelope.items) == 1 - assert envelope.items[0].headers["type"] == "statsd" - m = parse_metrics(envelope.items[0].payload.get_bytes()) - - assert len(m) == 1 - assert m[0][1] == "my-gauge@none" - assert m[0][2] == "g" - assert len(m[0][3]) == 5 - assert list(map(float, m[0][3])) == [30.0, 10.0, 30.0, 60.0, 3.0] - assert m[0][4] == { - "x": "y", - "release": "fun-release@1.0.0", - "environment": "not-fun-env", - } - - -@minimum_python_37_with_gevent -@pytest.mark.forked -def test_multiple(sentry_init, capture_envelopes): - sentry_init( - release="fun-release@1.0.0", - environment="not-fun-env", - _experiments={"enable_metrics": True, "metric_code_locations": False}, - ) - ts = time.time() - envelopes = capture_envelopes() - - metrics.gauge("my-gauge", 10.0, tags={"x": "y"}, timestamp=ts) - metrics.gauge("my-gauge", 20.0, tags={"x": "y"}, timestamp=ts) - metrics.gauge("my-gauge", 30.0, tags={"x": "y"}, timestamp=ts) - for _ in range(10): - metrics.increment("counter-1", 1.0, timestamp=ts) - metrics.increment("counter-2", 1.0, timestamp=ts) - - sentry_sdk.flush() - - (envelope,) = envelopes - - assert len(envelope.items) == 1 - assert envelope.items[0].headers["type"] == "statsd" - m = parse_metrics(envelope.items[0].payload.get_bytes()) - - assert len(m) == 3 - - assert m[0][1] == "counter-1@none" - assert m[0][2] == "c" - assert list(map(float, m[0][3])) == [10.0] - assert m[0][4] == { - "release": "fun-release@1.0.0", - "environment": "not-fun-env", - } - - assert m[1][1] == "counter-2@none" - assert m[1][2] == "c" - assert list(map(float, m[1][3])) == [1.0] - assert m[1][4] == { - "release": "fun-release@1.0.0", - "environment": "not-fun-env", - } - - assert m[2][1] == "my-gauge@none" - assert m[2][2] == "g" - assert len(m[2][3]) == 5 - assert list(map(float, m[2][3])) == [30.0, 10.0, 30.0, 60.0, 3.0] - assert m[2][4] == { - "x": "y", - "release": "fun-release@1.0.0", - "environment": "not-fun-env", - } - - -@minimum_python_37_with_gevent -@pytest.mark.forked -def test_transaction_name( - sentry_init, capture_envelopes, maybe_monkeypatched_threading -): - sentry_init( - release="fun-release@1.0.0", - environment="not-fun-env", - _experiments={"enable_metrics": True, "metric_code_locations": False}, - ) - ts = time.time() - envelopes = capture_envelopes() - - sentry_sdk.get_current_scope().set_transaction_name( - "/user/{user_id}", source=TransactionSource.ROUTE - ) - metrics.distribution("dist", 1.0, tags={"a": "b"}, timestamp=ts) - metrics.distribution("dist", 2.0, tags={"a": "b"}, timestamp=ts) - metrics.distribution("dist", 2.0, tags={"a": "b"}, timestamp=ts) - metrics.distribution("dist", 3.0, tags={"a": "b"}, timestamp=ts) - - sentry_sdk.flush() - - (envelope,) = envelopes - - assert len(envelope.items) == 1 - assert envelope.items[0].headers["type"] == "statsd" - m = parse_metrics(envelope.items[0].payload.get_bytes()) - - assert len(m) == 1 - assert m[0][1] == "dist@none" - assert m[0][2] == "d" - assert len(m[0][3]) == 4 - assert sorted(map(float, m[0][3])) == [1.0, 2.0, 2.0, 3.0] - assert m[0][4] == { - "a": "b", - "transaction": "/user/{user_id}", - "release": "fun-release@1.0.0", - "environment": "not-fun-env", - } - - -@minimum_python_37_with_gevent -@pytest.mark.forked -def test_metric_summaries( - sentry_init, capture_envelopes, maybe_monkeypatched_threading -): - sentry_init( - release="fun-release@1.0.0", - environment="not-fun-env", - enable_tracing=True, - ) - ts = time.time() - envelopes = capture_envelopes() - - with sentry_sdk.start_transaction( - op="stuff", name="/foo", source=TransactionSource.ROUTE - ) as transaction: - metrics.increment("root-counter", timestamp=ts) - with metrics.timing("my-timer-metric", tags={"a": "b"}, timestamp=ts): - for x in range(10): - metrics.distribution("my-dist", float(x), timestamp=ts) - - sentry_sdk.flush() - - (transaction, envelope) = envelopes - - # Metrics Emission - assert envelope.items[0].headers["type"] == "statsd" - m = parse_metrics(envelope.items[0].payload.get_bytes()) - - assert len(m) == 3 - - assert m[0][1] == "my-dist@none" - assert m[0][2] == "d" - assert len(m[0][3]) == 10 - assert sorted(m[0][3]) == list(map(str, map(float, range(10)))) - assert m[0][4] == { - "transaction": "/foo", - "release": "fun-release@1.0.0", - "environment": "not-fun-env", - } - - assert m[1][1] == "my-timer-metric@second" - assert m[1][2] == "d" - assert len(m[1][3]) == 1 - assert m[1][4] == { - "a": "b", - "transaction": "/foo", - "release": "fun-release@1.0.0", - "environment": "not-fun-env", - } - - assert m[2][1] == "root-counter@none" - assert m[2][2] == "c" - assert m[2][3] == ["1.0"] - assert m[2][4] == { - "transaction": "/foo", - "release": "fun-release@1.0.0", - "environment": "not-fun-env", - } - - # Measurement Attachment - t = transaction.items[0].get_transaction_event() - - assert t["_metrics_summary"] == { - "c:root-counter@none": [ - { - "count": 1, - "min": 1.0, - "max": 1.0, - "sum": 1.0, - "tags": { - "transaction": "/foo", - "release": "fun-release@1.0.0", - "environment": "not-fun-env", - }, - } - ] - } - - assert t["spans"][0]["_metrics_summary"]["d:my-dist@none"] == [ - { - "count": 10, - "min": 0.0, - "max": 9.0, - "sum": 45.0, - "tags": { - "environment": "not-fun-env", - "release": "fun-release@1.0.0", - "transaction": "/foo", - }, - } - ] - - assert t["spans"][0]["tags"] == {"a": "b"} - (timer,) = t["spans"][0]["_metrics_summary"]["d:my-timer-metric@second"] - assert timer["count"] == 1 - assert timer["max"] == timer["min"] == timer["sum"] - assert timer["sum"] > 0 - assert timer["tags"] == { - "a": "b", - "environment": "not-fun-env", - "release": "fun-release@1.0.0", - "transaction": "/foo", - } - - -@minimum_python_37_with_gevent -@pytest.mark.forked -@pytest.mark.parametrize( - "metric_name,metric_unit,expected_name", - [ - ("first-metric", "nano-second", "first-metric@nanosecond"), - ("another_metric?", "nano second", "another_metric_@nanosecond"), - ( - "metric", - "nanosecond", - "metric@nanosecond", - ), - ( - "my.amaze.metric I guess", - "nano|\nsecond", - "my.amaze.metric_I_guess@nanosecond", - ), - ("métríc", "nanöseconď", "m_tr_c@nansecon"), - ], -) -def test_metric_name_normalization( - sentry_init, - capture_envelopes, - metric_name, - metric_unit, - expected_name, - maybe_monkeypatched_threading, -): - sentry_init( - _experiments={"enable_metrics": True, "metric_code_locations": False}, - ) - envelopes = capture_envelopes() - - metrics.distribution(metric_name, 1.0, unit=metric_unit) - - sentry_sdk.flush() - - (envelope,) = envelopes - - assert len(envelope.items) == 1 - assert envelope.items[0].headers["type"] == "statsd" - - parsed_metrics = parse_metrics(envelope.items[0].payload.get_bytes()) - assert len(parsed_metrics) == 1 - - name = parsed_metrics[0][1] - assert name == expected_name - - -@minimum_python_37_with_gevent -@pytest.mark.forked -@pytest.mark.parametrize( - "metric_tag,expected_tag", - [ - ({"f-oo|bar": "%$foo/"}, {"f-oobar": "%$foo/"}), - ({"foo$.$.$bar": "blah{}"}, {"foo..bar": "blah{}"}), - ( - {"foö-bar": "snöwmän"}, - {"fo-bar": "snöwmän"}, - ), - ({"route": "GET /foo"}, {"route": "GET /foo"}), - ({"__bar__": "this | or , that"}, {"__bar__": "this \\u{7c} or \\u{2c} that"}), - ({"foo/": "hello!\n\r\t\\"}, {"foo/": "hello!\\n\\r\\t\\\\"}), - ], -) -def test_metric_tag_normalization( - sentry_init, - capture_envelopes, - metric_tag, - expected_tag, - maybe_monkeypatched_threading, -): - sentry_init( - _experiments={"enable_metrics": True, "metric_code_locations": False}, - ) - envelopes = capture_envelopes() - - metrics.distribution("a", 1.0, tags=metric_tag) - - sentry_sdk.flush() - - (envelope,) = envelopes - - assert len(envelope.items) == 1 - assert envelope.items[0].headers["type"] == "statsd" - - parsed_metrics = parse_metrics(envelope.items[0].payload.get_bytes()) - assert len(parsed_metrics) == 1 - - tags = parsed_metrics[0][4] - - expected_tag_key, expected_tag_value = expected_tag.popitem() - assert expected_tag_key in tags - assert tags[expected_tag_key] == expected_tag_value - - -@minimum_python_37_with_gevent -@pytest.mark.forked -def test_before_emit_metric( - sentry_init, capture_envelopes, maybe_monkeypatched_threading -): - def before_emit(key, value, unit, tags): - if key == "removed-metric" or value == 47 or unit == "unsupported": - return False - - tags["extra"] = "foo" - del tags["release"] - # this better be a noop! - metrics.increment("shitty-recursion") - return True - - sentry_init( - release="fun-release@1.0.0", - environment="not-fun-env", - _experiments={ - "enable_metrics": True, - "metric_code_locations": False, - "before_emit_metric": before_emit, - }, - ) - envelopes = capture_envelopes() - - metrics.increment("removed-metric", 1.0) - metrics.increment("another-removed-metric", 47) - metrics.increment("yet-another-removed-metric", 1.0, unit="unsupported") - metrics.increment("actual-metric", 1.0) - sentry_sdk.flush() - - (envelope,) = envelopes - - assert len(envelope.items) == 1 - assert envelope.items[0].headers["type"] == "statsd" - m = parse_metrics(envelope.items[0].payload.get_bytes()) - - assert len(m) == 1 - assert m[0][1] == "actual-metric@none" - assert m[0][3] == ["1.0"] - assert m[0][4] == { - "extra": "foo", - "environment": "not-fun-env", - } - - -@minimum_python_37_with_gevent -@pytest.mark.forked -def test_aggregator_flush( - sentry_init, capture_envelopes, maybe_monkeypatched_threading -): - sentry_init( - release="fun-release@1.0.0", - environment="not-fun-env", - _experiments={ - "enable_metrics": True, - }, - ) - envelopes = capture_envelopes() - - metrics.increment("a-metric", 1.0) - sentry_sdk.flush() - - assert len(envelopes) == 1 - assert sentry_sdk.get_client().metrics_aggregator.buckets == {} - - -@minimum_python_37_with_gevent -@pytest.mark.forked -def test_tag_serialization( - sentry_init, capture_envelopes, maybe_monkeypatched_threading -): - sentry_init( - release="fun-release", - environment="not-fun-env", - _experiments={"enable_metrics": True, "metric_code_locations": False}, - ) - envelopes = capture_envelopes() - - metrics.increment( - "counter", - tags={ - "no-value": None, - "an-int": 42, - "a-float": 23.0, - "a-string": "blah", - "more-than-one": [1, "zwei", "3.0", None], - }, - ) - sentry_sdk.flush() - - (envelope,) = envelopes - - assert len(envelope.items) == 1 - assert envelope.items[0].headers["type"] == "statsd" - m = parse_metrics(envelope.items[0].payload.get_bytes()) - - assert len(m) == 1 - assert m[0][4] == { - "an-int": "42", - "a-float": "23.0", - "a-string": "blah", - "more-than-one": ["1", "3.0", "zwei"], - "release": "fun-release", - "environment": "not-fun-env", - } - - -@minimum_python_37_with_gevent -@pytest.mark.forked -def test_flush_recursion_protection( - sentry_init, capture_envelopes, monkeypatch, maybe_monkeypatched_threading -): - sentry_init( - release="fun-release", - environment="not-fun-env", - _experiments={"enable_metrics": True}, - ) - envelopes = capture_envelopes() - test_client = sentry_sdk.get_client() - - real_capture_envelope = test_client.transport.capture_envelope - - def bad_capture_envelope(*args, **kwargs): - metrics.increment("bad-metric") - return real_capture_envelope(*args, **kwargs) - - monkeypatch.setattr(test_client.transport, "capture_envelope", bad_capture_envelope) - - metrics.increment("counter") - - # flush twice to see the inner metric - sentry_sdk.flush() - sentry_sdk.flush() - - (envelope,) = envelopes - m = parse_metrics(envelope.items[0].payload.get_bytes()) - assert len(m) == 1 - assert m[0][1] == "counter@none" - - -@minimum_python_37_with_gevent -@pytest.mark.forked -def test_flush_recursion_protection_background_flush( - sentry_init, capture_envelopes, monkeypatch, maybe_monkeypatched_threading -): - monkeypatch.setattr(metrics.MetricsAggregator, "FLUSHER_SLEEP_TIME", 0.01) - sentry_init( - release="fun-release", - environment="not-fun-env", - _experiments={"enable_metrics": True}, - ) - envelopes = capture_envelopes() - test_client = sentry_sdk.get_client() - - real_capture_envelope = test_client.transport.capture_envelope - - def bad_capture_envelope(*args, **kwargs): - metrics.increment("bad-metric") - return real_capture_envelope(*args, **kwargs) - - monkeypatch.setattr(test_client.transport, "capture_envelope", bad_capture_envelope) - - metrics.increment("counter") - - # flush via sleep and flag - sentry_sdk.get_client().metrics_aggregator._force_flush = True - time.sleep(0.5) - - (envelope,) = envelopes - m = parse_metrics(envelope.items[0].payload.get_bytes()) - assert len(m) == 1 - assert m[0][1] == "counter@none" - - -@pytest.mark.skipif( - not gevent or sys.version_info >= (3, 7), - reason="Python 3.6 or lower and gevent required", -) -@pytest.mark.forked -def test_disable_metrics_for_old_python_with_gevent( - sentry_init, capture_envelopes, maybe_monkeypatched_threading -): - if maybe_monkeypatched_threading != "greenlet": - pytest.skip("Test specifically for gevent/greenlet") - - sentry_init( - release="fun-release", - environment="not-fun-env", - _experiments={"enable_metrics": True}, - ) - envelopes = capture_envelopes() - - metrics.incr("counter") - - sentry_sdk.flush() - - assert sentry_sdk.get_client().metrics_aggregator is None - assert not envelopes diff --git a/tests/test_transport.py b/tests/test_transport.py index 68669fa24d..804105b010 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -590,43 +590,6 @@ def test_complex_limits_without_data_category( assert len(capturing_server.captured) == 0 -@pytest.mark.parametrize("response_code", [200, 429]) -def test_metric_bucket_limits(capturing_server, response_code, make_client): - client = make_client() - capturing_server.respond_with( - code=response_code, - headers={ - "X-Sentry-Rate-Limits": "4711:metric_bucket:organization:quota_exceeded:custom" - }, - ) - - envelope = Envelope() - envelope.add_item(Item(payload=b"{}", type="statsd")) - client.transport.capture_envelope(envelope) - client.flush() - - assert len(capturing_server.captured) == 1 - assert capturing_server.captured[0].path == "/api/132/envelope/" - capturing_server.clear_captured() - - assert set(client.transport._disabled_until) == {"metric_bucket"} - - client.transport.capture_envelope(envelope) - client.capture_event({"type": "transaction"}) - client.flush() - - assert len(capturing_server.captured) == 2 - - envelope = capturing_server.captured[0].envelope - assert envelope.items[0].type == "transaction" - envelope = capturing_server.captured[1].envelope - assert envelope.items[0].type == "client_report" - report = parse_json(envelope.items[0].get_bytes()) - assert report["discarded_events"] == [ - {"category": "metric_bucket", "reason": "ratelimit_backoff", "quantity": 1}, - ] - - @pytest.mark.parametrize("response_code", [200, 429]) def test_log_item_limits(capturing_server, response_code, make_client): client = make_client() @@ -664,80 +627,6 @@ def test_log_item_limits(capturing_server, response_code, make_client): ] -@pytest.mark.parametrize("response_code", [200, 429]) -def test_metric_bucket_limits_with_namespace( - capturing_server, response_code, make_client -): - client = make_client() - capturing_server.respond_with( - code=response_code, - headers={ - "X-Sentry-Rate-Limits": "4711:metric_bucket:organization:quota_exceeded:foo" - }, - ) - - envelope = Envelope() - envelope.add_item(Item(payload=b"{}", type="statsd")) - client.transport.capture_envelope(envelope) - client.flush() - - assert len(capturing_server.captured) == 1 - assert capturing_server.captured[0].path == "/api/132/envelope/" - capturing_server.clear_captured() - - assert set(client.transport._disabled_until) == set([]) - - client.transport.capture_envelope(envelope) - client.capture_event({"type": "transaction"}) - client.flush() - - assert len(capturing_server.captured) == 2 - - envelope = capturing_server.captured[0].envelope - assert envelope.items[0].type == "statsd" - envelope = capturing_server.captured[1].envelope - assert envelope.items[0].type == "transaction" - - -@pytest.mark.parametrize("response_code", [200, 429]) -def test_metric_bucket_limits_with_all_namespaces( - capturing_server, response_code, make_client -): - client = make_client() - capturing_server.respond_with( - code=response_code, - headers={ - "X-Sentry-Rate-Limits": "4711:metric_bucket:organization:quota_exceeded" - }, - ) - - envelope = Envelope() - envelope.add_item(Item(payload=b"{}", type="statsd")) - client.transport.capture_envelope(envelope) - client.flush() - - assert len(capturing_server.captured) == 1 - assert capturing_server.captured[0].path == "/api/132/envelope/" - capturing_server.clear_captured() - - assert set(client.transport._disabled_until) == set(["metric_bucket"]) - - client.transport.capture_envelope(envelope) - client.capture_event({"type": "transaction"}) - client.flush() - - assert len(capturing_server.captured) == 2 - - envelope = capturing_server.captured[0].envelope - assert envelope.items[0].type == "transaction" - envelope = capturing_server.captured[1].envelope - assert envelope.items[0].type == "client_report" - report = parse_json(envelope.items[0].get_bytes()) - assert report["discarded_events"] == [ - {"category": "metric_bucket", "reason": "ratelimit_backoff", "quantity": 1}, - ] - - def test_hub_cls_backwards_compat(): class TestCustomHubClass(Hub): pass