From 1f39f827640e43e458fc983231d2b0fd182b5795 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 25 Nov 2025 09:45:48 +0100 Subject: [PATCH 1/5] feat(logs): Record discarded log bytes --- sentry_sdk/_log_batcher.py | 1 + sentry_sdk/transport.py | 5 ++ tests/test_logs.py | 52 ++++++++++++++- tests/test_transport.py | 128 +++++++++++++++++++++++++++++++++++-- 4 files changed, 178 insertions(+), 8 deletions(-) diff --git a/sentry_sdk/_log_batcher.py b/sentry_sdk/_log_batcher.py index f7f6c80565..f0c1decf73 100644 --- a/sentry_sdk/_log_batcher.py +++ b/sentry_sdk/_log_batcher.py @@ -86,6 +86,7 @@ def add( self._record_lost_func( reason="queue_overflow", data_category="log_item", + item=LogBatcher._log_to_transport_format(log), quantity=1, ) return None diff --git a/sentry_sdk/transport.py b/sentry_sdk/transport.py index 645bfead19..78e4cd21c6 100644 --- a/sentry_sdk/transport.py +++ b/sentry_sdk/transport.py @@ -266,6 +266,11 @@ def record_lost_event( ) self.record_lost_event(reason, "span", quantity=span_count) + elif data_category == "log_item" and item: + # Also record size of lost logs in bytes + bytes_size = len(item.get_bytes()) + self.record_lost_event(reason, "log_byte", quantity=bytes_size) + elif data_category == "attachment": # quantity of 0 is actually 1 as we do not want to count # empty attachments as actually empty. diff --git a/tests/test_logs.py b/tests/test_logs.py index 6c0a9b14f9..d68f31aac8 100644 --- a/tests/test_logs.py +++ b/tests/test_logs.py @@ -4,6 +4,7 @@ import time from typing import List, Any, Mapping, Union import pytest +from unittest import mock import sentry_sdk import sentry_sdk.logger @@ -450,7 +451,7 @@ def test_logs_with_literal_braces( @minimum_python_37 def test_batcher_drops_logs(sentry_init, monkeypatch): - sentry_init(enable_logs=True) + sentry_init(enable_logs=True, server_name="test-server", release="1.0.0") client = sentry_sdk.get_client() def no_op_flush(): @@ -469,5 +470,52 @@ def record_lost_event(reason, data_category=None, item=None, *, quantity=1): sentry_sdk.logger.info("This is a 'info' log...") assert len(lost_event_calls) == 5 + + expected_dropped_item = { + "body": "This is a 'info' log...", + "level": "info", + "timestamp": mock.ANY, + "trace_id": mock.ANY, + "attributes": { + "sentry.environment": { + "type": "string", + "value": "production", + }, + "sentry.release": { + "type": "string", + "value": "1.0.0", + }, + "sentry.sdk.name": { + "type": "string", + "value": mock.ANY, + }, + "sentry.sdk.version": { + "type": "string", + "value": VERSION, + }, + "sentry.severity_number": { + "type": "integer", + "value": 9, + }, + "sentry.severity_text": { + "type": "string", + "value": "info", + }, + "sentry.trace.parent_span_id": { + "type": "string", + "value": mock.ANY, + }, + "server.address": { + "type": "string", + "value": "test-server", + }, + }, + } + for lost_event_call in lost_event_calls: - assert lost_event_call == ("queue_overflow", "log_item", None, 1) + assert lost_event_call == ( + "queue_overflow", + "log_item", + expected_dropped_item, + 1, + ) diff --git a/tests/test_transport.py b/tests/test_transport.py index 804105b010..fc64a1e53c 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -25,7 +25,7 @@ Hub, ) from sentry_sdk._compat import PY37, PY38 -from sentry_sdk.envelope import Envelope, Item, parse_json +from sentry_sdk.envelope import Envelope, Item, parse_json, PayloadRef from sentry_sdk.transport import ( KEEP_ALIVE_SOCKET_OPTIONS, _parse_rate_limits, @@ -591,7 +591,110 @@ def test_complex_limits_without_data_category( @pytest.mark.parametrize("response_code", [200, 429]) -def test_log_item_limits(capturing_server, response_code, make_client): +@pytest.mark.parametrize( + "item", + [ + Item(payload=b"{}", type="log"), + Item( + type="log", + content_type="application/vnd.sentry.items.log+json", + headers={ + "item_count": 2, + }, + payload=PayloadRef( + json={ + "items": [ + { + "body": "This is a 'info' log...", + "level": "info", + "timestamp": datetime( + 2025, 1, 1, tzinfo=timezone.utc + ).timestamp(), + "trace_id": "00000000-0000-0000-0000-000000000000", + "attributes": { + "sentry.environment": { + "value": "production", + "type": "string", + }, + "sentry.release": { + "value": "1.0.0", + "type": "string", + }, + "sentry.sdk.name": { + "value": "sentry.python", + "type": "string", + }, + "sentry.sdk.version": { + "value": "2.45.0", + "type": "string", + }, + "sentry.severity_number": { + "value": 9, + "type": "integer", + }, + "sentry.severity_text": { + "value": "info", + "type": "string", + }, + "server.address": { + "value": "test-server", + "type": "string", + }, + }, + }, + { + "body": "The recorded value was '2.0'", + "level": "warn", + "timestamp": datetime( + 2025, 1, 1, tzinfo=timezone.utc + ).timestamp(), + "trace_id": "00000000-0000-0000-0000-000000000000", + "attributes": { + "sentry.message.parameter.float_var": { + "value": 2.0, + "type": "double", + }, + "sentry.message.template": { + "value": "The recorded value was '{float_var}'", + "type": "string", + }, + "sentry.sdk.name": { + "value": "sentry.python", + "type": "string", + }, + "sentry.sdk.version": { + "value": "2.45.0", + "type": "string", + }, + "server.address": { + "value": "test-server", + "type": "string", + }, + "sentry.environment": { + "value": "production", + "type": "string", + }, + "sentry.release": { + "value": "1.0.0", + "type": "string", + }, + "sentry.severity_number": { + "value": 13, + "type": "integer", + }, + "sentry.severity_text": { + "value": "warn", + "type": "string", + }, + }, + }, + ] + } + ), + ), + ], +) +def test_log_item_limits(capturing_server, response_code, item, make_client): client = make_client() capturing_server.respond_with( code=response_code, @@ -601,7 +704,7 @@ def test_log_item_limits(capturing_server, response_code, make_client): ) envelope = Envelope() - envelope.add_item(Item(payload=b"{}", type="log")) + envelope.add_item(item) client.transport.capture_envelope(envelope) client.flush() @@ -622,9 +725,22 @@ def test_log_item_limits(capturing_server, response_code, make_client): envelope = capturing_server.captured[1].envelope assert envelope.items[0].type == "client_report" report = parse_json(envelope.items[0].get_bytes()) - assert report["discarded_events"] == [ - {"category": "log_item", "reason": "ratelimit_backoff", "quantity": 1}, - ] + + assert { + "category": "log_item", + "reason": "ratelimit_backoff", + "quantity": 1, + } in report["discarded_events"] + + expected_lost_bytes = 1243 + if item.payload.bytes == b"{}": + expected_lost_bytes = 2 + + assert { + "category": "log_byte", + "reason": "ratelimit_backoff", + "quantity": expected_lost_bytes, + } in report["discarded_events"] def test_hub_cls_backwards_compat(): From a61f8d6e623a7dbf8b7463c0c50cac081833d8aa Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 25 Nov 2025 09:48:57 +0100 Subject: [PATCH 2/5] add log_byte to EventDataCategory --- sentry_sdk/_types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sentry_sdk/_types.py b/sentry_sdk/_types.py index 66ed7df4f7..0426bf7a93 100644 --- a/sentry_sdk/_types.py +++ b/sentry_sdk/_types.py @@ -294,6 +294,7 @@ class SDKInfo(TypedDict): "monitor", "span", "log_item", + "log_byte", "trace_metric", ] SessionStatus = Literal["ok", "exited", "crashed", "abnormal"] From 5c38edb7cc691bbeea4644e11aa3435e522f17d8 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 25 Nov 2025 11:42:10 +0100 Subject: [PATCH 3/5] Item instead of dictionary argument --- sentry_sdk/_log_batcher.py | 13 ++++- sentry_sdk/client.py | 12 ++++- tests/test_logs.py | 105 ++++++++++++++++++++----------------- 3 files changed, 80 insertions(+), 50 deletions(-) diff --git a/sentry_sdk/_log_batcher.py b/sentry_sdk/_log_batcher.py index f0c1decf73..6d793aceb7 100644 --- a/sentry_sdk/_log_batcher.py +++ b/sentry_sdk/_log_batcher.py @@ -83,10 +83,21 @@ def add( with self._lock: if len(self._log_buffer) >= self.MAX_LOGS_BEFORE_DROP: + # Construct log envelope item without sending it to report lost bytes + log_item = Item( + type="log", + content_type="application/vnd.sentry.items.log+json", + headers={ + "item_count": 1, + }, + payload=PayloadRef( + json={"items": [LogBatcher._log_to_transport_format(log)]} + ), + ) self._record_lost_func( reason="queue_overflow", data_category="log_item", - item=LogBatcher._log_to_transport_format(log), + item=log_item, quantity=1, ) return None diff --git a/sentry_sdk/client.py b/sentry_sdk/client.py index 928fc3ea8b..e6ef7ffa6c 100644 --- a/sentry_sdk/client.py +++ b/sentry_sdk/client.py @@ -62,7 +62,15 @@ from typing import Union from typing import TypeVar - from sentry_sdk._types import Event, Hint, SDKInfo, Log, Metric, EventDataCategory + from sentry_sdk._types import ( + Event, + Hint, + SDKInfo, + Log, + Metric, + EventDataCategory, + Item, + ) from sentry_sdk.integrations import Integration from sentry_sdk.scope import Scope from sentry_sdk.session import Session @@ -360,6 +368,7 @@ def _capture_envelope(envelope): def _record_lost_event( reason, # type: str data_category, # type: EventDataCategory + item, # type: Item quantity=1, # type: int ): # type: (...) -> None @@ -367,6 +376,7 @@ def _record_lost_event( self.transport.record_lost_event( reason=reason, data_category=data_category, + item=item, quantity=quantity, ) diff --git a/tests/test_logs.py b/tests/test_logs.py index d68f31aac8..15baa9328b 100644 --- a/tests/test_logs.py +++ b/tests/test_logs.py @@ -9,7 +9,7 @@ import sentry_sdk import sentry_sdk.logger from sentry_sdk import get_client -from sentry_sdk.envelope import Envelope +from sentry_sdk.envelope import Envelope, Item, PayloadRef from sentry_sdk.types import Log from sentry_sdk.consts import SPANDATA, VERSION @@ -471,51 +471,60 @@ def record_lost_event(reason, data_category=None, item=None, *, quantity=1): assert len(lost_event_calls) == 5 - expected_dropped_item = { - "body": "This is a 'info' log...", - "level": "info", - "timestamp": mock.ANY, - "trace_id": mock.ANY, - "attributes": { - "sentry.environment": { - "type": "string", - "value": "production", - }, - "sentry.release": { - "type": "string", - "value": "1.0.0", - }, - "sentry.sdk.name": { - "type": "string", - "value": mock.ANY, - }, - "sentry.sdk.version": { - "type": "string", - "value": VERSION, - }, - "sentry.severity_number": { - "type": "integer", - "value": 9, - }, - "sentry.severity_text": { - "type": "string", - "value": "info", - }, - "sentry.trace.parent_span_id": { - "type": "string", - "value": mock.ANY, - }, - "server.address": { - "type": "string", - "value": "test-server", - }, - }, - } - for lost_event_call in lost_event_calls: - assert lost_event_call == ( - "queue_overflow", - "log_item", - expected_dropped_item, - 1, - ) + reason, data_category, item, quantity = lost_event_call + + assert reason == "queue_overflow" + assert data_category == "log_item" + assert quantity == 1 + + assert item.type == "log" + assert item.headers == { + "type": "log", + "item_count": 1, + "content_type": "application/vnd.sentry.items.log+json", + } + assert item.payload.json == { + "items": [ + { + "body": "This is a 'info' log...", + "level": "info", + "timestamp": mock.ANY, + "trace_id": mock.ANY, + "attributes": { + "sentry.environment": { + "type": "string", + "value": "production", + }, + "sentry.release": { + "type": "string", + "value": "1.0.0", + }, + "sentry.sdk.name": { + "type": "string", + "value": mock.ANY, + }, + "sentry.sdk.version": { + "type": "string", + "value": VERSION, + }, + "sentry.severity_number": { + "type": "integer", + "value": 9, + }, + "sentry.severity_text": { + "type": "string", + "value": "info", + }, + "sentry.trace.parent_span_id": { + "type": "string", + "value": mock.ANY, + }, + "server.address": { + "type": "string", + "value": "test-server", + }, + }, + } + ] + } From f8b152d87e85f86c5fc07e41061696f145eae518 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 25 Nov 2025 11:49:37 +0100 Subject: [PATCH 4/5] typing --- sentry_sdk/client.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/sentry_sdk/client.py b/sentry_sdk/client.py index e6ef7ffa6c..0fb44ef9e8 100644 --- a/sentry_sdk/client.py +++ b/sentry_sdk/client.py @@ -62,20 +62,12 @@ from typing import Union from typing import TypeVar - from sentry_sdk._types import ( - Event, - Hint, - SDKInfo, - Log, - Metric, - EventDataCategory, - Item, - ) + from sentry_sdk._types import Event, Hint, SDKInfo, Log, Metric, EventDataCategory from sentry_sdk.integrations import Integration from sentry_sdk.scope import Scope from sentry_sdk.session import Session from sentry_sdk.spotlight import SpotlightClient - from sentry_sdk.transport import Transport + from sentry_sdk.transport import Transport, Item from sentry_sdk._log_batcher import LogBatcher from sentry_sdk._metrics_batcher import MetricsBatcher From 3cf9ddb96509489ed0069a4944d7c355d1bb68de Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 25 Nov 2025 12:06:05 +0100 Subject: [PATCH 5/5] make item in lost item callback optional --- sentry_sdk/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/client.py b/sentry_sdk/client.py index 0fb44ef9e8..fa17dbe18c 100644 --- a/sentry_sdk/client.py +++ b/sentry_sdk/client.py @@ -360,7 +360,7 @@ def _capture_envelope(envelope): def _record_lost_event( reason, # type: str data_category, # type: EventDataCategory - item, # type: Item + item=None, # type: Optional[Item] quantity=1, # type: int ): # type: (...) -> None