Skip to content

Commit ab0103e

Browse files
alexander-alderman-webbalexander-alderman-webb
authored
feat(logs): Record discarded log bytes (#5144)
Report the size of discarded logs in bytes as part of client reports, based on the specification added in commit 5819cdd of `sentry-docs`. The reported size is based on the log format submitted to the transport. The bytes representation returned by `PayloadRef.get_bytes()` is used to calculate the size in bytes. In all common code paths, the representation is a compact utf-8 encoding of the log item payload.
1 parent c720feb commit ab0103e

File tree

6 files changed

+203
-10
lines changed

6 files changed

+203
-10
lines changed

‎sentry_sdk/_log_batcher.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,9 +83,21 @@ def add(
8383

8484
with self._lock:
8585
if len(self._log_buffer) >= self.MAX_LOGS_BEFORE_DROP:
86+
# Construct log envelope item without sending it to report lost bytes
87+
log_item = Item(
88+
type="log",
89+
content_type="application/vnd.sentry.items.log+json",
90+
headers={
91+
"item_count": 1,
92+
},
93+
payload=PayloadRef(
94+
json={"items": [LogBatcher._log_to_transport_format(log)]}
95+
),
96+
)
8697
self._record_lost_func(
8798
reason="queue_overflow",
8899
data_category="log_item",
100+
item=log_item,
89101
quantity=1,
90102
)
91103
return None

‎sentry_sdk/_types.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,7 @@ class SDKInfo(TypedDict):
294294
"monitor",
295295
"span",
296296
"log_item",
297+
"log_byte",
297298
"trace_metric",
298299
]
299300
SessionStatus = Literal["ok", "exited", "crashed", "abnormal"]

‎sentry_sdk/client.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@
6767
from sentry_sdk.scope import Scope
6868
from sentry_sdk.session import Session
6969
from sentry_sdk.spotlight import SpotlightClient
70-
from sentry_sdk.transport import Transport
70+
from sentry_sdk.transport import Transport, Item
7171
from sentry_sdk._log_batcher import LogBatcher
7272
from sentry_sdk._metrics_batcher import MetricsBatcher
7373

@@ -360,13 +360,15 @@ def _capture_envelope(envelope):
360360
def _record_lost_event(
361361
reason, # type: str
362362
data_category, # type: EventDataCategory
363+
item=None, # type: Optional[Item]
363364
quantity=1, # type: int
364365
):
365366
# type: (...) -> None
366367
if self.transport is not None:
367368
self.transport.record_lost_event(
368369
reason=reason,
369370
data_category=data_category,
371+
item=item,
370372
quantity=quantity,
371373
)
372374

‎sentry_sdk/transport.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,11 @@ def record_lost_event(
266266
)
267267
self.record_lost_event(reason, "span", quantity=span_count)
268268

269+
elif data_category == "log_item" and item:
270+
# Also record size of lost logs in bytes
271+
bytes_size = len(item.get_bytes())
272+
self.record_lost_event(reason, "log_byte", quantity=bytes_size)
273+
269274
elif data_category == "attachment":
270275
# quantity of 0 is actually 1 as we do not want to count
271276
# empty attachments as actually empty.

‎tests/test_logs.py‎

Lines changed: 60 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@
44
import time
55
from typing import List, Any, Mapping, Union
66
import pytest
7+
from unittest import mock
78

89
import sentry_sdk
910
import sentry_sdk.logger
1011
from sentry_sdk import get_client
11-
from sentry_sdk.envelope import Envelope
12+
from sentry_sdk.envelope import Envelope, Item, PayloadRef
1213
from sentry_sdk.types import Log
1314
from sentry_sdk.consts import SPANDATA, VERSION
1415

@@ -450,7 +451,7 @@ def test_logs_with_literal_braces(
450451

451452
@minimum_python_37
452453
def test_batcher_drops_logs(sentry_init, monkeypatch):
453-
sentry_init(enable_logs=True)
454+
sentry_init(enable_logs=True, server_name="test-server", release="1.0.0")
454455
client = sentry_sdk.get_client()
455456

456457
def no_op_flush():
@@ -469,5 +470,61 @@ def record_lost_event(reason, data_category=None, item=None, *, quantity=1):
469470
sentry_sdk.logger.info("This is a 'info' log...")
470471

471472
assert len(lost_event_calls) == 5
473+
472474
for lost_event_call in lost_event_calls:
473-
assert lost_event_call == ("queue_overflow", "log_item", None, 1)
475+
reason, data_category, item, quantity = lost_event_call
476+
477+
assert reason == "queue_overflow"
478+
assert data_category == "log_item"
479+
assert quantity == 1
480+
481+
assert item.type == "log"
482+
assert item.headers == {
483+
"type": "log",
484+
"item_count": 1,
485+
"content_type": "application/vnd.sentry.items.log+json",
486+
}
487+
assert item.payload.json == {
488+
"items": [
489+
{
490+
"body": "This is a 'info' log...",
491+
"level": "info",
492+
"timestamp": mock.ANY,
493+
"trace_id": mock.ANY,
494+
"attributes": {
495+
"sentry.environment": {
496+
"type": "string",
497+
"value": "production",
498+
},
499+
"sentry.release": {
500+
"type": "string",
501+
"value": "1.0.0",
502+
},
503+
"sentry.sdk.name": {
504+
"type": "string",
505+
"value": mock.ANY,
506+
},
507+
"sentry.sdk.version": {
508+
"type": "string",
509+
"value": VERSION,
510+
},
511+
"sentry.severity_number": {
512+
"type": "integer",
513+
"value": 9,
514+
},
515+
"sentry.severity_text": {
516+
"type": "string",
517+
"value": "info",
518+
},
519+
"sentry.trace.parent_span_id": {
520+
"type": "string",
521+
"value": mock.ANY,
522+
},
523+
"server.address": {
524+
"type": "string",
525+
"value": "test-server",
526+
},
527+
},
528+
}
529+
]
530+
}

‎tests/test_transport.py‎

Lines changed: 122 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
Hub,
2626
)
2727
from sentry_sdk._compat import PY37, PY38
28-
from sentry_sdk.envelope import Envelope, Item, parse_json
28+
from sentry_sdk.envelope import Envelope, Item, parse_json, PayloadRef
2929
from sentry_sdk.transport import (
3030
KEEP_ALIVE_SOCKET_OPTIONS,
3131
_parse_rate_limits,
@@ -591,7 +591,110 @@ def test_complex_limits_without_data_category(
591591

592592

593593
@pytest.mark.parametrize("response_code", [200, 429])
594-
def test_log_item_limits(capturing_server, response_code, make_client):
594+
@pytest.mark.parametrize(
595+
"item",
596+
[
597+
Item(payload=b"{}", type="log"),
598+
Item(
599+
type="log",
600+
content_type="application/vnd.sentry.items.log+json",
601+
headers={
602+
"item_count": 2,
603+
},
604+
payload=PayloadRef(
605+
json={
606+
"items": [
607+
{
608+
"body": "This is a 'info' log...",
609+
"level": "info",
610+
"timestamp": datetime(
611+
2025, 1, 1, tzinfo=timezone.utc
612+
).timestamp(),
613+
"trace_id": "00000000-0000-0000-0000-000000000000",
614+
"attributes": {
615+
"sentry.environment": {
616+
"value": "production",
617+
"type": "string",
618+
},
619+
"sentry.release": {
620+
"value": "1.0.0",
621+
"type": "string",
622+
},
623+
"sentry.sdk.name": {
624+
"value": "sentry.python",
625+
"type": "string",
626+
},
627+
"sentry.sdk.version": {
628+
"value": "2.45.0",
629+
"type": "string",
630+
},
631+
"sentry.severity_number": {
632+
"value": 9,
633+
"type": "integer",
634+
},
635+
"sentry.severity_text": {
636+
"value": "info",
637+
"type": "string",
638+
},
639+
"server.address": {
640+
"value": "test-server",
641+
"type": "string",
642+
},
643+
},
644+
},
645+
{
646+
"body": "The recorded value was '2.0'",
647+
"level": "warn",
648+
"timestamp": datetime(
649+
2025, 1, 1, tzinfo=timezone.utc
650+
).timestamp(),
651+
"trace_id": "00000000-0000-0000-0000-000000000000",
652+
"attributes": {
653+
"sentry.message.parameter.float_var": {
654+
"value": 2.0,
655+
"type": "double",
656+
},
657+
"sentry.message.template": {
658+
"value": "The recorded value was '{float_var}'",
659+
"type": "string",
660+
},
661+
"sentry.sdk.name": {
662+
"value": "sentry.python",
663+
"type": "string",
664+
},
665+
"sentry.sdk.version": {
666+
"value": "2.45.0",
667+
"type": "string",
668+
},
669+
"server.address": {
670+
"value": "test-server",
671+
"type": "string",
672+
},
673+
"sentry.environment": {
674+
"value": "production",
675+
"type": "string",
676+
},
677+
"sentry.release": {
678+
"value": "1.0.0",
679+
"type": "string",
680+
},
681+
"sentry.severity_number": {
682+
"value": 13,
683+
"type": "integer",
684+
},
685+
"sentry.severity_text": {
686+
"value": "warn",
687+
"type": "string",
688+
},
689+
},
690+
},
691+
]
692+
}
693+
),
694+
),
695+
],
696+
)
697+
def test_log_item_limits(capturing_server, response_code, item, make_client):
595698
client = make_client()
596699
capturing_server.respond_with(
597700
code=response_code,
@@ -601,7 +704,7 @@ def test_log_item_limits(capturing_server, response_code, make_client):
601704
)
602705

603706
envelope = Envelope()
604-
envelope.add_item(Item(payload=b"{}", type="log"))
707+
envelope.add_item(item)
605708
client.transport.capture_envelope(envelope)
606709
client.flush()
607710

@@ -622,9 +725,22 @@ def test_log_item_limits(capturing_server, response_code, make_client):
622725
envelope = capturing_server.captured[1].envelope
623726
assert envelope.items[0].type == "client_report"
624727
report = parse_json(envelope.items[0].get_bytes())
625-
assert report["discarded_events"] == [
626-
{"category": "log_item", "reason": "ratelimit_backoff", "quantity": 1},
627-
]
728+
729+
assert {
730+
"category": "log_item",
731+
"reason": "ratelimit_backoff",
732+
"quantity": 1,
733+
} in report["discarded_events"]
734+
735+
expected_lost_bytes = 1243
736+
if item.payload.bytes == b"{}":
737+
expected_lost_bytes = 2
738+
739+
assert {
740+
"category": "log_byte",
741+
"reason": "ratelimit_backoff",
742+
"quantity": expected_lost_bytes,
743+
} in report["discarded_events"]
628744

629745

630746
def test_hub_cls_backwards_compat():

0 commit comments

Comments
 (0)