chore(ourlogs): Use new transport (#4317)

We've added a more efficient transport for logs handling, use that.

Solves LOGS-60

Author: colin-sentry

sentry_sdk/_log_batcher.py

@@ -5,7 +5,7 @@
 from typing import Optional, List, Callable, TYPE_CHECKING, Any
 
 from sentry_sdk.utils import format_timestamp, safe_repr
-from sentry_sdk.envelope import Envelope
+from sentry_sdk.envelope import Envelope, Item, PayloadRef
 
 if TYPE_CHECKING:
     from sentry_sdk._types import Log
@@ -97,34 +97,36 @@ def flush(self):
         self._flush()
 
     @staticmethod
-    def _log_to_otel(log):
+    def _log_to_transport_format(log):
         # type: (Log) -> Any
-        def format_attribute(key, val):
-            # type: (str, int | float | str | bool) -> Any
+        def format_attribute(val):
+            # type: (int | float | str | bool) -> Any
             if isinstance(val, bool):
-                return {"key": key, "value": {"boolValue": val}}
+                return {"value": val, "type": "boolean"}
             if isinstance(val, int):
-                return {"key": key, "value": {"intValue": str(val)}}
+                return {"value": val, "type": "integer"}
             if isinstance(val, float):
-                return {"key": key, "value": {"doubleValue": val}}
+                return {"value": val, "type": "double"}
             if isinstance(val, str):
-                return {"key": key, "value": {"stringValue": val}}
-            return {"key": key, "value": {"stringValue": safe_repr(val)}}
-
-        otel_log = {
-            "severityText": log["severity_text"],
-            "severityNumber": log["severity_number"],
-            "body": {"stringValue": log["body"]},
-            "timeUnixNano": str(log["time_unix_nano"]),
-            "attributes": [
-                format_attribute(k, v) for (k, v) in log["attributes"].items()
-            ],
+                return {"value": val, "type": "string"}
+            return {"value": safe_repr(val), "type": "string"}
+
+        if "sentry.severity_number" not in log["attributes"]:
+            log["attributes"]["sentry.severity_number"] = log["severity_number"]
+        if "sentry.severity_text" not in log["attributes"]:
+            log["attributes"]["sentry.severity_text"] = log["severity_text"]
+
+        res = {
+            "timestamp": int(log["time_unix_nano"]) / 1.0e9,
+            "trace_id": log.get("trace_id", "00000000-0000-0000-0000-000000000000"),
+            "level": str(log["severity_text"]),
+            "body": str(log["body"]),
+            "attributes": {
+                k: format_attribute(v) for (k, v) in log["attributes"].items()
+            },
         }
 
-        if "trace_id" in log:
-            otel_log["traceId"] = log["trace_id"]
-
-        return otel_log
+        return res
 
     def _flush(self):
         # type: (...) -> Optional[Envelope]
@@ -133,10 +135,27 @@ def _flush(self):
             headers={"sent_at": format_timestamp(datetime.now(timezone.utc))}
         )
         with self._lock:
-            for log in self._log_buffer:
-                envelope.add_log(self._log_to_otel(log))
+            if len(self._log_buffer) == 0:
+                return None
+
+            envelope.add_item(
+                Item(
+                    type="log",
+                    content_type="application/vnd.sentry.items.log+json",
+                    headers={
+                        "item_count": len(self._log_buffer),
+                    },
+                    payload=PayloadRef(
+                        json={
+                            "items": [
+                                self._log_to_transport_format(log)
+                                for log in self._log_buffer
+                            ]
+                        }
+                    ),
+                )
+            )
             self._log_buffer.clear()
-        if envelope.items:
-            self._capture_func(envelope)
-            return envelope
-        return None
+
+        self._capture_func(envelope)
+        return envelope

sentry_sdk/envelope.py

@@ -106,12 +106,6 @@ def add_sessions(
         # type: (...) -> None
         self.add_item(Item(payload=PayloadRef(json=sessions), type="sessions"))
 
-    def add_log(
-        self, log  # type: Any
-    ):
-        # type: (...) -> None
-        self.add_item(Item(payload=PayloadRef(json=log), type="otel_log"))
-
     def add_item(
         self, item  # type: Item
     ):
@@ -278,7 +272,7 @@ def data_category(self):
             return "transaction"
         elif ty == "event":
             return "error"
-        elif ty == "otel_log":
+        elif ty == "log":
             return "log"
         elif ty == "client_report":
             return "internal"

tests/test_logs.py

@@ -19,42 +19,44 @@
 
 
 def otel_attributes_to_dict(otel_attrs):
-    # type: (List[Mapping[str, Any]]) -> Mapping[str, Any]
+    # type: (Mapping[str, Any]) -> Mapping[str, Any]
     def _convert_attr(attr):
         # type: (Mapping[str, Union[str, float, bool]]) -> Any
-        if "boolValue" in attr:
-            return bool(attr["boolValue"])
-        if "doubleValue" in attr:
-            return float(attr["doubleValue"])
-        if "intValue" in attr:
-            return int(attr["intValue"])
-        if attr["stringValue"].startswith("{"):
+        if attr["type"] == "boolean":
+            return attr["value"]
+        if attr["type"] == "double":
+            return attr["value"]
+        if attr["type"] == "integer":
+            return attr["value"]
+        if attr["value"].startswith("{"):
             try:
                 return json.loads(attr["stringValue"])
             except ValueError:
                 pass
-        return str(attr["stringValue"])
+        return str(attr["value"])
 
-    return {item["key"]: _convert_attr(item["value"]) for item in otel_attrs}
+    return {k: _convert_attr(v) for (k, v) in otel_attrs.items()}
 
 
 def envelopes_to_logs(envelopes: List[Envelope]) -> List[Log]:
     res = []  # type: List[Log]
     for envelope in envelopes:
         for item in envelope.items:
-            if item.type == "otel_log":
-                log_json = item.payload.json
-                log = {
-                    "severity_text": log_json["severityText"],
-                    "severity_number": log_json["severityNumber"],
-                    "body": log_json["body"]["stringValue"],
-                    "attributes": otel_attributes_to_dict(log_json["attributes"]),
-                    "time_unix_nano": int(log_json["timeUnixNano"]),
-                    "trace_id": None,
-                }  # type: Log
-                if "traceId" in log_json:
-                    log["trace_id"] = log_json["traceId"]
-                res.append(log)
+            if item.type == "log":
+                for log_json in item.payload.json["items"]:
+                    log = {
+                        "severity_text": log_json["attributes"]["sentry.severity_text"][
+                            "value"
+                        ],
+                        "severity_number": int(
+                            log_json["attributes"]["sentry.severity_number"]["value"]
+                        ),
+                        "body": log_json["body"],
+                        "attributes": otel_attributes_to_dict(log_json["attributes"]),
+                        "time_unix_nano": int(float(log_json["timestamp"]) * 1e9),
+                        "trace_id": log_json["trace_id"],
+                    }  # type: Log
+                    res.append(log)
     return res