diff --git a/opentelemetry-exporter-gcp-logging/README.rst b/opentelemetry-exporter-gcp-logging/README.rst index 89fa70be..9788bc03 100644 --- a/opentelemetry-exporter-gcp-logging/README.rst +++ b/opentelemetry-exporter-gcp-logging/README.rst @@ -68,6 +68,30 @@ Usage logger1.warning("string log %s", "here") +If your code is running in a GCP environment with a supported Cloud Logging agent (like GKE, +Cloud Run, GCE, etc.), you can write logs to stdout in Cloud Logging `structured JSON format +`_. Pass the ``structured_json_file`` +argument and use ``SimpleLogRecordProcessor``: + +.. code:: python + + import sys + from opentelemetry.exporter.cloud_logging import ( + CloudLoggingExporter, + ) + from opentelemetry._logs import set_logger_provider + from opentelemetry.sdk._logs import LoggerProvider + from opentelemetry.sdk._logs.export import SimpleLogRecordProcessor + + logger_provider = LoggerProvider() + set_logger_provider(logger_provider) + exporter = CloudLoggingExporter(structured_json_file=sys.stdout) + logger_provider.add_log_record_processor(SimpleLogRecordProcessor(exporter)) + + + otel_logger = logger_provider.get_logger(__name__) + otel_logger.emit(attributes={"hello": "world"}, body={"foo": {"bar": "baz"}}) + References ---------- diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py index ec77c72f..96f4fe1f 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py @@ -19,7 +19,16 @@ import logging import re from base64 import b64encode -from typing import Any, Mapping, MutableMapping, Optional, Sequence +from functools import partial +from typing import ( + Any, + Mapping, + MutableMapping, + Optional, + Sequence, + TextIO, + cast, +) import google.auth from google.api.monitored_resource_pb2 import ( # pylint: disable = no-name-in-module @@ -36,6 +45,7 @@ from google.logging.type.log_severity_pb2 import ( # pylint: disable = no-name-in-module LogSeverity, ) +from google.protobuf.json_format import MessageToDict from google.protobuf.struct_pb2 import ( # pylint: disable = no-name-in-module Struct, ) @@ -52,6 +62,9 @@ from opentelemetry.sdk.resources import Resource from opentelemetry.trace import format_span_id, format_trace_id from opentelemetry.util.types import AnyValue +from proto.datetime_helpers import ( # type: ignore[import] + DatetimeWithNanoseconds, +) DEFAULT_MAX_ENTRY_SIZE = 256000 # 256 KB DEFAULT_MAX_REQUEST_SIZE = 10000000 # 10 MB @@ -205,24 +218,59 @@ def __init__( project_id: Optional[str] = None, default_log_name: Optional[str] = None, client: Optional[LoggingServiceV2Client] = None, - ): + *, + structured_json_file: Optional[TextIO] = None, + ) -> None: + """Create a CloudLoggingExporter + + Args: + project_id: The GCP project ID to which the logs will be sent. If not + provided, the exporter will infer it from Application Default Credentials. + default_log_name: The default log name to use for log entries. + If not provided, a default name will be used. + client: An optional `LoggingServiceV2Client` instance to use for + sending logs. If not provided and ``structured_json_file`` is not provided, a + new client will be created. Passing both ``client`` and + ``structured_json_file`` is not supported. + structured_json_file: An optional file-like object (like `sys.stdout`) to write + logs to in Cloud Logging `structured JSON format + `_. If provided, + ``client`` must not be provided and logs will only be written to the file-like + object. + """ + self.project_id: str if not project_id: _, default_project_id = google.auth.default() self.project_id = str(default_project_id) else: self.project_id = project_id + if default_log_name: self.default_log_name = default_log_name else: self.default_log_name = "otel_python_inprocess_log_name_temp" - self.client = client or LoggingServiceV2Client( - transport=LoggingServiceV2GrpcTransport( - channel=LoggingServiceV2GrpcTransport.create_channel( - options=_OPTIONS, + + if client and structured_json_file: + raise ValueError( + "Cannot specify both client and structured_json_file" + ) + + if structured_json_file: + self._write_log_entries = partial( + self._write_log_entries_to_file, structured_json_file + ) + else: + client = client or LoggingServiceV2Client( + transport=LoggingServiceV2GrpcTransport( + channel=LoggingServiceV2GrpcTransport.create_channel( + options=_OPTIONS, + ) ) ) - ) + self._write_log_entries = partial( + self._write_log_entries_to_client, client + ) def pick_log_id(self, log_name_attr: Any, event_name: str | None) -> str: if log_name_attr and isinstance(log_name_attr, str): @@ -288,7 +336,58 @@ def export(self, batch: Sequence[LogData]): self._write_log_entries(log_entries) - def _write_log_entries(self, log_entries: list[LogEntry]): + @staticmethod + def _write_log_entries_to_file(file: TextIO, log_entries: list[LogEntry]): + """Formats logs into the Cloud Logging structured log format, and writes them to the + specified file-like object + + See https://cloud.google.com/logging/docs/structured-logging + """ + # TODO: this is not resilient to exceptions which can cause recursion when using OTel's + # logging handler. See + # https://github.com/open-telemetry/opentelemetry-python/issues/4261 for outstanding + # issue in OTel. + + for entry in log_entries: + json_dict: dict[str, Any] = {} + + # These are not added in export() so not added to the JSON here. + # - httpRequest + # - logging.googleapis.com/sourceLocation + # - logging.googleapis.com/operation + # - logging.googleapis.com/insertId + + # https://cloud.google.com/logging/docs/agent/logging/configuration#timestamp-processing + timestamp = cast(DatetimeWithNanoseconds, entry.timestamp) + json_dict["time"] = timestamp.rfc3339() + + json_dict["severity"] = LogSeverity.Name( + cast(LogSeverity.ValueType, entry.severity) + ) + json_dict["logging.googleapis.com/labels"] = dict(entry.labels) + json_dict["logging.googleapis.com/spanId"] = entry.span_id + json_dict[ + "logging.googleapis.com/trace_sampled" + ] = entry.trace_sampled + json_dict["logging.googleapis.com/trace"] = entry.trace + + if entry.text_payload: + json_dict["message"] = entry.text_payload + if entry.json_payload: + json_dict.update( + MessageToDict(LogEntry.pb(entry).json_payload) + ) + + # Use dumps to avoid invalid json written to the stream if serialization fails for any reason + file.write( + json.dumps(json_dict, separators=(",", ":"), sort_keys=True) + + "\n" + ) + + @staticmethod + def _write_log_entries_to_client( + client: LoggingServiceV2Client, log_entries: list[LogEntry] + ): batch: list[LogEntry] = [] batch_byte_size = 0 for entry in log_entries: @@ -302,7 +401,7 @@ def _write_log_entries(self, log_entries: list[LogEntry]): continue if msg_size + batch_byte_size > DEFAULT_MAX_REQUEST_SIZE: try: - self.client.write_log_entries( + client.write_log_entries( WriteLogEntriesRequest( entries=batch, partial_success=True ) @@ -319,7 +418,7 @@ def _write_log_entries(self, log_entries: list[LogEntry]): batch_byte_size += msg_size if batch: try: - self.client.write_log_entries( + client.write_log_entries( WriteLogEntriesRequest(entries=batch, partial_success=True) ) # pylint: disable=broad-except diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_gen_ai_body.json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_gen_ai_body[client].json similarity index 100% rename from opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_gen_ai_body.json rename to opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_gen_ai_body[client].json diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_gen_ai_body[structured_json].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_gen_ai_body[structured_json].json new file mode 100644 index 00000000..1fa1c19d --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_gen_ai_body[structured_json].json @@ -0,0 +1,81 @@ +[ + { + "gen_ai.input.messages": [ + { + "parts": [ + { + "content": "Get weather details in New Delhi and San Francisco?", + "type": "text" + } + ], + "role": "user" + }, + { + "parts": [ + { + "arguments": { + "location": "New Delhi" + }, + "id": "get_current_weather_0", + "name": "get_current_weather", + "type": "tool_call" + }, + { + "arguments": { + "location": "San Francisco" + }, + "id": "get_current_weather_1", + "name": "get_current_weather", + "type": "tool_call" + } + ], + "role": "model" + }, + { + "parts": [ + { + "id": "get_current_weather_0", + "response": { + "content": "{\"temperature\": 35, \"unit\": \"C\"}" + }, + "type": "tool_call_response" + }, + { + "id": "get_current_weather_1", + "response": { + "content": "{\"temperature\": 25, \"unit\": \"C\"}" + }, + "type": "tool_call_response" + } + ], + "role": "user" + } + ], + "gen_ai.output.messages": [ + { + "finish_reason": "stop", + "parts": [ + { + "content": "The current temperature in New Delhi is 35°C, and in San Francisco, it is 25°C.", + "type": "text" + } + ], + "role": "model" + } + ], + "gen_ai.system_instructions": [ + { + "content": "You are a clever language model", + "type": "text" + } + ], + "logging.googleapis.com/labels": { + "event.name": "gen_ai.client.inference.operation.details" + }, + "logging.googleapis.com/spanId": "", + "logging.googleapis.com/trace": "", + "logging.googleapis.com/trace_sampled": false, + "severity": "DEFAULT", + "time": "2025-01-15T21:25:10.997977393Z" + } +] diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes.json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes[client].json similarity index 100% rename from opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes.json rename to opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes[client].json diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes[structured_json].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes[structured_json].json new file mode 100644 index 00000000..4a262aef --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes[structured_json].json @@ -0,0 +1,11 @@ +[ + { + "logging.googleapis.com/labels": {}, + "logging.googleapis.com/spanId": "", + "logging.googleapis.com/trace": "", + "logging.googleapis.com/trace_sampled": false, + "message": "MTIz", + "severity": "DEFAULT", + "time": "2025-01-15T21:25:10.997977393Z" + } +] diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_dict_body.json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_dict_body[client].json similarity index 100% rename from opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_dict_body.json rename to opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_dict_body[client].json diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_dict_body[structured_json].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_dict_body[structured_json].json new file mode 100644 index 00000000..843245b5 --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_dict_body[structured_json].json @@ -0,0 +1,30 @@ +[ + { + "kvlistValue": { + "bytes_field": "Ynl0ZXM=", + "repeated_bytes_field": [ + "Ynl0ZXM=", + "Ynl0ZXM=", + "Ynl0ZXM=" + ], + "values": [ + { + "key": "content", + "value": { + "stringValue": "You're a helpful assistant." + } + } + ] + }, + "logging.googleapis.com/labels": { + "event.name": "random.genai.event", + "gen_ai.system": "true", + "test": "23" + }, + "logging.googleapis.com/spanId": "0000000000000016", + "logging.googleapis.com/trace": "projects/fakeproject/traces/00000000000000000000000000000019", + "logging.googleapis.com/trace_sampled": false, + "severity": "ERROR", + "time": "2025-01-15T21:25:10.997977393Z" + } +] diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body.json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body[client].json similarity index 100% rename from opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body.json rename to opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body[client].json diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body[structured_json].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body[structured_json].json new file mode 100644 index 00000000..7fa6b200 --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body[structured_json].json @@ -0,0 +1,24 @@ +[ + { + "Classe": [ + "Email addresses", + "Passwords" + ], + "CreationDate": "2012-05-05", + "Date": "2016-05-21T21:35:40Z", + "Link": "http://some_link.com", + "LogoType": "png", + "Ref": 164611595.0, + "logging.googleapis.com/labels": { + "boolArray": "[true,false,true,true]", + "float": "25.43231", + "int": "25", + "intArray": "[21,18,23,17]" + }, + "logging.googleapis.com/spanId": "", + "logging.googleapis.com/trace": "", + "logging.googleapis.com/trace_sampled": false, + "severity": "DEFAULT", + "time": "2025-01-15T21:25:10.997977393Z" + } +] diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_attributes.json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_attributes[client].json similarity index 100% rename from opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_attributes.json rename to opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_attributes[client].json diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_attributes[structured_json].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_attributes[structured_json].json new file mode 100644 index 00000000..cd8b0c3d --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_attributes[structured_json].json @@ -0,0 +1,15 @@ +[ + { + "logging.googleapis.com/labels": { + "a": "[{\"key\":\"Ynl0ZXM=\"}]", + "b": "[true,false,false,true]", + "c": "{\"a_dict\":\"abcd\",\"akey\":1234}", + "d": "{\"gen_ai.input.messages\":[{\"role\":\"user\",\"parts\":[{\"type\":\"text\",\"content\":\"Get weather details in New Delhi and San Francisco?\"}]},{\"role\":\"model\",\"parts\":[{\"type\":\"tool_call\",\"arguments\":{\"location\":\"New Delhi\"},\"name\":\"get_current_weather\",\"id\":\"get_current_weather_0\"},{\"type\":\"tool_call\",\"arguments\":{\"location\":\"San Francisco\"},\"name\":\"get_current_weather\",\"id\":\"get_current_weather_1\"}]},{\"role\":\"user\",\"parts\":[{\"type\":\"tool_call_response\",\"response\":{\"content\":\"{\\\"temperature\\\": 35, \\\"unit\\\": \\\"C\\\"}\"},\"id\":\"get_current_weather_0\"},{\"type\":\"tool_call_response\",\"response\":{\"content\":\"{\\\"temperature\\\": 25, \\\"unit\\\": \\\"C\\\"}\"},\"id\":\"get_current_weather_1\"}]}],\"gen_ai.system_instructions\":[{\"type\":\"text\",\"content\":\"You are a clever language model\"}],\"gen_ai.output.messages\":[{\"role\":\"model\",\"parts\":[{\"type\":\"text\",\"content\":\"The current temperature in New Delhi is 35\\u00b0C, and in San Francisco, it is 25\\u00b0C.\"}],\"finish_reason\":\"stop\"}]}" + }, + "logging.googleapis.com/spanId": "", + "logging.googleapis.com/trace": "", + "logging.googleapis.com/trace_sampled": false, + "severity": "DEFAULT", + "time": "2025-01-15T21:25:10.997977393Z" + } +] diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[None].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[client-None].json similarity index 100% rename from opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[None].json rename to opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[client-None].json diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[bool].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[client-bool].json similarity index 100% rename from opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[bool].json rename to opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[client-bool].json diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[list_of_bools].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[client-list_of_bools].json similarity index 100% rename from opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[list_of_bools].json rename to opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[client-list_of_bools].json diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[list_of_dicts_with_bytes].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[client-list_of_dicts_with_bytes].json similarity index 100% rename from opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[list_of_dicts_with_bytes].json rename to opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[client-list_of_dicts_with_bytes].json diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[list_of_mixed_sequence].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[client-list_of_mixed_sequence].json similarity index 100% rename from opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[list_of_mixed_sequence].json rename to opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[client-list_of_mixed_sequence].json diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[str].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[client-str].json similarity index 100% rename from opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[str].json rename to opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[client-str].json diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[structured_json-None].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[structured_json-None].json new file mode 100644 index 00000000..805d458b --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[structured_json-None].json @@ -0,0 +1,10 @@ +[ + { + "logging.googleapis.com/labels": {}, + "logging.googleapis.com/spanId": "", + "logging.googleapis.com/trace": "", + "logging.googleapis.com/trace_sampled": false, + "severity": "DEFAULT", + "time": "2025-01-15T21:25:10.997977393Z" + } +] diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[structured_json-bool].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[structured_json-bool].json new file mode 100644 index 00000000..2a6821e3 --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[structured_json-bool].json @@ -0,0 +1,11 @@ +[ + { + "logging.googleapis.com/labels": {}, + "logging.googleapis.com/spanId": "", + "logging.googleapis.com/trace": "", + "logging.googleapis.com/trace_sampled": false, + "message": "true", + "severity": "DEFAULT", + "time": "2025-01-15T21:25:10.997977393Z" + } +] diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[structured_json-list_of_bools].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[structured_json-list_of_bools].json new file mode 100644 index 00000000..c29f540e --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[structured_json-list_of_bools].json @@ -0,0 +1,16 @@ +[ + { + "logging.googleapis.com/labels": {}, + "logging.googleapis.com/spanId": "", + "logging.googleapis.com/trace": "", + "logging.googleapis.com/trace_sampled": false, + "my_dict": [ + true, + false, + false, + true + ], + "severity": "DEFAULT", + "time": "2025-01-15T21:25:10.997977393Z" + } +] diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[structured_json-list_of_dicts_with_bytes].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[structured_json-list_of_dicts_with_bytes].json new file mode 100644 index 00000000..ea2182bb --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[structured_json-list_of_dicts_with_bytes].json @@ -0,0 +1,15 @@ +[ + { + "logging.googleapis.com/labels": {}, + "logging.googleapis.com/spanId": "", + "logging.googleapis.com/trace": "", + "logging.googleapis.com/trace_sampled": false, + "my_dict": [ + { + "key": "Ynl0ZXM=" + } + ], + "severity": "DEFAULT", + "time": "2025-01-15T21:25:10.997977393Z" + } +] diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[structured_json-list_of_mixed_sequence].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[structured_json-list_of_mixed_sequence].json new file mode 100644 index 00000000..e7001ddc --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[structured_json-list_of_mixed_sequence].json @@ -0,0 +1,16 @@ +[ + { + "logging.googleapis.com/labels": {}, + "logging.googleapis.com/spanId": "", + "logging.googleapis.com/trace": "", + "logging.googleapis.com/trace_sampled": false, + "my_dict": [ + true, + "str", + 1.0, + 0.234 + ], + "severity": "DEFAULT", + "time": "2025-01-15T21:25:10.997977393Z" + } +] diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[structured_json-str].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[structured_json-str].json new file mode 100644 index 00000000..56fd88b6 --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[structured_json-str].json @@ -0,0 +1,11 @@ +[ + { + "logging.googleapis.com/labels": {}, + "logging.googleapis.com/spanId": "", + "logging.googleapis.com/trace": "", + "logging.googleapis.com/trace_sampled": false, + "message": "A text body", + "severity": "DEFAULT", + "time": "2025-01-15T21:25:10.997977393Z" + } +] diff --git a/opentelemetry-exporter-gcp-logging/tests/conftest.py b/opentelemetry-exporter-gcp-logging/tests/conftest.py index a23c6dbc..48889dd4 100644 --- a/opentelemetry-exporter-gcp-logging/tests/conftest.py +++ b/opentelemetry-exporter-gcp-logging/tests/conftest.py @@ -12,8 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: disable=unused-import +import pytest + +pytest.register_assert_rewrite("fixtures.cloud_logging_fake") # import fixtures to be made available to other tests -from fixtures.cloud_logging_fake import fixture_cloudloggingfake -from fixtures.snapshot_logging_calls import fixture_snapshot_writelogentrycalls +from fixtures.cloud_logging_fake import ( # noqa: E402 pylint: disable=wrong-import-position + fixture_cloudloggingfake, + fixture_export_and_assert_snapshot, +) + +__all__ = ["fixture_cloudloggingfake", "fixture_export_and_assert_snapshot"] diff --git a/opentelemetry-exporter-gcp-logging/tests/fixtures/cloud_logging_fake.py b/opentelemetry-exporter-gcp-logging/tests/fixtures/cloud_logging_fake.py index 7ecf64f9..72858fb9 100644 --- a/opentelemetry-exporter-gcp-logging/tests/fixtures/cloud_logging_fake.py +++ b/opentelemetry-exporter-gcp-logging/tests/fixtures/cloud_logging_fake.py @@ -11,14 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from functools import partial -from typing import Callable, Iterable, List, cast +from io import StringIO +from typing import Callable, Iterable, List, Sequence, cast from unittest.mock import patch import grpc import pytest +from fixtures.snapshot_logging_calls import WriteLogEntryCallSnapshotExtension from google.cloud.logging_v2.services.logging_service_v2.transports.grpc import ( LoggingServiceV2GrpcTransport, ) @@ -34,6 +37,9 @@ unary_unary_rpc_method_handler, ) from opentelemetry.exporter.cloud_logging import CloudLoggingExporter +from opentelemetry.sdk._logs import LogData +from syrupy.assertion import SnapshotAssertion +from syrupy.extensions.json import JSONSnapshotExtension @dataclass @@ -125,3 +131,41 @@ def fixture_cloudloggingfake() -> Iterable[CloudLoggingFake]: finally: if server: server.stop(None) + + +ExportAndAssertSnapshot = Callable[[Sequence[LogData]], None] + + +@pytest.fixture( + name="export_and_assert_snapshot", params=["client", "structured_json"] +) +def fixture_export_and_assert_snapshot( + request: pytest.FixtureRequest, + snapshot: SnapshotAssertion, +) -> ExportAndAssertSnapshot: + if request.param == "client": + cloudloggingfake: CloudLoggingFake = request.getfixturevalue( + "cloudloggingfake" + ) + + def export_and_assert_snapshot(log_data: Sequence[LogData]) -> None: + cloudloggingfake.exporter.export(log_data) + + assert cloudloggingfake.get_calls() == snapshot( + extension_class=WriteLogEntryCallSnapshotExtension + ) + + return export_and_assert_snapshot + + # pylint: disable=function-redefined + def export_and_assert_snapshot(log_data: Sequence[LogData]) -> None: + buf = StringIO() + exporter = CloudLoggingExporter( + project_id=PROJECT_ID, structured_json_file=buf + ) + exporter.export(log_data) + buf.seek(0) + as_dict = [json.loads(line) for line in buf] + assert as_dict == snapshot(extension_class=JSONSnapshotExtension) + + return export_and_assert_snapshot diff --git a/opentelemetry-exporter-gcp-logging/tests/fixtures/snapshot_logging_calls.py b/opentelemetry-exporter-gcp-logging/tests/fixtures/snapshot_logging_calls.py index eb6dd44d..35bf2a83 100644 --- a/opentelemetry-exporter-gcp-logging/tests/fixtures/snapshot_logging_calls.py +++ b/opentelemetry-exporter-gcp-logging/tests/fixtures/snapshot_logging_calls.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional, cast -import pytest -from fixtures.cloud_logging_fake import WriteLogEntriesCall +from typing import TYPE_CHECKING, List, Optional, cast + from google.protobuf import json_format from syrupy.extensions.json import JSONSnapshotExtension from syrupy.types import ( @@ -25,6 +24,9 @@ SerializedData, ) +if TYPE_CHECKING: + from fixtures.cloud_logging_fake import WriteLogEntriesCall + # pylint: disable=too-many-ancestors class WriteLogEntryCallSnapshotExtension(JSONSnapshotExtension): @@ -43,12 +45,6 @@ def serialize( call.write_log_entries_request ) ) - for call in cast(List[WriteLogEntriesCall], data) + for call in cast(List["WriteLogEntriesCall"], data) ] return super().serialize(json, exclude=exclude, matcher=matcher) - - -@pytest.fixture(name="snapshot_writelogentrycalls") -def fixture_snapshot_writelogentrycalls(snapshot): - """Fixture for snapshot testing of WriteLogEntriesCalls""" - return snapshot.use_extension(WriteLogEntryCallSnapshotExtension)() diff --git a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py index 0f6d5b01..6149ff4c 100644 --- a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py +++ b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py @@ -26,10 +26,15 @@ Be sure to review the changes. """ import re -from typing import List, Mapping, Union +from io import StringIO +from textwrap import dedent +from typing import Mapping, Union import pytest -from fixtures.cloud_logging_fake import CloudLoggingFake, WriteLogEntriesCall +from fixtures.cloud_logging_fake import ( + CloudLoggingFake, + ExportAndAssertSnapshot, +) from google.auth.credentials import AnonymousCredentials from google.cloud.logging_v2.services.logging_service_v2 import ( LoggingServiceV2Client, @@ -138,9 +143,30 @@ def test_too_large_log_raises_warning(caplog) -> None: ) +def test_user_agent(cloudloggingfake: CloudLoggingFake) -> None: + cloudloggingfake.exporter.export( + [ + LogData( + log_record=LogRecord( + body="abc", + resource=Resource({}), + ), + instrumentation_scope=InstrumentationScope("test"), + ) + ] + ) + for call in cloudloggingfake.get_calls(): + assert ( + re.match( + r"^opentelemetry-python \S+; google-cloud-logging-exporter \S+ grpc-python/\S+", + call.user_agent, + ) + is not None + ) + + def test_convert_otlp_dict_body( - cloudloggingfake: CloudLoggingFake, - snapshot_writelogentrycalls: List[WriteLogEntriesCall], + export_and_assert_snapshot: ExportAndAssertSnapshot, ) -> None: log_data = [ LogData( @@ -172,21 +198,11 @@ def test_convert_otlp_dict_body( instrumentation_scope=InstrumentationScope("test"), ) ] - cloudloggingfake.exporter.export(log_data) - assert cloudloggingfake.get_calls() == snapshot_writelogentrycalls - for call in cloudloggingfake.get_calls(): - assert ( - re.match( - r"^opentelemetry-python \S+; google-cloud-logging-exporter \S+ grpc-python/\S+", - call.user_agent, - ) - is not None - ) + export_and_assert_snapshot(log_data) def test_convert_otlp_various_different_types_in_attrs_and_bytes_body( - cloudloggingfake: CloudLoggingFake, - snapshot_writelogentrycalls: List[WriteLogEntriesCall], + export_and_assert_snapshot: ExportAndAssertSnapshot, ) -> None: log_data = [ LogData( @@ -203,13 +219,11 @@ def test_convert_otlp_various_different_types_in_attrs_and_bytes_body( instrumentation_scope=InstrumentationScope("test"), ) ] - cloudloggingfake.exporter.export(log_data) - assert cloudloggingfake.get_calls() == snapshot_writelogentrycalls + export_and_assert_snapshot(log_data) def test_convert_non_json_dict_bytes( - cloudloggingfake: CloudLoggingFake, - snapshot_writelogentrycalls: List[WriteLogEntriesCall], + export_and_assert_snapshot: ExportAndAssertSnapshot, ) -> None: log_data = [ LogData( @@ -220,13 +234,11 @@ def test_convert_non_json_dict_bytes( instrumentation_scope=InstrumentationScope("test"), ) ] - cloudloggingfake.exporter.export(log_data) - assert cloudloggingfake.get_calls() == snapshot_writelogentrycalls + export_and_assert_snapshot(log_data) def test_convert_gen_ai_body( - cloudloggingfake: CloudLoggingFake, - snapshot_writelogentrycalls: List[WriteLogEntriesCall], + export_and_assert_snapshot: ExportAndAssertSnapshot, ) -> None: log_data = [ LogData( @@ -238,8 +250,7 @@ def test_convert_gen_ai_body( instrumentation_scope=InstrumentationScope("test"), ) ] - cloudloggingfake.exporter.export(log_data) - assert cloudloggingfake.get_calls() == snapshot_writelogentrycalls + export_and_assert_snapshot(log_data) def test_is_log_id_valid(): @@ -290,8 +301,7 @@ def test_pick_log_id() -> None: ], ) def test_convert_various_types_of_bodies( - cloudloggingfake: CloudLoggingFake, - snapshot_writelogentrycalls: List[WriteLogEntriesCall], + export_and_assert_snapshot: ExportAndAssertSnapshot, body: Union[str, bool, None, Mapping], ) -> None: log_data = [ @@ -303,13 +313,11 @@ def test_convert_various_types_of_bodies( instrumentation_scope=InstrumentationScope("test"), ) ] - cloudloggingfake.exporter.export(log_data) - assert cloudloggingfake.get_calls() == snapshot_writelogentrycalls + export_and_assert_snapshot(log_data) def test_convert_various_types_of_attributes( - cloudloggingfake: CloudLoggingFake, - snapshot_writelogentrycalls: List[WriteLogEntriesCall], + export_and_assert_snapshot: ExportAndAssertSnapshot, ) -> None: log_data = [ LogData( @@ -325,5 +333,37 @@ def test_convert_various_types_of_attributes( instrumentation_scope=InstrumentationScope("test"), ) ] - cloudloggingfake.exporter.export(log_data) - assert cloudloggingfake.get_calls() == snapshot_writelogentrycalls + export_and_assert_snapshot(log_data) + + +def test_structured_json_lines(): + buf = StringIO() + exporter = CloudLoggingExporter( + project_id=PROJECT_ID, structured_json_file=buf + ) + exporter.export( + [ + LogData( + log_record=LogRecord( + event_name="foo", + timestamp=1736976310997977393, + severity_number=SeverityNumber(20), + trace_id=25, + span_id=22, + attributes={"key": f"{i}"}, + body="hello", + ), + instrumentation_scope=InstrumentationScope("test"), + ) + for i in range(5) + ] + ) + assert buf.getvalue() == dedent( + """\ + {"logging.googleapis.com/labels":{"event.name":"foo","key":"0"},"logging.googleapis.com/spanId":"0000000000000016","logging.googleapis.com/trace":"projects/fakeproject/traces/00000000000000000000000000000019","logging.googleapis.com/trace_sampled":false,"message":"hello","severity":"ERROR","time":"2025-01-15T21:25:10.997977393Z"} + {"logging.googleapis.com/labels":{"event.name":"foo","key":"1"},"logging.googleapis.com/spanId":"0000000000000016","logging.googleapis.com/trace":"projects/fakeproject/traces/00000000000000000000000000000019","logging.googleapis.com/trace_sampled":false,"message":"hello","severity":"ERROR","time":"2025-01-15T21:25:10.997977393Z"} + {"logging.googleapis.com/labels":{"event.name":"foo","key":"2"},"logging.googleapis.com/spanId":"0000000000000016","logging.googleapis.com/trace":"projects/fakeproject/traces/00000000000000000000000000000019","logging.googleapis.com/trace_sampled":false,"message":"hello","severity":"ERROR","time":"2025-01-15T21:25:10.997977393Z"} + {"logging.googleapis.com/labels":{"event.name":"foo","key":"3"},"logging.googleapis.com/spanId":"0000000000000016","logging.googleapis.com/trace":"projects/fakeproject/traces/00000000000000000000000000000019","logging.googleapis.com/trace_sampled":false,"message":"hello","severity":"ERROR","time":"2025-01-15T21:25:10.997977393Z"} + {"logging.googleapis.com/labels":{"event.name":"foo","key":"4"},"logging.googleapis.com/spanId":"0000000000000016","logging.googleapis.com/trace":"projects/fakeproject/traces/00000000000000000000000000000019","logging.googleapis.com/trace_sampled":false,"message":"hello","severity":"ERROR","time":"2025-01-15T21:25:10.997977393Z"} + """ + ), "Each `LogData` should be on its own line"