From 66f9933ffcb596d5238d28b56915e18dcd06f2e3 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Mon, 6 Oct 2025 20:13:44 +0000 Subject: [PATCH 1/3] Initial commit to handle all attrs types --- .../exporter/cloud_logging/__init__.py | 28 +++- ...fferent_types_in_attrs_and_bytes_body.json | 4 +- ...t_convert_various_types_of_attributes.json | 25 +++ .../tests/test_cloud_logging.py | 158 ++++++++++-------- 4 files changed, 140 insertions(+), 75 deletions(-) create mode 100644 opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_attributes.json diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py index 204ee113..d62dde63 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py @@ -18,7 +18,7 @@ import json import logging import re -import urllib.parse +from base64 import b64encode from typing import Any, Mapping, MutableMapping, Optional, Sequence import google.auth @@ -106,15 +106,27 @@ INVALID_LOG_NAME_MESSAGE = "%s is not a valid log name. log name must be <512 characters and only contain characters: A-Za-z0-9/-_." -def _convert_any_value_to_string(value: Any) -> str: +class _GenAiJsonEncoder(json.JSONEncoder): + def default(self, o: Any) -> Any: + if isinstance(o, bytes): + return b64encode(o).decode() + return super().default(o) + + +def _convert_any_value_to_string(value: Any, debug_location: str) -> str: if isinstance(value, bool): return "true" if value else "false" if isinstance(value, bytes): return base64.b64encode(value).decode() if isinstance(value, (int, float, str)): return str(value) - if isinstance(value, (list, tuple)): - return json.dumps(value) + if isinstance(value, (list, tuple, Mapping)): + return json.dumps(value, separators=(",", ":"), cls=_GenAiJsonEncoder) + logging.warning( + "Unexpected type %s found in %s, this field will not be added to the LogEntry.", + type(value), + debug_location, + ) return "" @@ -176,7 +188,9 @@ def _set_payload_in_log_entry(log_entry: LogEntry, body: AnyValue): else: log_entry.text_payload = base64.b64encode(body).decode() elif body is not None: - log_entry.text_payload = _convert_any_value_to_string(body) + log_entry.text_payload = _convert_any_value_to_string( + body, "LogRecord.body" + ) def is_log_id_valid(log_id: str) -> bool: @@ -264,7 +278,9 @@ def export(self, batch: Sequence[LogData]): log_record.severity_number.value # type: ignore[index] ] log_entry.labels = { - k: _convert_any_value_to_string(v) + k: _convert_any_value_to_string( + v, "LogRecord.Attribute {}".format(k) + ) for k, v in attributes.items() } if log_record.event_name: diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body.json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body.json index bbf8dfcc..55fba3bb 100644 --- a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body.json +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body.json @@ -14,10 +14,10 @@ "Ref": 164611595.0 }, "labels": { - "boolArray": "[true, false, true, true]", + "boolArray": "[true,false,true,true]", "float": "25.43231", "int": "25", - "intArray": "[21, 18, 23, 17]" + "intArray": "[21,18,23,17]" }, "logName": "projects/fakeproject/logs/test", "resource": { diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_attributes.json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_attributes.json new file mode 100644 index 00000000..8bec635e --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_attributes.json @@ -0,0 +1,25 @@ +[ + { + "entries": [ + { + "labels": { + "a": "[{\"key\":\"Ynl0ZXM=\"}]", + "b": "[true,false,false,true]", + "c": "{\"a_dict\":\"abcd\",\"akey\":1234}", + "d": "{\"gen_ai.input.messages\":[{\"role\":\"user\",\"parts\":[{\"type\":\"text\",\"content\":\"Get weather details in New Delhi and San Francisco?\"}]},{\"role\":\"model\",\"parts\":[{\"type\":\"tool_call\",\"arguments\":{\"location\":\"New Delhi\"},\"name\":\"get_current_weather\",\"id\":\"get_current_weather_0\"},{\"type\":\"tool_call\",\"arguments\":{\"location\":\"San Francisco\"},\"name\":\"get_current_weather\",\"id\":\"get_current_weather_1\"}]},{\"role\":\"user\",\"parts\":[{\"type\":\"tool_call_response\",\"response\":{\"content\":\"{\\\"temperature\\\": 35, \\\"unit\\\": \\\"C\\\"}\"},\"id\":\"get_current_weather_0\"},{\"type\":\"tool_call_response\",\"response\":{\"content\":\"{\\\"temperature\\\": 25, \\\"unit\\\": \\\"C\\\"}\"},\"id\":\"get_current_weather_1\"}]}],\"gen_ai.system_instructions\":[{\"type\":\"text\",\"content\":\"You are a clever language model\"}],\"gen_ai.output.messages\":[{\"role\":\"model\",\"parts\":[{\"type\":\"text\",\"content\":\"The current temperature in New Delhi is 35\\u00b0C, and in San Francisco, it is 25\\u00b0C.\"}],\"finish_reason\":\"stop\"}]}" + }, + "logName": "projects/fakeproject/logs/test", + "resource": { + "labels": { + "location": "global", + "namespace": "", + "node_id": "" + }, + "type": "generic_node" + }, + "timestamp": "2025-01-15T21:25:10.997977393Z" + } + ], + "partialSuccess": true + } +] diff --git a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py index 57fe8bbe..0f6d5b01 100644 --- a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py +++ b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py @@ -46,6 +46,74 @@ PROJECT_ID = "fakeproject" +GEN_AI_DICT = { + "gen_ai.input.messages": ( + { + "role": "user", + "parts": ( + { + "type": "text", + "content": "Get weather details in New Delhi and San Francisco?", + }, + ), + }, + { + "role": "model", + "parts": ( + { + "type": "tool_call", + "arguments": {"location": "New Delhi"}, + "name": "get_current_weather", + "id": "get_current_weather_0", + }, + { + "type": "tool_call", + "arguments": {"location": "San Francisco"}, + "name": "get_current_weather", + "id": "get_current_weather_1", + }, + ), + }, + { + "role": "user", + "parts": ( + { + "type": "tool_call_response", + "response": { + "content": '{"temperature": 35, "unit": "C"}' + }, + "id": "get_current_weather_0", + }, + { + "type": "tool_call_response", + "response": { + "content": '{"temperature": 25, "unit": "C"}' + }, + "id": "get_current_weather_1", + }, + ), + }, + ), + "gen_ai.system_instructions": ( + { + "type": "text", + "content": "You are a clever language model", + }, + ), + "gen_ai.output.messages": ( + { + "role": "model", + "parts": ( + { + "type": "text", + "content": "The current temperature in New Delhi is 35°C, and in San Francisco, it is 25°C.", + }, + ), + "finish_reason": "stop", + }, + ), +} + def test_too_large_log_raises_warning(caplog) -> None: client = LoggingServiceV2Client(credentials=AnonymousCredentials()) @@ -165,73 +233,7 @@ def test_convert_gen_ai_body( log_record=LogRecord( event_name="gen_ai.client.inference.operation.details", timestamp=1736976310997977393, - body={ - "gen_ai.input.messages": ( - { - "role": "user", - "parts": ( - { - "type": "text", - "content": "Get weather details in New Delhi and San Francisco?", - }, - ), - }, - { - "role": "model", - "parts": ( - { - "type": "tool_call", - "arguments": {"location": "New Delhi"}, - "name": "get_current_weather", - "id": "get_current_weather_0", - }, - { - "type": "tool_call", - "arguments": {"location": "San Francisco"}, - "name": "get_current_weather", - "id": "get_current_weather_1", - }, - ), - }, - { - "role": "user", - "parts": ( - { - "type": "tool_call_response", - "response": { - "content": '{"temperature": 35, "unit": "C"}' - }, - "id": "get_current_weather_0", - }, - { - "type": "tool_call_response", - "response": { - "content": '{"temperature": 25, "unit": "C"}' - }, - "id": "get_current_weather_1", - }, - ), - }, - ), - "gen_ai.system_instructions": ( - { - "type": "text", - "content": "You are a clever language model", - }, - ), - "gen_ai.output.messages": ( - { - "role": "model", - "parts": ( - { - "type": "text", - "content": "The current temperature in New Delhi is 35°C, and in San Francisco, it is 25°C.", - }, - ), - "finish_reason": "stop", - }, - ), - }, + body=GEN_AI_DICT, ), instrumentation_scope=InstrumentationScope("test"), ) @@ -303,3 +305,25 @@ def test_convert_various_types_of_bodies( ] cloudloggingfake.exporter.export(log_data) assert cloudloggingfake.get_calls() == snapshot_writelogentrycalls + + +def test_convert_various_types_of_attributes( + cloudloggingfake: CloudLoggingFake, + snapshot_writelogentrycalls: List[WriteLogEntriesCall], +) -> None: + log_data = [ + LogData( + log_record=LogRecord( + attributes={ + "a": [{"key": b"bytes"}], + "b": [True, False, False, True], + "c": {"a_dict": "abcd", "akey": 1234}, + "d": GEN_AI_DICT, + }, + timestamp=1736976310997977393, + ), + instrumentation_scope=InstrumentationScope("test"), + ) + ] + cloudloggingfake.exporter.export(log_data) + assert cloudloggingfake.get_calls() == snapshot_writelogentrycalls From eb9fe9c8d8009251e017f2814a8c6851f7eca6ee Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Tue, 7 Oct 2025 13:20:47 +0000 Subject: [PATCH 2/3] Add changelog, log exception --- .../CHANGELOG.md | 4 ++-- .../exporter/cloud_logging/__init__.py | 20 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/opentelemetry-exporter-gcp-logging/CHANGELOG.md b/opentelemetry-exporter-gcp-logging/CHANGELOG.md index 3f4b8c84..2f848f85 100644 --- a/opentelemetry-exporter-gcp-logging/CHANGELOG.md +++ b/opentelemetry-exporter-gcp-logging/CHANGELOG.md @@ -2,8 +2,8 @@ ## Unreleased -- Added support for when a `Mapping[str, bytes]` or `Mapping[str, List[bytes]]` is in `LogRecord.body`. -- Added support for when a `Mapping[str, List[Mapping]]` is in `LogRecord.body`. +- Added support for when a `Mapping[str, bytes]` or `Mapping[str, List[bytes]]` is in `LogRecord.body` or `LogRecord.attributes`. +- Added support for when a `Mapping[str, List[Mapping]]` is in `LogRecord.body` or `LogRecord.attributes`. - Do not call `logging.warning` when `LogRecord.body` is of None type, instead leave `LogEntry.payload` empty. - Update opentelemetry-api/sdk dependencies to 1.3. diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py index d62dde63..3adf881e 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py @@ -113,7 +113,7 @@ def default(self, o: Any) -> Any: return super().default(o) -def _convert_any_value_to_string(value: Any, debug_location: str) -> str: +def _convert_any_value_to_string(value: Any) -> str: if isinstance(value, bool): return "true" if value else "false" if isinstance(value, bytes): @@ -122,12 +122,14 @@ def _convert_any_value_to_string(value: Any, debug_location: str) -> str: return str(value) if isinstance(value, (list, tuple, Mapping)): return json.dumps(value, separators=(",", ":"), cls=_GenAiJsonEncoder) - logging.warning( - "Unexpected type %s found in %s, this field will not be added to the LogEntry.", - type(value), - debug_location, - ) - return "" + try: + return str(value) + except Exception as exc: # pylint: disable=broad-except + logging.exception( + "Error mapping AnyValue to string, this field will not be added to the LogEntry: %s", + exc, + ) + return "" # Be careful not to mutate original body. Make copies of anything that needs to change. @@ -189,7 +191,7 @@ def _set_payload_in_log_entry(log_entry: LogEntry, body: AnyValue): log_entry.text_payload = base64.b64encode(body).decode() elif body is not None: log_entry.text_payload = _convert_any_value_to_string( - body, "LogRecord.body" + body ) @@ -279,7 +281,7 @@ def export(self, batch: Sequence[LogData]): ] log_entry.labels = { k: _convert_any_value_to_string( - v, "LogRecord.Attribute {}".format(k) + v ) for k, v in attributes.items() } From 16836e824657db1be9c0d47df528fd39e7002064 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Tue, 7 Oct 2025 13:22:39 +0000 Subject: [PATCH 3/3] Fix formatting --- .../opentelemetry/exporter/cloud_logging/__init__.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py index 3adf881e..ec77c72f 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py @@ -124,7 +124,7 @@ def _convert_any_value_to_string(value: Any) -> str: return json.dumps(value, separators=(",", ":"), cls=_GenAiJsonEncoder) try: return str(value) - except Exception as exc: # pylint: disable=broad-except + except Exception as exc: # pylint: disable=broad-except logging.exception( "Error mapping AnyValue to string, this field will not be added to the LogEntry: %s", exc, @@ -190,9 +190,7 @@ def _set_payload_in_log_entry(log_entry: LogEntry, body: AnyValue): else: log_entry.text_payload = base64.b64encode(body).decode() elif body is not None: - log_entry.text_payload = _convert_any_value_to_string( - body - ) + log_entry.text_payload = _convert_any_value_to_string(body) def is_log_id_valid(log_id: str) -> bool: @@ -280,9 +278,7 @@ def export(self, batch: Sequence[LogData]): log_record.severity_number.value # type: ignore[index] ] log_entry.labels = { - k: _convert_any_value_to_string( - v - ) + k: _convert_any_value_to_string(v) for k, v in attributes.items() } if log_record.event_name: