From 2b86723875a0fe847d83af3da332a1f89be641e0 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Fri, 24 Jan 2025 13:59:15 +0000 Subject: [PATCH 01/13] Create a OTLP to GCP Logs Exporter. Takes OTLP logs, converts them to LogEntry protos and writes them to Cloud Logging. Uses the same conversion logic as the go logs exporter. --- .gitignore | 1 + .../CHANGELOG.md | 3 + opentelemetry-exporter-gcp-logging/LICENSE | 201 ++++++++++++++++++ .../MANIFEST.in | 9 + opentelemetry-exporter-gcp-logging/README.rst | 85 ++++++++ opentelemetry-exporter-gcp-logging/mypy.ini | 7 + opentelemetry-exporter-gcp-logging/setup.cfg | 44 ++++ opentelemetry-exporter-gcp-logging/setup.py | 34 +++ .../exporter/cloud_logging/__init__.py | 189 ++++++++++++++++ .../cloud_logging/environment_variables.py | 23 ++ .../exporter/cloud_logging/version.py | 15 ++ .../test_cloud_logging/test_convert_otlp.json | 36 ++++ .../tests/conftest.py | 19 ++ .../tests/fixtures/cloud_logging_fake.py | 116 ++++++++++ .../tests/fixtures/snapshot_logging_calls.py | 52 +++++ .../tests/test_cloud_logging.py | 94 ++++++++ release.py | 1 + tox.ini | 16 +- 18 files changed, 939 insertions(+), 6 deletions(-) create mode 100644 opentelemetry-exporter-gcp-logging/CHANGELOG.md create mode 100644 opentelemetry-exporter-gcp-logging/LICENSE create mode 100644 opentelemetry-exporter-gcp-logging/MANIFEST.in create mode 100644 opentelemetry-exporter-gcp-logging/README.rst create mode 100644 opentelemetry-exporter-gcp-logging/mypy.ini create mode 100644 opentelemetry-exporter-gcp-logging/setup.cfg create mode 100644 opentelemetry-exporter-gcp-logging/setup.py create mode 100644 opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py create mode 100644 opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/environment_variables.py create mode 100644 opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/version.py create mode 100644 opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp.json create mode 100644 opentelemetry-exporter-gcp-logging/tests/conftest.py create mode 100644 opentelemetry-exporter-gcp-logging/tests/fixtures/cloud_logging_fake.py create mode 100644 opentelemetry-exporter-gcp-logging/tests/fixtures/snapshot_logging_calls.py create mode 100644 opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py diff --git a/.gitignore b/.gitignore index d0b21a34..fbe27547 100644 --- a/.gitignore +++ b/.gitignore @@ -115,6 +115,7 @@ venv/ ENV/ env.bak/ venv.bak/ +myproject/ # Spyder project settings .spyderproject diff --git a/opentelemetry-exporter-gcp-logging/CHANGELOG.md b/opentelemetry-exporter-gcp-logging/CHANGELOG.md new file mode 100644 index 00000000..1512c421 --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/CHANGELOG.md @@ -0,0 +1,3 @@ +# Changelog + +## Unreleased diff --git a/opentelemetry-exporter-gcp-logging/LICENSE b/opentelemetry-exporter-gcp-logging/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/opentelemetry-exporter-gcp-logging/MANIFEST.in b/opentelemetry-exporter-gcp-logging/MANIFEST.in new file mode 100644 index 00000000..aed3e332 --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/MANIFEST.in @@ -0,0 +1,9 @@ +graft src +graft tests +global-exclude *.pyc +global-exclude *.pyo +global-exclude __pycache__/* +include CHANGELOG.md +include MANIFEST.in +include README.rst +include LICENSE diff --git a/opentelemetry-exporter-gcp-logging/README.rst b/opentelemetry-exporter-gcp-logging/README.rst new file mode 100644 index 00000000..822ef153 --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/README.rst @@ -0,0 +1,85 @@ +OpenTelemetry Google Cloud Monitoring Exporter +============================================== + +.. image:: https://badge.fury.io/py/opentelemetry-exporter-gcp-monitoring.svg + :target: https://badge.fury.io/py/opentelemetry-exporter-gcp-monitoring + +.. image:: https://readthedocs.org/projects/google-cloud-opentelemetry/badge/?version=latest + :target: https://google-cloud-opentelemetry.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + +This library provides support for exporting metrics to Google Cloud +Monitoring. + +To get started with instrumentation in Google Cloud, see `Generate traces and metrics with +Python `_. + +To learn more about instrumentation and observability, including opinionated recommendations +for Google Cloud Observability, visit `Instrumentation and observability +`_. + +For resource detection and GCP trace context propagation, see +`opentelemetry-tools-google-cloud +`_. For the +Google Cloud Trace exporter, see `opentelemetry-exporter-gcp-trace +`_. + +Installation +------------ + +.. code:: bash + + pip install opentelemetry-exporter-gcp-monitoring + +Usage +----- + +.. code:: python + + import time + + from opentelemetry import metrics + from opentelemetry.exporter.cloud_monitoring import ( + CloudMonitoringMetricsExporter, + ) + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.resources import Resource + + metrics.set_meter_provider( + MeterProvider( + metric_readers=[ + PeriodicExportingMetricReader( + CloudMonitoringMetricsExporter(), export_interval_millis=5000 + ) + ], + resource=Resource.create( + { + "service.name": "basic_metrics", + "service.namespace": "examples", + "service.instance.id": "instance123", + } + ), + ) + ) + meter = metrics.get_meter(__name__) + + # Creates metric workload.googleapis.com/request_counter with monitored resource generic_task + requests_counter = meter.create_counter( + name="request_counter", + description="number of requests", + unit="1", + ) + + staging_labels = {"environment": "staging"} + + for i in range(20): + requests_counter.add(25, staging_labels) + time.sleep(5) + + +References +---------- + +* `Cloud Monitoring `_ +* `OpenTelemetry Project `_ diff --git a/opentelemetry-exporter-gcp-logging/mypy.ini b/opentelemetry-exporter-gcp-logging/mypy.ini new file mode 100644 index 00000000..f6455a4e --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/mypy.ini @@ -0,0 +1,7 @@ +[mypy] +namespace_packages = True +explicit_package_bases = True +mypy_path = $MYPY_CONFIG_FILE_DIR/src + +[mypy-google.auth.*] +ignore_missing_imports = True diff --git a/opentelemetry-exporter-gcp-logging/setup.cfg b/opentelemetry-exporter-gcp-logging/setup.cfg new file mode 100644 index 00000000..d14f5d78 --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/setup.cfg @@ -0,0 +1,44 @@ +[metadata] +name = opentelemetry-exporter-gcp-logging +description = Google Cloud Logging exporter for OpenTelemetry +long_description = file: README.rst +long_description_content_type = text/x-rst +author = Google +author_email = opentelemetry-pypi@google.com +url = https://github.com/GoogleCloudPlatform/opentelemetry-operations-python/tree/main/opentelemetry-exporter-gcp-logging +platforms = any +license = Apache-2.0 +classifiers = + Development Status :: 4 - Beta + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 + +[options] +python_requires = >=3.7 +package_dir= + =src +packages=find_namespace: +install_requires = + google-cloud-logging ~= 3.0 + opentelemetry-sdk ~= 1.0 + opentelemetry-resourcedetector-gcp >= 1.5.0dev0, == 1.* + +[options.packages.find] +where = src + +[options.extras_require] +test = + +[options.entry_points] +opentelemetry_metrics_exporter = + gcp_logging = opentelemetry.exporter.cloud_logging:CloudLoggingExporter +opentelemetry_environment_variables = + gcp_logging = opentelemetry.exporter.cloud_logging.environment_variables diff --git a/opentelemetry-exporter-gcp-logging/setup.py b/opentelemetry-exporter-gcp-logging/setup.py new file mode 100644 index 00000000..c50ae35f --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/setup.py @@ -0,0 +1,34 @@ +# Copyright 2021 The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import setuptools + +BASE_DIR = os.path.dirname(__file__) +VERSION_FILENAME = os.path.join( + BASE_DIR, + "src", + "opentelemetry", + "exporter", + "cloud_logging", + "version.py", +) +PACKAGE_INFO = {} +with open(VERSION_FILENAME) as f: + exec(f.read(), PACKAGE_INFO) + +setuptools.setup( + version=PACKAGE_INFO["__version__"], + package_data={"opentelemetry": ["py.typed"]}, +) diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py new file mode 100644 index 00000000..ead4dce6 --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py @@ -0,0 +1,189 @@ +from opentelemetry.sdk._logs.export import LogExporter +from opentelemetry.sdk._logs import LogData +import google.auth +import datetime +from typing import Optional, Sequence +import logging +from google.logging.type.log_severity_pb2 import LogSeverity # type: ignore +from google.cloud.logging_v2.services.logging_service_v2 import LoggingServiceV2Client +from google.cloud.logging_v2.services.logging_service_v2.transports.grpc import ( + LoggingServiceV2GrpcTransport, +) +from google.cloud.logging_v2.types.log_entry import LogEntry +from opentelemetry.resourcedetector.gcp_resource_detector._mapping import ( + get_monitored_resource, +) +from google.api.monitored_resource_pb2 import ( + MonitoredResource, # type: ignore +) +from google.protobuf.timestamp_pb2 import Timestamp +import urllib.parse +from google.protobuf.struct_pb2 import Struct +from opentelemetry.exporter.cloud_logging.version import __version__ +from opentelemetry.sdk import version as opentelemetry_sdk_version + + +DEFAULT_MAX_ENTRY_SIZE = 256000 # 256 KB +DEFAULT_MAX_REQUEST_SIZE = 10000000 # 10 MB + +HTTP_REQUEST_ATTRIBUTE_KEY = "gcp.http_request" +LOG_NAME_ATTRIBUTE_KEY = "gcp.log_name" +SOURCE_LOCATION_ATTRIBUTE_KEY = "gcp.source_location" +TRACE_SAMPLED_ATTRIBUTE_KEY = "gcp.trace_sampled" +PROJECT_ID_ATTRIBUTE_KEY = "gcp.project_id" +_OTEL_SDK_VERSION = opentelemetry_sdk_version.__version__ +_USER_AGENT = f"opentelemetry-python {_OTEL_SDK_VERSION}; google-cloud-logging-exporter {__version__}" + +# Set user-agent metadata, see https://github.com/grpc/grpc/issues/23644 and default options +# from +# https://github.com/googleapis/python-logging/blob/5309478c054d0f2b9301817fd835f2098f51dc3a/google/cloud/logging_v2/services/logging_service_v2/transports/grpc.py#L179-L182 +_OPTIONS = [ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ("grpc.primary_user_agent", _USER_AGENT), +] + +# severityMapping maps the integer severity level values from OTel [0-24] +# to matching Cloud Logging severity levels. +SEVERITY_MAPPING = { + 0: LogSeverity.DEFAULT, # Default, 0 + 1: LogSeverity.DEBUG, # + 2: LogSeverity.DEBUG, # + 3: LogSeverity.DEBUG, # + 4: LogSeverity.DEBUG, # + 5: LogSeverity.DEBUG, # + 6: LogSeverity.DEBUG, # + 7: LogSeverity.DEBUG, # + 8: LogSeverity.DEBUG, # 1-8 -> Debug + 9: LogSeverity.INFO, # + 10: LogSeverity.INFO, # 9-10 -> Info + 11: LogSeverity.NOTICE, # + 12: LogSeverity.NOTICE, # 11-12 -> Notice + 13: LogSeverity.WARNING, # + 14: LogSeverity.WARNING, # + 15: LogSeverity.WARNING, # + 16: LogSeverity.WARNING, # 13-16 -> Warning + 17: LogSeverity.ERROR, # + 18: LogSeverity.ERROR, # + 19: LogSeverity.ERROR, # + 20: LogSeverity.ERROR, # 17-20 -> Error + 21: LogSeverity.CRITICAL, # + 22: LogSeverity.CRITICAL, # 21-22 -> Critical + 23: LogSeverity.ALERT, # 23 -> Alert + 24: LogSeverity.EMERGENCY, # 24 -> Emergency +} + + +class CloudLoggingExporter(LogExporter): + def __init__( + self, + project_id: Optional[str] = None, + default_log_name: Optional[str] = None, + client: Optional[LoggingServiceV2Client] = None, + ): + self.project_id: str + if not project_id: + _, default_project_id = google.auth.default() + self.project_id = str(default_project_id) + else: + self.project_id = project_id + self.default_log_name = default_log_name + self.client = client or LoggingServiceV2Client( + transport=LoggingServiceV2GrpcTransport( + channel=LoggingServiceV2GrpcTransport.create_channel( + options=_OPTIONS, + ) + ) + ) + self.service_resource_labels = True + + def export(self, batch: Sequence[LogData]): + log_entries = [] + for log_data in batch: + log_record = log_data.log_record + attributes = log_record.attributes or {} + project_id = self.project_id + if attributes.get(PROJECT_ID_ATTRIBUTE_KEY): + project_id = str(attributes.get(PROJECT_ID_ATTRIBUTE_KEY)) + log_name = self.default_log_name + if attributes.get(LOG_NAME_ATTRIBUTE_KEY): + log_name = str(attributes.get(LOG_NAME_ATTRIBUTE_KEY)) + if not log_name: + logging.warning( + "No log name provided, cannot write log to Cloud Logging. Set the 'default_log_name' option, or add the 'gcp.log_name' attribute to set a log name." + ) + continue + monitored_resource_data = get_monitored_resource(log_record.resource) + # convert it to proto + monitored_resource = ( + MonitoredResource( + type=monitored_resource_data.type, + labels=monitored_resource_data.labels, + ) + if monitored_resource_data + else None + ) + # if timestamp is unset, fall back to observed_time_unix_nano as recommended + # (see https://github.com/open-telemetry/opentelemetry-proto/blob/4abbb78/opentelemetry/proto/logs/v1/logs.proto#L176-L179) + ts = Timestamp() + if log_record.timestamp or log_record.observed_timestamp: + ts.FromNanoseconds( + log_record.timestamp or log_record.observed_timestamp + ) + else: + ts.FromDatetime(datetime.datetime.now()) + log_name = "projects/{}/logs/{}".format( + project_id, urllib.parse.quote_plus(log_name) + ) + log_entry = LogEntry() + log_entry.timestamp = ts + log_entry.log_name = log_name + log_entry.resource = monitored_resource + attrs_map = {k: v for k, v in attributes.items()} + log_entry.trace_sampled = ( + log_record.trace_flags is not None and log_record.trace_flags.sampled + ) + if TRACE_SAMPLED_ATTRIBUTE_KEY in attrs_map: + log_entry.trace_sampled |= bool(attrs_map[TRACE_SAMPLED_ATTRIBUTE_KEY]) + del attrs_map[TRACE_SAMPLED_ATTRIBUTE_KEY] + if log_record.trace_id: + log_entry.trace = "projects/{}/traces/{}".format( + project_id, log_record.trace_id + ) + if log_record.span_id: + log_entry.span_id = str(hex(log_record.span_id))[2:] + if log_record.severity_number in SEVERITY_MAPPING: + log_entry.severity = SEVERITY_MAPPING[log_record.severity_number] + log_entry.labels = {k: str(v) for k, v in attrs_map.items()} + if type(log_record.body) is dict: + s = Struct() + s.update(log_record.body) + log_entry.json_payload = s + log_entries.append(log_entry) + + self._write_log_entries(log_entries) + + def _write_log_entries(self, log_entries: list[LogEntry]): + batch = [] + batch_byte_size = 0 + for entry in log_entries: + msg_size = LogEntry.pb(entry).ByteSize() + if msg_size > DEFAULT_MAX_ENTRY_SIZE: + logging.warning( + "Cannot write log that is {} bytes which exceeds Cloud Logging's maximum limit of {}.".format( + msg_size, DEFAULT_MAX_ENTRY_SIZE + ) + ) + continue + if msg_size + batch_byte_size > DEFAULT_MAX_REQUEST_SIZE: + self.client.write_log_entries(entries=batch) + batch = [entry] + batch_byte_size = msg_size + else: + batch.append(entry) + batch_byte_size += msg_size + if batch: + self.client.write_log_entries(entries=batch) + + def shutdown(self): + pass diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/environment_variables.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/environment_variables.py new file mode 100644 index 00000000..48cf7283 --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/environment_variables.py @@ -0,0 +1,23 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +OTEL_EXPORTER_GCP_LOGGING_PROJECT_ID = ( + "OTEL_EXPORTER_GCP_LOGGING_PROJECT_ID" +) +""" +.. envvar:: OTEL_EXPORTER_GCP_LOGGING_PROJECT_ID + + GCP project ID for the exporter to send logs to. Equivalent to constructor parameter to + :class:`opentelemetry.exporter.cloud_logging.CloudLoggingExporter`. +""" diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/version.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/version.py new file mode 100644 index 00000000..553aa709 --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/version.py @@ -0,0 +1,15 @@ +# Copyright 2021 The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.9b0" diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp.json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp.json new file mode 100644 index 00000000..ad6a6d68 --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp.json @@ -0,0 +1,36 @@ +[ + { + "entries": [ + { + "jsonPayload": { + "kvlistValue": { + "values": [ + { + "key": "content", + "value": { + "stringValue": "You're a helpful assistant." + } + } + ] + } + }, + "labels": { + "event.name": "gen_ai.system.message", + "gen_ai.system": "openai" + }, + "logName": "projects/fakeproject/logs/test", + "resource": { + "labels": { + "location": "global", + "namespace": "", + "node_id": "" + }, + "type": "generic_node" + }, + "spanId": "16", + "timestamp": "2025-01-15T21:25:10.997977393Z", + "trace": "projects/fakeproject/traces/25" + } + ] + } +] diff --git a/opentelemetry-exporter-gcp-logging/tests/conftest.py b/opentelemetry-exporter-gcp-logging/tests/conftest.py new file mode 100644 index 00000000..c8ab2ac5 --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/conftest.py @@ -0,0 +1,19 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=unused-import + +# import fixtures to be made available to other tests +from fixtures.cloud_logging_fake import fixture_cloudloggingfake +from fixtures.snapshot_logging_calls import fixture_snapshot_writelogentrycalls diff --git a/opentelemetry-exporter-gcp-logging/tests/fixtures/cloud_logging_fake.py b/opentelemetry-exporter-gcp-logging/tests/fixtures/cloud_logging_fake.py new file mode 100644 index 00000000..f23e28c5 --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/fixtures/cloud_logging_fake.py @@ -0,0 +1,116 @@ +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from functools import partial +from typing import Callable, Iterable, List, cast +from unittest.mock import patch +from google.cloud.logging_v2.services.logging_service_v2.transports.grpc import ( + LoggingServiceV2GrpcTransport, +) + +from google.cloud.logging_v2.types.logging import WriteLogEntriesRequest +import grpc +import pytest + +# pylint: disable=no-name-in-module +from google.protobuf.empty_pb2 import Empty +from grpc import ( + GenericRpcHandler, + ServicerContext, + insecure_channel, + method_handlers_generic_handler, + unary_unary_rpc_method_handler, +) +from opentelemetry.exporter.cloud_logging import ( + CloudLoggingExporter, +) + + +@dataclass +class WriteLogEntriesCall: + write_log_entries_request: WriteLogEntriesRequest + user_agent: str + + +PROJECT_ID = "fakeproject" + + +class FakeWriteLogEntriesHandler(GenericRpcHandler): + """gRPC handler captures request protos made to Cloud Logging's WriteLogEntries.""" + + _service = "google.logging.v2.LoggingServiceV2" + _method = "WriteLogEntries" + + def __init__(self): + # pylint: disable=no-member + super().__init__() + self._calls: List[WriteLogEntriesCall] = [] + + def write_log_entries_request_handler( + req: WriteLogEntriesRequest, context: ServicerContext + ) -> Empty: + metadata_dict = dict(context.invocation_metadata()) + self._calls.append( + WriteLogEntriesCall( + write_log_entries_request=req, + user_agent=cast(str, metadata_dict["user-agent"]), + ) + ) + return Empty() + + self._wrapped = method_handlers_generic_handler( + self._service, + { + self._method: unary_unary_rpc_method_handler( + write_log_entries_request_handler, + WriteLogEntriesRequest.deserialize, + Empty.SerializeToString, + ) + }, + ) + + def service(self, handler_call_details): + res = self._wrapped.service(handler_call_details) + return res + + def get_calls(self) -> List[WriteLogEntriesCall]: + """Returns calls made to WriteLogEntries""" + return self._calls + + +@dataclass +class CloudLoggingFake: + exporter: CloudLoggingExporter + get_calls: Callable[[], List[WriteLogEntriesCall]] + + +@pytest.fixture(name="cloudloggingfake") +def fixture_cloudloggingfake() -> Iterable[CloudLoggingFake]: + """Fixture providing faked Cloud Logging api with captured requests""" + + handler = FakeWriteLogEntriesHandler() + server = None + + try: + # Run in a single thread to serialize requests + with ThreadPoolExecutor(1) as executor: + server = grpc.server(executor, handlers=[handler]) + port = server.add_insecure_port("localhost:0") + server.start() + + # patch LoggingServiceV2Transport.create_channel staticmethod to return an insecure + # channel but otherwise respect any parameters passed to it + with patch.object( + LoggingServiceV2GrpcTransport, + "create_channel", + partial(insecure_channel, target=f"localhost:{port}"), + ): + yield CloudLoggingFake( + exporter=CloudLoggingExporter( + project_id=PROJECT_ID, + default_log_name="test", + ), + get_calls=handler.get_calls, + ) + finally: + if server: + server.stop(None) diff --git a/opentelemetry-exporter-gcp-logging/tests/fixtures/snapshot_logging_calls.py b/opentelemetry-exporter-gcp-logging/tests/fixtures/snapshot_logging_calls.py new file mode 100644 index 00000000..7141b39a --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/fixtures/snapshot_logging_calls.py @@ -0,0 +1,52 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, cast, List + +import pytest +from fixtures.cloud_logging_fake import WriteLogEntriesCall +from google.protobuf import json_format +from syrupy.extensions.json import JSONSnapshotExtension +from syrupy.types import ( + PropertyFilter, + PropertyMatcher, + SerializableData, + SerializedData, +) + + +# pylint: disable=too-many-ancestors +class WriteLogEntryCallSnapshotExtension(JSONSnapshotExtension): + """syrupy extension to serialize WriteLogEntriesRequest to JSON for storing as a snapshot.""" + + def serialize( + self, + data: SerializableData, + *, + exclude: Optional[PropertyFilter] = None, + matcher: Optional[PropertyMatcher] = None, + ) -> SerializedData: + json = [ + json_format.MessageToDict( + type(call.write_log_entries_request).pb(call.write_log_entries_request) + ) + for call in cast(List[WriteLogEntriesCall], data) + ] + return super().serialize(json, exclude=exclude, matcher=matcher) + + +@pytest.fixture(name="snapshot_writelogentrycalls") +def fixture_snapshot_writelogentrycalls(snapshot): + """Fixture for snapshot testing of WriteLogEntriesCalls""" + return snapshot.use_extension(WriteLogEntryCallSnapshotExtension)() diff --git a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py new file mode 100644 index 00000000..1b394429 --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py @@ -0,0 +1,94 @@ +""" +Some tests in this file use [syrupy](https://github.com/tophat/syrupy) for snapshot testing aka +golden testing. The Cloud Logging API calls are captured with a gRPC fake and compared to the existing +snapshot file in the __snapshots__ directory. + +If an expected behavior change is made to the exporter causing these tests to fail, regenerate +the snapshots by running tox to pass the --snapshot-update flag to pytest: + +```sh +tox -e py310-ci-test-cloudlogging -- --snapshot-update +``` + +Be sure to review the changes. +""" + +from google.cloud.logging_v2.services.logging_service_v2 import LoggingServiceV2Client + +from fixtures.cloud_logging_fake import CloudLoggingFake +from opentelemetry.sdk._logs._internal import LogRecord +from opentelemetry._logs.severity import SeverityNumber +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk._logs import LogData +from opentelemetry.sdk.util.instrumentation import InstrumentationScope +from opentelemetry.exporter.cloud_logging import ( + CloudLoggingExporter, +) +from typing import List +from fixtures.cloud_logging_fake import WriteLogEntriesCall +import re + +PROJECT_ID = "fakeproject" + + +def test_create_cloud_logging_exporter(caplog) -> None: + CloudLoggingExporter(default_log_name="test") + client = LoggingServiceV2Client() + CloudLoggingExporter(project_id=PROJECT_ID, client=client) + + +def test_invalid_otlp_entries_raise_warnings(caplog) -> None: + client = LoggingServiceV2Client() + no_default_logname = CloudLoggingExporter(project_id=PROJECT_ID, client=client) + no_default_logname.export( + [ + LogData( + log_record=LogRecord(resource=Resource({})), + instrumentation_scope=InstrumentationScope("test"), + ) + ] + ) + assert len(caplog.records) == 1 + assert "No log name provided" in caplog.text + + +def test_convert_otlp( + cloudloggingfake: CloudLoggingFake, + snapshot_writelogentrycalls: List[WriteLogEntriesCall], +) -> None: + # Create a new LogRecord object + log_record = LogRecord( + timestamp=1736976310997977393, + severity_number=SeverityNumber(20), + trace_id=25, + span_id=22, + attributes={"gen_ai.system": "openai", "event.name": "gen_ai.system.message"}, + body={ + "kvlistValue": { + "values": [ + { + "key": "content", + "value": {"stringValue": "You're a helpful assistant."}, + } + ] + } + }, + # Not sure why I'm getting AttributeError: 'NoneType' object has no attribute 'attributes' when unset. + resource=Resource({}), + ) + + log_data = [ + LogData( + log_record=log_record, instrumentation_scope=InstrumentationScope("test") + ) + ] + cloudloggingfake.exporter.export(log_data) + assert cloudloggingfake.get_calls() == snapshot_writelogentrycalls + for call in cloudloggingfake.get_calls(): + assert ( + re.match( + r"^opentelemetry-python \S+; google-cloud-logging-exporter \S+ grpc-python/\S+", + call.user_agent, + ) + is not None + ) diff --git a/release.py b/release.py index 1d5b8417..db87abc9 100755 --- a/release.py +++ b/release.py @@ -54,6 +54,7 @@ ALTERNATE_SUFFIXES = { # Mark monitoring and resource detector alpha "opentelemetry-exporter-gcp-monitoring": "a0", + "opentelemetry-exporter-gcp-logging": "a0", "opentelemetry-resourcedetector-gcp": "a0", } diff --git a/tox.ini b/tox.ini index 7b2e84e9..68746852 100644 --- a/tox.ini +++ b/tox.ini @@ -5,13 +5,13 @@ requires = tox>=4 envlist = ; Add the `ci` factor to any env that should be running during CI. - py3{7,8,9,10,11,12}-ci-test-{cloudtrace,cloudmonitoring,propagator,resourcedetector} - {lint,mypy}-ci-{cloudtrace,cloudmonitoring,propagator,resourcedetector} + py3{7,8,9,10,11,12}-ci-test-{cloudtrace,cloudmonitoring,propagator,resourcedetector, cloudlogging} + {lint,mypy}-ci-{cloudtrace,cloudmonitoring,propagator,resourcedetector, cloudlogging} docs-ci ; These are development commands and share the same virtualenv within each ; package root directory - {fix}-{cloudtrace,cloudmonitoring,propagator,resourcedetector} + {fix}-{cloudtrace,cloudmonitoring,propagator,resourcedetector, cloudlogging} ; Installs dev depenedencies and all packages in this repo with editable ; install into a single env. Useful for editor autocompletion @@ -27,6 +27,7 @@ base_deps = monorepo_deps = cloudmonitoring: -e {toxinidir}/opentelemetry-resourcedetector-gcp/ cloudtrace: -e {toxinidir}/opentelemetry-resourcedetector-gcp/ + cloudlogging: -e {toxinidir}/opentelemetry-resourcedetector-gcp/ dev_basepython = python3.10 dev_deps = @@ -55,10 +56,11 @@ setenv = ; for package specific commands, use these envvars to cd into the directory cloudtrace: PACKAGE_NAME = opentelemetry-exporter-gcp-trace cloudmonitoring: PACKAGE_NAME = opentelemetry-exporter-gcp-monitoring + cloudlogging: PACKAGE_NAME = opentelemetry-exporter-gcp-logging propagator: PACKAGE_NAME = opentelemetry-propagator-gcp resourcedetector: PACKAGE_NAME = opentelemetry-resourcedetector-gcp -[testenv:py3{7,8,9,10,11,12}-ci-test-{cloudtrace,cloudmonitoring,propagator,resourcedetector}] +[testenv:py3{7,8,9,10,11,12}-ci-test-{cloudtrace,cloudmonitoring,propagator,resourcedetector, cloudlogging}] deps = ; editable install the package itself -e {toxinidir}/{env:PACKAGE_NAME} @@ -79,7 +81,7 @@ allowlist_externals = bash {toxinidir}/get_mock_server.sh -[testenv:{lint,mypy}-ci-{cloudtrace,cloudmonitoring,propagator,resourcedetector}] +[testenv:{lint,mypy}-ci-{cloudtrace,cloudmonitoring,propagator,resourcedetector, cloudlogging}] basepython = {[constants]dev_basepython} deps = ; editable install the package itself @@ -107,13 +109,14 @@ allowlist_externals = make bash -[testenv:{fix}-{cloudtrace,cloudmonitoring,propagator,resourcedetector}] +[testenv:{fix}-{cloudtrace,cloudmonitoring,propagator,resourcedetector, cloudlogging}] basepython = {[constants]dev_basepython} envdir = cloudtrace: opentelemetry-exporter-gcp-trace/venv cloudmonitoring: opentelemetry-exporter-gcp-monitoring/venv propagator: opentelemetry-propagator-gcp/venv resourcedetector: opentelemetry-resourcedetector-gcp/venv + cloudlogging: opentelemetry-exporter-gcp-logging/venv deps = {[constants]dev_deps} {[constants]monorepo_deps} @@ -134,3 +137,4 @@ deps = -e opentelemetry-exporter-gcp-trace -e opentelemetry-propagator-gcp -e opentelemetry-resourcedetector-gcp + -e opentelemetry-exporter-gcp-logging From 56c33797487dc3deb192e6c4f690a5f301f3ae86 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Mon, 27 Jan 2025 17:27:53 +0000 Subject: [PATCH 02/13] Make sure the apache license and current year is on every file. Give log name a default value. Fix some types / formatting that running the tox commands surfaced. --- opentelemetry-exporter-gcp-logging/README.rst | 85 +++++++++---------- opentelemetry-exporter-gcp-logging/setup.cfg | 2 +- opentelemetry-exporter-gcp-logging/setup.py | 2 +- .../exporter/cloud_logging/__init__.py | 81 +++++++++++------- .../cloud_logging/environment_variables.py | 6 +- .../exporter/cloud_logging/version.py | 2 +- .../tests/conftest.py | 2 +- .../tests/fixtures/cloud_logging_fake.py | 23 +++-- .../tests/fixtures/snapshot_logging_calls.py | 8 +- .../tests/test_cloud_logging.py | 55 ++++++++---- 10 files changed, 160 insertions(+), 106 deletions(-) diff --git a/opentelemetry-exporter-gcp-logging/README.rst b/opentelemetry-exporter-gcp-logging/README.rst index 822ef153..0d6f7f50 100644 --- a/opentelemetry-exporter-gcp-logging/README.rst +++ b/opentelemetry-exporter-gcp-logging/README.rst @@ -1,15 +1,15 @@ -OpenTelemetry Google Cloud Monitoring Exporter +OpenTelemetry Google Cloud Logging Exporter ============================================== -.. image:: https://badge.fury.io/py/opentelemetry-exporter-gcp-monitoring.svg - :target: https://badge.fury.io/py/opentelemetry-exporter-gcp-monitoring +.. image:: https://badge.fury.io/py/opentelemetry-exporter-gcp-logging.svg + :target: https://badge.fury.io/py/opentelemetry-exporter-gcp-logging .. image:: https://readthedocs.org/projects/google-cloud-opentelemetry/badge/?version=latest :target: https://google-cloud-opentelemetry.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status -This library provides support for exporting metrics to Google Cloud -Monitoring. +This library provides support for exporting logs to Google Cloud +Logging. To get started with instrumentation in Google Cloud, see `Generate traces and metrics with Python `_. @@ -29,57 +29,54 @@ Installation .. code:: bash - pip install opentelemetry-exporter-gcp-monitoring + pip install opentelemetry-exporter-gcp-logging Usage ----- .. code:: python - import time - - from opentelemetry import metrics - from opentelemetry.exporter.cloud_monitoring import ( - CloudMonitoringMetricsExporter, + from opentelemetry.exporter.cloud_logging import ( + CloudLoggingExporter, ) - from opentelemetry.sdk.metrics import MeterProvider - from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk._logs._internal import LogRecord + from opentelemetry._logs.severity import SeverityNumber from opentelemetry.sdk.resources import Resource - - metrics.set_meter_provider( - MeterProvider( - metric_readers=[ - PeriodicExportingMetricReader( - CloudMonitoringMetricsExporter(), export_interval_millis=5000 - ) - ], - resource=Resource.create( - { - "service.name": "basic_metrics", - "service.namespace": "examples", - "service.instance.id": "instance123", - } - ), - ) - ) - meter = metrics.get_meter(__name__) - - # Creates metric workload.googleapis.com/request_counter with monitored resource generic_task - requests_counter = meter.create_counter( - name="request_counter", - description="number of requests", - unit="1", + from opentelemetry.sdk._logs import LogData + from opentelemetry.sdk.util.instrumentation import InstrumentationScope + + + exporter = CloudLoggingExporter(default_log_name='my_log') + exporter.export( + [ + LogData( + log_record=LogRecord( + resource=Resource({}), + timestamp=1736976310997977393, + severity_number=SeverityNumber(20), + attributes={ + "gen_ai.system": "openai", + "event.name": "gen_ai.system.message", + }, + body={ + "kvlistValue": { + "values": [ + { + "key": "content", + "value": {"stringValue": "You're a helpful assistant."}, + } + ] + } + }, + ), + instrumentation_scope=InstrumentationScope("test"), + ) + ] ) - staging_labels = {"environment": "staging"} - - for i in range(20): - requests_counter.add(25, staging_labels) - time.sleep(5) - References ---------- -* `Cloud Monitoring `_ +* `Cloud Logging `_ * `OpenTelemetry Project `_ diff --git a/opentelemetry-exporter-gcp-logging/setup.cfg b/opentelemetry-exporter-gcp-logging/setup.cfg index d14f5d78..7d8850bd 100644 --- a/opentelemetry-exporter-gcp-logging/setup.cfg +++ b/opentelemetry-exporter-gcp-logging/setup.cfg @@ -38,7 +38,7 @@ where = src test = [options.entry_points] -opentelemetry_metrics_exporter = +opentelemetry_logging_exporter = gcp_logging = opentelemetry.exporter.cloud_logging:CloudLoggingExporter opentelemetry_environment_variables = gcp_logging = opentelemetry.exporter.cloud_logging.environment_variables diff --git a/opentelemetry-exporter-gcp-logging/setup.py b/opentelemetry-exporter-gcp-logging/setup.py index c50ae35f..e57f525a 100644 --- a/opentelemetry-exporter-gcp-logging/setup.py +++ b/opentelemetry-exporter-gcp-logging/setup.py @@ -1,4 +1,4 @@ -# Copyright 2021 The OpenTelemetry Authors +# Copyright 2025 The OpenTelemetry Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py index ead4dce6..32abeea9 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py @@ -1,27 +1,41 @@ -from opentelemetry.sdk._logs.export import LogExporter -from opentelemetry.sdk._logs import LogData -import google.auth +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import datetime -from typing import Optional, Sequence import logging -from google.logging.type.log_severity_pb2 import LogSeverity # type: ignore -from google.cloud.logging_v2.services.logging_service_v2 import LoggingServiceV2Client +import urllib.parse +from typing import Optional, Sequence + +import google.auth +from google.api.monitored_resource_pb2 import MonitoredResource # type: ignore +from google.cloud.logging_v2.services.logging_service_v2 import ( + LoggingServiceV2Client, +) from google.cloud.logging_v2.services.logging_service_v2.transports.grpc import ( LoggingServiceV2GrpcTransport, ) from google.cloud.logging_v2.types.log_entry import LogEntry +from google.logging.type.log_severity_pb2 import LogSeverity # type: ignore +from google.protobuf.struct_pb2 import Struct +from google.protobuf.timestamp_pb2 import Timestamp +from opentelemetry.exporter.cloud_logging.version import __version__ from opentelemetry.resourcedetector.gcp_resource_detector._mapping import ( get_monitored_resource, ) -from google.api.monitored_resource_pb2 import ( - MonitoredResource, # type: ignore -) -from google.protobuf.timestamp_pb2 import Timestamp -import urllib.parse -from google.protobuf.struct_pb2 import Struct -from opentelemetry.exporter.cloud_logging.version import __version__ from opentelemetry.sdk import version as opentelemetry_sdk_version - +from opentelemetry.sdk._logs import LogData +from opentelemetry.sdk._logs.export import LogExporter +from opentelemetry.sdk.resources import Resource DEFAULT_MAX_ENTRY_SIZE = 256000 # 256 KB DEFAULT_MAX_REQUEST_SIZE = 10000000 # 10 MB @@ -45,7 +59,7 @@ # severityMapping maps the integer severity level values from OTel [0-24] # to matching Cloud Logging severity levels. -SEVERITY_MAPPING = { +SEVERITY_MAPPING: dict[int, int] = { 0: LogSeverity.DEFAULT, # Default, 0 1: LogSeverity.DEBUG, # 2: LogSeverity.DEBUG, # @@ -87,7 +101,10 @@ def __init__( self.project_id = str(default_project_id) else: self.project_id = project_id - self.default_log_name = default_log_name + if default_log_name: + self.default_log_name = default_log_name + else: + self.default_log_name = "otel_python_inprocess_log_name_temp" self.client = client or LoggingServiceV2Client( transport=LoggingServiceV2GrpcTransport( channel=LoggingServiceV2GrpcTransport.create_channel( @@ -108,14 +125,11 @@ def export(self, batch: Sequence[LogData]): log_name = self.default_log_name if attributes.get(LOG_NAME_ATTRIBUTE_KEY): log_name = str(attributes.get(LOG_NAME_ATTRIBUTE_KEY)) - if not log_name: - logging.warning( - "No log name provided, cannot write log to Cloud Logging. Set the 'default_log_name' option, or add the 'gcp.log_name' attribute to set a log name." - ) - continue - monitored_resource_data = get_monitored_resource(log_record.resource) + monitored_resource_data = get_monitored_resource( + log_record.resource or Resource({}) + ) # convert it to proto - monitored_resource = ( + monitored_resource: Optional[MonitoredResource] = ( MonitoredResource( type=monitored_resource_data.type, labels=monitored_resource_data.labels, @@ -138,13 +152,17 @@ def export(self, batch: Sequence[LogData]): log_entry = LogEntry() log_entry.timestamp = ts log_entry.log_name = log_name - log_entry.resource = monitored_resource + if monitored_resource: + log_entry.resource = monitored_resource attrs_map = {k: v for k, v in attributes.items()} log_entry.trace_sampled = ( - log_record.trace_flags is not None and log_record.trace_flags.sampled + log_record.trace_flags is not None + and log_record.trace_flags.sampled ) if TRACE_SAMPLED_ATTRIBUTE_KEY in attrs_map: - log_entry.trace_sampled |= bool(attrs_map[TRACE_SAMPLED_ATTRIBUTE_KEY]) + log_entry.trace_sampled |= bool( + attrs_map[TRACE_SAMPLED_ATTRIBUTE_KEY] + ) del attrs_map[TRACE_SAMPLED_ATTRIBUTE_KEY] if log_record.trace_id: log_entry.trace = "projects/{}/traces/{}".format( @@ -152,8 +170,13 @@ def export(self, batch: Sequence[LogData]): ) if log_record.span_id: log_entry.span_id = str(hex(log_record.span_id))[2:] - if log_record.severity_number in SEVERITY_MAPPING: - log_entry.severity = SEVERITY_MAPPING[log_record.severity_number] + if ( + log_record.severity_number + and log_record.severity_number in SEVERITY_MAPPING + ): + log_entry.severity = SEVERITY_MAPPING[ + log_record.severity_number + ] log_entry.labels = {k: str(v) for k, v in attrs_map.items()} if type(log_record.body) is dict: s = Struct() @@ -164,7 +187,7 @@ def export(self, batch: Sequence[LogData]): self._write_log_entries(log_entries) def _write_log_entries(self, log_entries: list[LogEntry]): - batch = [] + batch: list[LogEntry] = [] batch_byte_size = 0 for entry in log_entries: msg_size = LogEntry.pb(entry).ByteSize() diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/environment_variables.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/environment_variables.py index 48cf7283..02d8b75e 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/environment_variables.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/environment_variables.py @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -OTEL_EXPORTER_GCP_LOGGING_PROJECT_ID = ( - "OTEL_EXPORTER_GCP_LOGGING_PROJECT_ID" -) +OTEL_EXPORTER_GCP_LOGGING_PROJECT_ID = "OTEL_EXPORTER_GCP_LOGGING_PROJECT_ID" """ .. envvar:: OTEL_EXPORTER_GCP_LOGGING_PROJECT_ID diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/version.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/version.py index 553aa709..5464e913 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/version.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/version.py @@ -1,4 +1,4 @@ -# Copyright 2021 The OpenTelemetry Authors +# Copyright 2025 The OpenTelemetry Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/opentelemetry-exporter-gcp-logging/tests/conftest.py b/opentelemetry-exporter-gcp-logging/tests/conftest.py index c8ab2ac5..a23c6dbc 100644 --- a/opentelemetry-exporter-gcp-logging/tests/conftest.py +++ b/opentelemetry-exporter-gcp-logging/tests/conftest.py @@ -1,4 +1,4 @@ -# Copyright 2022 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/opentelemetry-exporter-gcp-logging/tests/fixtures/cloud_logging_fake.py b/opentelemetry-exporter-gcp-logging/tests/fixtures/cloud_logging_fake.py index f23e28c5..7ecf64f9 100644 --- a/opentelemetry-exporter-gcp-logging/tests/fixtures/cloud_logging_fake.py +++ b/opentelemetry-exporter-gcp-logging/tests/fixtures/cloud_logging_fake.py @@ -1,15 +1,28 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from functools import partial from typing import Callable, Iterable, List, cast from unittest.mock import patch + +import grpc +import pytest from google.cloud.logging_v2.services.logging_service_v2.transports.grpc import ( LoggingServiceV2GrpcTransport, ) - from google.cloud.logging_v2.types.logging import WriteLogEntriesRequest -import grpc -import pytest # pylint: disable=no-name-in-module from google.protobuf.empty_pb2 import Empty @@ -20,9 +33,7 @@ method_handlers_generic_handler, unary_unary_rpc_method_handler, ) -from opentelemetry.exporter.cloud_logging import ( - CloudLoggingExporter, -) +from opentelemetry.exporter.cloud_logging import CloudLoggingExporter @dataclass diff --git a/opentelemetry-exporter-gcp-logging/tests/fixtures/snapshot_logging_calls.py b/opentelemetry-exporter-gcp-logging/tests/fixtures/snapshot_logging_calls.py index 7141b39a..eb6dd44d 100644 --- a/opentelemetry-exporter-gcp-logging/tests/fixtures/snapshot_logging_calls.py +++ b/opentelemetry-exporter-gcp-logging/tests/fixtures/snapshot_logging_calls.py @@ -1,4 +1,4 @@ -# Copyright 2022 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, cast, List +from typing import List, Optional, cast import pytest from fixtures.cloud_logging_fake import WriteLogEntriesCall @@ -39,7 +39,9 @@ def serialize( ) -> SerializedData: json = [ json_format.MessageToDict( - type(call.write_log_entries_request).pb(call.write_log_entries_request) + type(call.write_log_entries_request).pb( + call.write_log_entries_request + ) ) for call in cast(List[WriteLogEntriesCall], data) ] diff --git a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py index 1b394429..33dedddb 100644 --- a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py +++ b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py @@ -1,3 +1,16 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ Some tests in this file use [syrupy](https://github.com/tophat/syrupy) for snapshot testing aka golden testing. The Cloud Logging API calls are captured with a gRPC fake and compared to the existing @@ -13,20 +26,19 @@ Be sure to review the changes. """ -from google.cloud.logging_v2.services.logging_service_v2 import LoggingServiceV2Client +import re +from typing import List -from fixtures.cloud_logging_fake import CloudLoggingFake -from opentelemetry.sdk._logs._internal import LogRecord +from fixtures.cloud_logging_fake import CloudLoggingFake, WriteLogEntriesCall +from google.cloud.logging_v2.services.logging_service_v2 import ( + LoggingServiceV2Client, +) from opentelemetry._logs.severity import SeverityNumber -from opentelemetry.sdk.resources import Resource +from opentelemetry.exporter.cloud_logging import CloudLoggingExporter from opentelemetry.sdk._logs import LogData +from opentelemetry.sdk._logs._internal import LogRecord +from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.util.instrumentation import InstrumentationScope -from opentelemetry.exporter.cloud_logging import ( - CloudLoggingExporter, -) -from typing import List -from fixtures.cloud_logging_fake import WriteLogEntriesCall -import re PROJECT_ID = "fakeproject" @@ -39,17 +51,22 @@ def test_create_cloud_logging_exporter(caplog) -> None: def test_invalid_otlp_entries_raise_warnings(caplog) -> None: client = LoggingServiceV2Client() - no_default_logname = CloudLoggingExporter(project_id=PROJECT_ID, client=client) + no_default_logname = CloudLoggingExporter( + project_id=PROJECT_ID, client=client + ) + attrs = {str(i):"i" * 10000 for i in range(1000)} + log_record = LogRecord(resource=Resource({}), + attributes=attrs) no_default_logname.export( [ LogData( - log_record=LogRecord(resource=Resource({})), + log_record=log_record, instrumentation_scope=InstrumentationScope("test"), ) ] ) assert len(caplog.records) == 1 - assert "No log name provided" in caplog.text + assert "exceeds Cloud Logging's maximum limit of 256000.\n" in caplog.text def test_convert_otlp( @@ -62,13 +79,18 @@ def test_convert_otlp( severity_number=SeverityNumber(20), trace_id=25, span_id=22, - attributes={"gen_ai.system": "openai", "event.name": "gen_ai.system.message"}, + attributes={ + "gen_ai.system": "openai", + "event.name": "gen_ai.system.message", + }, body={ "kvlistValue": { "values": [ { "key": "content", - "value": {"stringValue": "You're a helpful assistant."}, + "value": { + "stringValue": "You're a helpful assistant." + }, } ] } @@ -79,7 +101,8 @@ def test_convert_otlp( log_data = [ LogData( - log_record=log_record, instrumentation_scope=InstrumentationScope("test") + log_record=log_record, + instrumentation_scope=InstrumentationScope("test"), ) ] cloudloggingfake.exporter.export(log_data) From 1a8a1b61fe4af49b8af1548d4eaa80ad37124e40 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Mon, 27 Jan 2025 17:27:53 +0000 Subject: [PATCH 03/13] Make sure the apache license and current year is on every file. Give log name a default value. Fix some types / formatting that running the tox commands surfaced. --- opentelemetry-exporter-gcp-logging/README.rst | 85 +++++++++---------- opentelemetry-exporter-gcp-logging/setup.cfg | 2 +- opentelemetry-exporter-gcp-logging/setup.py | 2 +- .../exporter/cloud_logging/__init__.py | 81 +++++++++++------- .../cloud_logging/environment_variables.py | 6 +- .../exporter/cloud_logging/version.py | 2 +- .../tests/conftest.py | 2 +- .../tests/fixtures/cloud_logging_fake.py | 23 +++-- .../tests/fixtures/snapshot_logging_calls.py | 8 +- .../tests/test_cloud_logging.py | 55 ++++++++---- 10 files changed, 160 insertions(+), 106 deletions(-) diff --git a/opentelemetry-exporter-gcp-logging/README.rst b/opentelemetry-exporter-gcp-logging/README.rst index 822ef153..0d6f7f50 100644 --- a/opentelemetry-exporter-gcp-logging/README.rst +++ b/opentelemetry-exporter-gcp-logging/README.rst @@ -1,15 +1,15 @@ -OpenTelemetry Google Cloud Monitoring Exporter +OpenTelemetry Google Cloud Logging Exporter ============================================== -.. image:: https://badge.fury.io/py/opentelemetry-exporter-gcp-monitoring.svg - :target: https://badge.fury.io/py/opentelemetry-exporter-gcp-monitoring +.. image:: https://badge.fury.io/py/opentelemetry-exporter-gcp-logging.svg + :target: https://badge.fury.io/py/opentelemetry-exporter-gcp-logging .. image:: https://readthedocs.org/projects/google-cloud-opentelemetry/badge/?version=latest :target: https://google-cloud-opentelemetry.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status -This library provides support for exporting metrics to Google Cloud -Monitoring. +This library provides support for exporting logs to Google Cloud +Logging. To get started with instrumentation in Google Cloud, see `Generate traces and metrics with Python `_. @@ -29,57 +29,54 @@ Installation .. code:: bash - pip install opentelemetry-exporter-gcp-monitoring + pip install opentelemetry-exporter-gcp-logging Usage ----- .. code:: python - import time - - from opentelemetry import metrics - from opentelemetry.exporter.cloud_monitoring import ( - CloudMonitoringMetricsExporter, + from opentelemetry.exporter.cloud_logging import ( + CloudLoggingExporter, ) - from opentelemetry.sdk.metrics import MeterProvider - from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk._logs._internal import LogRecord + from opentelemetry._logs.severity import SeverityNumber from opentelemetry.sdk.resources import Resource - - metrics.set_meter_provider( - MeterProvider( - metric_readers=[ - PeriodicExportingMetricReader( - CloudMonitoringMetricsExporter(), export_interval_millis=5000 - ) - ], - resource=Resource.create( - { - "service.name": "basic_metrics", - "service.namespace": "examples", - "service.instance.id": "instance123", - } - ), - ) - ) - meter = metrics.get_meter(__name__) - - # Creates metric workload.googleapis.com/request_counter with monitored resource generic_task - requests_counter = meter.create_counter( - name="request_counter", - description="number of requests", - unit="1", + from opentelemetry.sdk._logs import LogData + from opentelemetry.sdk.util.instrumentation import InstrumentationScope + + + exporter = CloudLoggingExporter(default_log_name='my_log') + exporter.export( + [ + LogData( + log_record=LogRecord( + resource=Resource({}), + timestamp=1736976310997977393, + severity_number=SeverityNumber(20), + attributes={ + "gen_ai.system": "openai", + "event.name": "gen_ai.system.message", + }, + body={ + "kvlistValue": { + "values": [ + { + "key": "content", + "value": {"stringValue": "You're a helpful assistant."}, + } + ] + } + }, + ), + instrumentation_scope=InstrumentationScope("test"), + ) + ] ) - staging_labels = {"environment": "staging"} - - for i in range(20): - requests_counter.add(25, staging_labels) - time.sleep(5) - References ---------- -* `Cloud Monitoring `_ +* `Cloud Logging `_ * `OpenTelemetry Project `_ diff --git a/opentelemetry-exporter-gcp-logging/setup.cfg b/opentelemetry-exporter-gcp-logging/setup.cfg index d14f5d78..7d8850bd 100644 --- a/opentelemetry-exporter-gcp-logging/setup.cfg +++ b/opentelemetry-exporter-gcp-logging/setup.cfg @@ -38,7 +38,7 @@ where = src test = [options.entry_points] -opentelemetry_metrics_exporter = +opentelemetry_logging_exporter = gcp_logging = opentelemetry.exporter.cloud_logging:CloudLoggingExporter opentelemetry_environment_variables = gcp_logging = opentelemetry.exporter.cloud_logging.environment_variables diff --git a/opentelemetry-exporter-gcp-logging/setup.py b/opentelemetry-exporter-gcp-logging/setup.py index c50ae35f..e57f525a 100644 --- a/opentelemetry-exporter-gcp-logging/setup.py +++ b/opentelemetry-exporter-gcp-logging/setup.py @@ -1,4 +1,4 @@ -# Copyright 2021 The OpenTelemetry Authors +# Copyright 2025 The OpenTelemetry Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py index ead4dce6..32abeea9 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py @@ -1,27 +1,41 @@ -from opentelemetry.sdk._logs.export import LogExporter -from opentelemetry.sdk._logs import LogData -import google.auth +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import datetime -from typing import Optional, Sequence import logging -from google.logging.type.log_severity_pb2 import LogSeverity # type: ignore -from google.cloud.logging_v2.services.logging_service_v2 import LoggingServiceV2Client +import urllib.parse +from typing import Optional, Sequence + +import google.auth +from google.api.monitored_resource_pb2 import MonitoredResource # type: ignore +from google.cloud.logging_v2.services.logging_service_v2 import ( + LoggingServiceV2Client, +) from google.cloud.logging_v2.services.logging_service_v2.transports.grpc import ( LoggingServiceV2GrpcTransport, ) from google.cloud.logging_v2.types.log_entry import LogEntry +from google.logging.type.log_severity_pb2 import LogSeverity # type: ignore +from google.protobuf.struct_pb2 import Struct +from google.protobuf.timestamp_pb2 import Timestamp +from opentelemetry.exporter.cloud_logging.version import __version__ from opentelemetry.resourcedetector.gcp_resource_detector._mapping import ( get_monitored_resource, ) -from google.api.monitored_resource_pb2 import ( - MonitoredResource, # type: ignore -) -from google.protobuf.timestamp_pb2 import Timestamp -import urllib.parse -from google.protobuf.struct_pb2 import Struct -from opentelemetry.exporter.cloud_logging.version import __version__ from opentelemetry.sdk import version as opentelemetry_sdk_version - +from opentelemetry.sdk._logs import LogData +from opentelemetry.sdk._logs.export import LogExporter +from opentelemetry.sdk.resources import Resource DEFAULT_MAX_ENTRY_SIZE = 256000 # 256 KB DEFAULT_MAX_REQUEST_SIZE = 10000000 # 10 MB @@ -45,7 +59,7 @@ # severityMapping maps the integer severity level values from OTel [0-24] # to matching Cloud Logging severity levels. -SEVERITY_MAPPING = { +SEVERITY_MAPPING: dict[int, int] = { 0: LogSeverity.DEFAULT, # Default, 0 1: LogSeverity.DEBUG, # 2: LogSeverity.DEBUG, # @@ -87,7 +101,10 @@ def __init__( self.project_id = str(default_project_id) else: self.project_id = project_id - self.default_log_name = default_log_name + if default_log_name: + self.default_log_name = default_log_name + else: + self.default_log_name = "otel_python_inprocess_log_name_temp" self.client = client or LoggingServiceV2Client( transport=LoggingServiceV2GrpcTransport( channel=LoggingServiceV2GrpcTransport.create_channel( @@ -108,14 +125,11 @@ def export(self, batch: Sequence[LogData]): log_name = self.default_log_name if attributes.get(LOG_NAME_ATTRIBUTE_KEY): log_name = str(attributes.get(LOG_NAME_ATTRIBUTE_KEY)) - if not log_name: - logging.warning( - "No log name provided, cannot write log to Cloud Logging. Set the 'default_log_name' option, or add the 'gcp.log_name' attribute to set a log name." - ) - continue - monitored_resource_data = get_monitored_resource(log_record.resource) + monitored_resource_data = get_monitored_resource( + log_record.resource or Resource({}) + ) # convert it to proto - monitored_resource = ( + monitored_resource: Optional[MonitoredResource] = ( MonitoredResource( type=monitored_resource_data.type, labels=monitored_resource_data.labels, @@ -138,13 +152,17 @@ def export(self, batch: Sequence[LogData]): log_entry = LogEntry() log_entry.timestamp = ts log_entry.log_name = log_name - log_entry.resource = monitored_resource + if monitored_resource: + log_entry.resource = monitored_resource attrs_map = {k: v for k, v in attributes.items()} log_entry.trace_sampled = ( - log_record.trace_flags is not None and log_record.trace_flags.sampled + log_record.trace_flags is not None + and log_record.trace_flags.sampled ) if TRACE_SAMPLED_ATTRIBUTE_KEY in attrs_map: - log_entry.trace_sampled |= bool(attrs_map[TRACE_SAMPLED_ATTRIBUTE_KEY]) + log_entry.trace_sampled |= bool( + attrs_map[TRACE_SAMPLED_ATTRIBUTE_KEY] + ) del attrs_map[TRACE_SAMPLED_ATTRIBUTE_KEY] if log_record.trace_id: log_entry.trace = "projects/{}/traces/{}".format( @@ -152,8 +170,13 @@ def export(self, batch: Sequence[LogData]): ) if log_record.span_id: log_entry.span_id = str(hex(log_record.span_id))[2:] - if log_record.severity_number in SEVERITY_MAPPING: - log_entry.severity = SEVERITY_MAPPING[log_record.severity_number] + if ( + log_record.severity_number + and log_record.severity_number in SEVERITY_MAPPING + ): + log_entry.severity = SEVERITY_MAPPING[ + log_record.severity_number + ] log_entry.labels = {k: str(v) for k, v in attrs_map.items()} if type(log_record.body) is dict: s = Struct() @@ -164,7 +187,7 @@ def export(self, batch: Sequence[LogData]): self._write_log_entries(log_entries) def _write_log_entries(self, log_entries: list[LogEntry]): - batch = [] + batch: list[LogEntry] = [] batch_byte_size = 0 for entry in log_entries: msg_size = LogEntry.pb(entry).ByteSize() diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/environment_variables.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/environment_variables.py index 48cf7283..02d8b75e 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/environment_variables.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/environment_variables.py @@ -1,4 +1,4 @@ -# Copyright 2023 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -OTEL_EXPORTER_GCP_LOGGING_PROJECT_ID = ( - "OTEL_EXPORTER_GCP_LOGGING_PROJECT_ID" -) +OTEL_EXPORTER_GCP_LOGGING_PROJECT_ID = "OTEL_EXPORTER_GCP_LOGGING_PROJECT_ID" """ .. envvar:: OTEL_EXPORTER_GCP_LOGGING_PROJECT_ID diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/version.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/version.py index 553aa709..5464e913 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/version.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/version.py @@ -1,4 +1,4 @@ -# Copyright 2021 The OpenTelemetry Authors +# Copyright 2025 The OpenTelemetry Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/opentelemetry-exporter-gcp-logging/tests/conftest.py b/opentelemetry-exporter-gcp-logging/tests/conftest.py index c8ab2ac5..a23c6dbc 100644 --- a/opentelemetry-exporter-gcp-logging/tests/conftest.py +++ b/opentelemetry-exporter-gcp-logging/tests/conftest.py @@ -1,4 +1,4 @@ -# Copyright 2022 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/opentelemetry-exporter-gcp-logging/tests/fixtures/cloud_logging_fake.py b/opentelemetry-exporter-gcp-logging/tests/fixtures/cloud_logging_fake.py index f23e28c5..7ecf64f9 100644 --- a/opentelemetry-exporter-gcp-logging/tests/fixtures/cloud_logging_fake.py +++ b/opentelemetry-exporter-gcp-logging/tests/fixtures/cloud_logging_fake.py @@ -1,15 +1,28 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from functools import partial from typing import Callable, Iterable, List, cast from unittest.mock import patch + +import grpc +import pytest from google.cloud.logging_v2.services.logging_service_v2.transports.grpc import ( LoggingServiceV2GrpcTransport, ) - from google.cloud.logging_v2.types.logging import WriteLogEntriesRequest -import grpc -import pytest # pylint: disable=no-name-in-module from google.protobuf.empty_pb2 import Empty @@ -20,9 +33,7 @@ method_handlers_generic_handler, unary_unary_rpc_method_handler, ) -from opentelemetry.exporter.cloud_logging import ( - CloudLoggingExporter, -) +from opentelemetry.exporter.cloud_logging import CloudLoggingExporter @dataclass diff --git a/opentelemetry-exporter-gcp-logging/tests/fixtures/snapshot_logging_calls.py b/opentelemetry-exporter-gcp-logging/tests/fixtures/snapshot_logging_calls.py index 7141b39a..eb6dd44d 100644 --- a/opentelemetry-exporter-gcp-logging/tests/fixtures/snapshot_logging_calls.py +++ b/opentelemetry-exporter-gcp-logging/tests/fixtures/snapshot_logging_calls.py @@ -1,4 +1,4 @@ -# Copyright 2022 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, cast, List +from typing import List, Optional, cast import pytest from fixtures.cloud_logging_fake import WriteLogEntriesCall @@ -39,7 +39,9 @@ def serialize( ) -> SerializedData: json = [ json_format.MessageToDict( - type(call.write_log_entries_request).pb(call.write_log_entries_request) + type(call.write_log_entries_request).pb( + call.write_log_entries_request + ) ) for call in cast(List[WriteLogEntriesCall], data) ] diff --git a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py index 1b394429..33dedddb 100644 --- a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py +++ b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py @@ -1,3 +1,16 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ Some tests in this file use [syrupy](https://github.com/tophat/syrupy) for snapshot testing aka golden testing. The Cloud Logging API calls are captured with a gRPC fake and compared to the existing @@ -13,20 +26,19 @@ Be sure to review the changes. """ -from google.cloud.logging_v2.services.logging_service_v2 import LoggingServiceV2Client +import re +from typing import List -from fixtures.cloud_logging_fake import CloudLoggingFake -from opentelemetry.sdk._logs._internal import LogRecord +from fixtures.cloud_logging_fake import CloudLoggingFake, WriteLogEntriesCall +from google.cloud.logging_v2.services.logging_service_v2 import ( + LoggingServiceV2Client, +) from opentelemetry._logs.severity import SeverityNumber -from opentelemetry.sdk.resources import Resource +from opentelemetry.exporter.cloud_logging import CloudLoggingExporter from opentelemetry.sdk._logs import LogData +from opentelemetry.sdk._logs._internal import LogRecord +from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.util.instrumentation import InstrumentationScope -from opentelemetry.exporter.cloud_logging import ( - CloudLoggingExporter, -) -from typing import List -from fixtures.cloud_logging_fake import WriteLogEntriesCall -import re PROJECT_ID = "fakeproject" @@ -39,17 +51,22 @@ def test_create_cloud_logging_exporter(caplog) -> None: def test_invalid_otlp_entries_raise_warnings(caplog) -> None: client = LoggingServiceV2Client() - no_default_logname = CloudLoggingExporter(project_id=PROJECT_ID, client=client) + no_default_logname = CloudLoggingExporter( + project_id=PROJECT_ID, client=client + ) + attrs = {str(i):"i" * 10000 for i in range(1000)} + log_record = LogRecord(resource=Resource({}), + attributes=attrs) no_default_logname.export( [ LogData( - log_record=LogRecord(resource=Resource({})), + log_record=log_record, instrumentation_scope=InstrumentationScope("test"), ) ] ) assert len(caplog.records) == 1 - assert "No log name provided" in caplog.text + assert "exceeds Cloud Logging's maximum limit of 256000.\n" in caplog.text def test_convert_otlp( @@ -62,13 +79,18 @@ def test_convert_otlp( severity_number=SeverityNumber(20), trace_id=25, span_id=22, - attributes={"gen_ai.system": "openai", "event.name": "gen_ai.system.message"}, + attributes={ + "gen_ai.system": "openai", + "event.name": "gen_ai.system.message", + }, body={ "kvlistValue": { "values": [ { "key": "content", - "value": {"stringValue": "You're a helpful assistant."}, + "value": { + "stringValue": "You're a helpful assistant." + }, } ] } @@ -79,7 +101,8 @@ def test_convert_otlp( log_data = [ LogData( - log_record=log_record, instrumentation_scope=InstrumentationScope("test") + log_record=log_record, + instrumentation_scope=InstrumentationScope("test"), ) ] cloudloggingfake.exporter.export(log_data) From 00be214b615f75d0ae385b4abbb28ce74758bc42 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Mon, 27 Jan 2025 21:51:16 +0000 Subject: [PATCH 04/13] Use anonymous creds in the test. Fix mypy type issues with LogSeverity. --- .../opentelemetry/exporter/cloud_logging/__init__.py | 10 ++++++---- .../test_cloud_logging/test_convert_otlp.json | 1 + .../tests/test_cloud_logging.py | 8 ++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py index 32abeea9..781a1925 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# Remove after this program no longer support Python 3.8.* +from __future__ import annotations import datetime import logging import urllib.parse @@ -59,7 +61,7 @@ # severityMapping maps the integer severity level values from OTel [0-24] # to matching Cloud Logging severity levels. -SEVERITY_MAPPING: dict[int, int] = { +SEVERITY_MAPPING = { 0: LogSeverity.DEFAULT, # Default, 0 1: LogSeverity.DEBUG, # 2: LogSeverity.DEBUG, # @@ -172,10 +174,10 @@ def export(self, batch: Sequence[LogData]): log_entry.span_id = str(hex(log_record.span_id))[2:] if ( log_record.severity_number - and log_record.severity_number in SEVERITY_MAPPING + and log_record.severity_number.value in SEVERITY_MAPPING ): - log_entry.severity = SEVERITY_MAPPING[ - log_record.severity_number + log_entry.severity = SEVERITY_MAPPING[ #type: ignore[assignment] + log_record.severity_number.value #type: ignore[index] ] log_entry.labels = {k: str(v) for k, v in attrs_map.items()} if type(log_record.body) is dict: diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp.json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp.json index ad6a6d68..883ab0c6 100644 --- a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp.json +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp.json @@ -27,6 +27,7 @@ }, "type": "generic_node" }, + "severity": "ERROR", "spanId": "16", "timestamp": "2025-01-15T21:25:10.997977393Z", "trace": "projects/fakeproject/traces/25" diff --git a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py index 33dedddb..0abe09d9 100644 --- a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py +++ b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py @@ -25,7 +25,6 @@ Be sure to review the changes. """ - import re from typing import List @@ -39,18 +38,19 @@ from opentelemetry.sdk._logs._internal import LogRecord from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.util.instrumentation import InstrumentationScope +from google.auth.credentials import AnonymousCredentials PROJECT_ID = "fakeproject" -def test_create_cloud_logging_exporter(caplog) -> None: +def test_create_cloud_logging_exporter() -> None: CloudLoggingExporter(default_log_name="test") - client = LoggingServiceV2Client() + client = LoggingServiceV2Client(credentials=AnonymousCredentials()) CloudLoggingExporter(project_id=PROJECT_ID, client=client) def test_invalid_otlp_entries_raise_warnings(caplog) -> None: - client = LoggingServiceV2Client() + client = LoggingServiceV2Client(credentials=AnonymousCredentials()) no_default_logname = CloudLoggingExporter( project_id=PROJECT_ID, client=client ) From 87d7a895cb59659b31a541ec4ed5803e866444d4 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Mon, 27 Jan 2025 22:09:36 +0000 Subject: [PATCH 05/13] Fix merge issues --- .../exporter/cloud_logging/__init__.py | 34 +------------------ .../tests/test_cloud_logging.py | 14 ++------ 2 files changed, 3 insertions(+), 45 deletions(-) diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py index 68c92888..5c582f92 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py @@ -1,17 +1,3 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Remove after this program no longer support Python 3.8.* from __future__ import annotations # Copyright 2025 Google LLC # @@ -29,16 +15,8 @@ import datetime import logging import urllib.parse -import logging -import urllib.parse from typing import Optional, Sequence -import google.auth -from google.api.monitored_resource_pb2 import MonitoredResource # type: ignore -from google.cloud.logging_v2.services.logging_service_v2 import ( - LoggingServiceV2Client, -) - import google.auth from google.api.monitored_resource_pb2 import MonitoredResource # type: ignore from google.cloud.logging_v2.services.logging_service_v2 import ( @@ -52,10 +30,6 @@ from google.protobuf.struct_pb2 import Struct from google.protobuf.timestamp_pb2 import Timestamp from opentelemetry.exporter.cloud_logging.version import __version__ -from google.logging.type.log_severity_pb2 import LogSeverity # type: ignore -from google.protobuf.struct_pb2 import Struct -from google.protobuf.timestamp_pb2 import Timestamp -from opentelemetry.exporter.cloud_logging.version import __version__ from opentelemetry.resourcedetector.gcp_resource_detector._mapping import ( get_monitored_resource, ) @@ -63,9 +37,6 @@ from opentelemetry.sdk._logs import LogData from opentelemetry.sdk._logs.export import LogExporter from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk._logs import LogData -from opentelemetry.sdk._logs.export import LogExporter -from opentelemetry.sdk.resources import Resource DEFAULT_MAX_ENTRY_SIZE = 256000 # 256 KB DEFAULT_MAX_REQUEST_SIZE = 10000000 # 10 MB @@ -166,7 +137,6 @@ def export(self, batch: Sequence[LogData]): log_record.resource or Resource({}) ) # convert it to proto - monitored_resource: Optional[MonitoredResource] = ( monitored_resource: Optional[MonitoredResource] = ( MonitoredResource( type=monitored_resource_data.type, @@ -198,8 +168,6 @@ def export(self, batch: Sequence[LogData]): log_entry.trace_sampled = ( log_record.trace_flags is not None and log_record.trace_flags.sampled - log_record.trace_flags is not None - and log_record.trace_flags.sampled ) if TRACE_SAMPLED_ATTRIBUTE_KEY in attrs_map: log_entry.trace_sampled |= bool( @@ -255,4 +223,4 @@ def _write_log_entries(self, log_entries: list[LogEntry]): self.client.write_log_entries(entries=batch) def shutdown(self): - pass + pass \ No newline at end of file diff --git a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py index 755d46a2..f68ffcf4 100644 --- a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py +++ b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py @@ -54,21 +54,17 @@ def test_invalid_otlp_entries_raise_warnings(caplog) -> None: no_default_logname = CloudLoggingExporter( project_id=PROJECT_ID, client=client ) - attrs = {str(i):"i" * 10000 for i in range(1000)} - log_record = LogRecord(resource=Resource({}), - attributes=attrs) no_default_logname.export( [ LogData( - log_record=log_record, - log_record=log_record, + log_record = LogRecord(resource=Resource({}), + attributes={str(i):"i" * 10000 for i in range(1000)}), instrumentation_scope=InstrumentationScope("test"), ) ] ) assert len(caplog.records) == 1 assert "exceeds Cloud Logging's maximum limit of 256000.\n" in caplog.text - assert "exceeds Cloud Logging's maximum limit of 256000.\n" in caplog.text def test_convert_otlp( @@ -85,10 +81,6 @@ def test_convert_otlp( "gen_ai.system": "openai", "event.name": "gen_ai.system.message", }, - attributes={ - "gen_ai.system": "openai", - "event.name": "gen_ai.system.message", - }, body={ "kvlistValue": { "values": [ @@ -112,8 +104,6 @@ def test_convert_otlp( LogData( log_record=log_record, instrumentation_scope=InstrumentationScope("test"), - log_record=log_record, - instrumentation_scope=InstrumentationScope("test"), ) ] cloudloggingfake.exporter.export(log_data) From 219e9f7585f3bc0dcfb8d71125f21e7a61adbf28 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Tue, 28 Jan 2025 18:10:45 +0000 Subject: [PATCH 06/13] Fix broken test, reformat file. --- .../exporter/cloud_logging/__init__.py | 8 ++++---- .../tests/test_cloud_logging.py | 17 +++++------------ 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py index 5c582f92..a7573632 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py @@ -1,4 +1,5 @@ from __future__ import annotations + # Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -187,8 +188,8 @@ def export(self, batch: Sequence[LogData]): log_record.severity_number and log_record.severity_number.value in SEVERITY_MAPPING ): - log_entry.severity = SEVERITY_MAPPING[ #type: ignore[assignment] - log_record.severity_number.value #type: ignore[index] + log_entry.severity = SEVERITY_MAPPING[ # type: ignore[assignment] + log_record.severity_number.value # type: ignore[index] ] log_entry.labels = {k: str(v) for k, v in attrs_map.items()} if type(log_record.body) is dict: @@ -200,7 +201,6 @@ def export(self, batch: Sequence[LogData]): self._write_log_entries(log_entries) def _write_log_entries(self, log_entries: list[LogEntry]): - batch: list[LogEntry] = [] batch: list[LogEntry] = [] batch_byte_size = 0 for entry in log_entries: @@ -223,4 +223,4 @@ def _write_log_entries(self, log_entries: list[LogEntry]): self.client.write_log_entries(entries=batch) def shutdown(self): - pass \ No newline at end of file + pass diff --git a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py index f68ffcf4..bc69e53e 100644 --- a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py +++ b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py @@ -29,6 +29,7 @@ from typing import List from fixtures.cloud_logging_fake import CloudLoggingFake, WriteLogEntriesCall +from google.auth.credentials import AnonymousCredentials from google.cloud.logging_v2.services.logging_service_v2 import ( LoggingServiceV2Client, ) @@ -38,17 +39,10 @@ from opentelemetry.sdk._logs._internal import LogRecord from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.util.instrumentation import InstrumentationScope -from google.auth.credentials import AnonymousCredentials PROJECT_ID = "fakeproject" -def test_create_cloud_logging_exporter() -> None: - CloudLoggingExporter(default_log_name="test") - client = LoggingServiceV2Client(credentials=AnonymousCredentials()) - CloudLoggingExporter(project_id=PROJECT_ID, client=client) - - def test_invalid_otlp_entries_raise_warnings(caplog) -> None: client = LoggingServiceV2Client(credentials=AnonymousCredentials()) no_default_logname = CloudLoggingExporter( @@ -57,8 +51,10 @@ def test_invalid_otlp_entries_raise_warnings(caplog) -> None: no_default_logname.export( [ LogData( - log_record = LogRecord(resource=Resource({}), - attributes={str(i):"i" * 10000 for i in range(1000)}), + log_record=LogRecord( + resource=Resource({}), + attributes={str(i): "i" * 10000 for i in range(1000)}, + ), instrumentation_scope=InstrumentationScope("test"), ) ] @@ -89,9 +85,6 @@ def test_convert_otlp( "value": { "stringValue": "You're a helpful assistant." }, - "value": { - "stringValue": "You're a helpful assistant." - }, } ] } From bc51c25be36dfdf8220c5080455777dd4008e5e3 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Wed, 29 Jan 2025 18:37:16 +0000 Subject: [PATCH 07/13] Address comments in PR. --- opentelemetry-exporter-gcp-logging/README.rst | 40 ++++--- .../exporter/cloud_logging/__init__.py | 109 ++++++++---------- .../test_cloud_logging/test_convert_otlp.json | 4 +- 3 files changed, 73 insertions(+), 80 deletions(-) diff --git a/opentelemetry-exporter-gcp-logging/README.rst b/opentelemetry-exporter-gcp-logging/README.rst index 0d6f7f50..3f446910 100644 --- a/opentelemetry-exporter-gcp-logging/README.rst +++ b/opentelemetry-exporter-gcp-logging/README.rst @@ -35,22 +35,39 @@ Usage ----- .. code:: python - + import logging from opentelemetry.exporter.cloud_logging import ( CloudLoggingExporter, ) from opentelemetry.sdk._logs._internal import LogRecord from opentelemetry._logs.severity import SeverityNumber from opentelemetry.sdk.resources import Resource - from opentelemetry.sdk._logs import LogData - from opentelemetry.sdk.util.instrumentation import InstrumentationScope + from opentelemetry._logs import set_logger_provider + from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + + logger_provider = LoggerProvider( + resource=Resource.create( + { + "service.name": "shoppingcart", + "service.instance.id": "instance-12", + } + ), + ) + set_logger_provider(logger_provider) + exporter = CloudLoggingExporter(default_log_name='my_log') + logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter)) + handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider) + # Attach OTLP handler to root logger + logging.getLogger().addHandler(handler) - exporter = CloudLoggingExporter(default_log_name='my_log') - exporter.export( - [ - LogData( - log_record=LogRecord( + # Create namespaced logger + # It is recommended to not use the root logger with OTLP handler + # so telemetry is collected only for the application + logger1 = logging.getLogger("myapp.area1") + + logger1.debug(LogRecord( resource=Resource({}), timestamp=1736976310997977393, severity_number=SeverityNumber(20), @@ -68,12 +85,7 @@ Usage ] } }, - ), - instrumentation_scope=InstrumentationScope("test"), - ) - ] - ) - + )) References ---------- diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py index a7573632..14138be0 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py @@ -1,5 +1,3 @@ -from __future__ import annotations - # Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + import datetime import logging import urllib.parse @@ -38,6 +38,7 @@ from opentelemetry.sdk._logs import LogData from opentelemetry.sdk._logs.export import LogExporter from opentelemetry.sdk.resources import Resource +from opentelemetry.trace import format_span_id, format_trace_id DEFAULT_MAX_ENTRY_SIZE = 256000 # 256 KB DEFAULT_MAX_REQUEST_SIZE = 10000000 # 10 MB @@ -62,31 +63,31 @@ # severityMapping maps the integer severity level values from OTel [0-24] # to matching Cloud Logging severity levels. SEVERITY_MAPPING: dict[int, int] = { - 0: LogSeverity.DEFAULT, # Default, 0 - 1: LogSeverity.DEBUG, # - 2: LogSeverity.DEBUG, # - 3: LogSeverity.DEBUG, # - 4: LogSeverity.DEBUG, # - 5: LogSeverity.DEBUG, # - 6: LogSeverity.DEBUG, # - 7: LogSeverity.DEBUG, # - 8: LogSeverity.DEBUG, # 1-8 -> Debug - 9: LogSeverity.INFO, # - 10: LogSeverity.INFO, # 9-10 -> Info - 11: LogSeverity.NOTICE, # - 12: LogSeverity.NOTICE, # 11-12 -> Notice - 13: LogSeverity.WARNING, # - 14: LogSeverity.WARNING, # - 15: LogSeverity.WARNING, # - 16: LogSeverity.WARNING, # 13-16 -> Warning - 17: LogSeverity.ERROR, # - 18: LogSeverity.ERROR, # - 19: LogSeverity.ERROR, # - 20: LogSeverity.ERROR, # 17-20 -> Error - 21: LogSeverity.CRITICAL, # - 22: LogSeverity.CRITICAL, # 21-22 -> Critical - 23: LogSeverity.ALERT, # 23 -> Alert - 24: LogSeverity.EMERGENCY, # 24 -> Emergency + 0: LogSeverity.DEFAULT, + 1: LogSeverity.DEBUG, + 2: LogSeverity.DEBUG, + 3: LogSeverity.DEBUG, + 4: LogSeverity.DEBUG, + 5: LogSeverity.DEBUG, + 6: LogSeverity.DEBUG, + 7: LogSeverity.DEBUG, + 8: LogSeverity.DEBUG, + 9: LogSeverity.INFO, + 10: LogSeverity.INFO, + 11: LogSeverity.NOTICE, + 12: LogSeverity.NOTICE, + 13: LogSeverity.WARNING, + 14: LogSeverity.WARNING, + 15: LogSeverity.WARNING, + 16: LogSeverity.WARNING, + 17: LogSeverity.ERROR, + 18: LogSeverity.ERROR, + 19: LogSeverity.ERROR, + 20: LogSeverity.ERROR, + 21: LogSeverity.CRITICAL, + 22: LogSeverity.CRITICAL, + 23: LogSeverity.ALERT, + 24: LogSeverity.EMERGENCY, } @@ -107,10 +108,6 @@ def __init__( self.default_log_name = default_log_name else: self.default_log_name = "otel_python_inprocess_log_name_temp" - if default_log_name: - self.default_log_name = default_log_name - else: - self.default_log_name = "otel_python_inprocess_log_name_temp" self.client = client or LoggingServiceV2Client( transport=LoggingServiceV2GrpcTransport( channel=LoggingServiceV2GrpcTransport.create_channel( @@ -118,21 +115,22 @@ def __init__( ) ) ) - self.service_resource_labels = True def export(self, batch: Sequence[LogData]): + now = datetime.datetime.now() log_entries = [] for log_data in batch: log_record = log_data.log_record attributes = log_record.attributes or {} - project_id = self.project_id - if attributes.get(PROJECT_ID_ATTRIBUTE_KEY): - project_id = str(attributes.get(PROJECT_ID_ATTRIBUTE_KEY)) - log_name = self.default_log_name - if attributes.get(LOG_NAME_ATTRIBUTE_KEY): - log_name = str(attributes.get(LOG_NAME_ATTRIBUTE_KEY)) - monitored_resource_data = get_monitored_resource( - log_record.resource or Resource({}) + project_id = str( + attributes.get(PROJECT_ID_ATTRIBUTE_KEY, self.project_id) + ) + log_suffix = urllib.parse.quote_plus( + str( + attributes.get( + LOG_NAME_ATTRIBUTE_KEY, self.default_log_name + ) + ) ) monitored_resource_data = get_monitored_resource( log_record.resource or Resource({}) @@ -146,44 +144,27 @@ def export(self, batch: Sequence[LogData]): if monitored_resource_data else None ) - # if timestamp is unset, fall back to observed_time_unix_nano as recommended - # (see https://github.com/open-telemetry/opentelemetry-proto/blob/4abbb78/opentelemetry/proto/logs/v1/logs.proto#L176-L179) + # If timestamp is unset fall back to observed_time_unix_nano as recommended, + # see https://github.com/open-telemetry/opentelemetry-proto/blob/4abbb78/opentelemetry/proto/logs/v1/logs.proto#L176-L179 ts = Timestamp() if log_record.timestamp or log_record.observed_timestamp: ts.FromNanoseconds( log_record.timestamp or log_record.observed_timestamp ) else: - ts.FromDatetime(datetime.datetime.now()) - log_name = "projects/{}/logs/{}".format( - project_id, urllib.parse.quote_plus(log_name) - ) + ts.FromDatetime(now) + log_name = f"projects/{project_id}/logs/{log_suffix}" log_entry = LogEntry() log_entry.timestamp = ts log_entry.log_name = log_name - if monitored_resource: - log_entry.resource = monitored_resource if monitored_resource: log_entry.resource = monitored_resource attrs_map = {k: v for k, v in attributes.items()} - log_entry.trace_sampled = ( - log_record.trace_flags is not None - and log_record.trace_flags.sampled - ) - if TRACE_SAMPLED_ATTRIBUTE_KEY in attrs_map: - log_entry.trace_sampled |= bool( - attrs_map[TRACE_SAMPLED_ATTRIBUTE_KEY] - ) - log_entry.trace_sampled |= bool( - attrs_map[TRACE_SAMPLED_ATTRIBUTE_KEY] - ) - del attrs_map[TRACE_SAMPLED_ATTRIBUTE_KEY] + log_entry.trace_sampled = bool(log_record.trace_flags) if log_record.trace_id: - log_entry.trace = "projects/{}/traces/{}".format( - project_id, log_record.trace_id - ) + log_entry.trace = f"projects/{project_id}/traces/{format_trace_id(log_record.trace_id)}" if log_record.span_id: - log_entry.span_id = str(hex(log_record.span_id))[2:] + log_entry.span_id = format_span_id(log_record.span_id) if ( log_record.severity_number and log_record.severity_number.value in SEVERITY_MAPPING diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp.json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp.json index 883ab0c6..a0788cd6 100644 --- a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp.json +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp.json @@ -28,9 +28,9 @@ "type": "generic_node" }, "severity": "ERROR", - "spanId": "16", + "spanId": "0000000000000016", "timestamp": "2025-01-15T21:25:10.997977393Z", - "trace": "projects/fakeproject/traces/25" + "trace": "projects/fakeproject/traces/00000000000000000000000000000019" } ] } From 91a2026678f2766c0fb8c196cf1d6505929ac3ea Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Wed, 29 Jan 2025 18:51:49 +0000 Subject: [PATCH 08/13] Convert .format string to f string --- .../src/opentelemetry/exporter/cloud_logging/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py index 14138be0..c491b9c8 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py @@ -188,9 +188,7 @@ def _write_log_entries(self, log_entries: list[LogEntry]): msg_size = LogEntry.pb(entry).ByteSize() if msg_size > DEFAULT_MAX_ENTRY_SIZE: logging.warning( - "Cannot write log that is {} bytes which exceeds Cloud Logging's maximum limit of {}.".format( - msg_size, DEFAULT_MAX_ENTRY_SIZE - ) + f"Cannot write log that is {msg_size} bytes which exceeds Cloud Logging's maximum limit of {DEFAULT_MAX_ENTRY_SIZE}." ) continue if msg_size + batch_byte_size > DEFAULT_MAX_REQUEST_SIZE: From 2363dd19da5d72bb06018af0d14d8eba6f58e8f0 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Thu, 30 Jan 2025 16:25:21 +0000 Subject: [PATCH 09/13] Fix example usage in README, make other minor changes based on feedback in PR. --- opentelemetry-exporter-gcp-logging/README.rst | 24 ++----------------- opentelemetry-exporter-gcp-logging/setup.cfg | 2 +- .../exporter/cloud_logging/__init__.py | 3 +-- 3 files changed, 4 insertions(+), 25 deletions(-) diff --git a/opentelemetry-exporter-gcp-logging/README.rst b/opentelemetry-exporter-gcp-logging/README.rst index 3f446910..13608838 100644 --- a/opentelemetry-exporter-gcp-logging/README.rst +++ b/opentelemetry-exporter-gcp-logging/README.rst @@ -39,8 +39,6 @@ Usage from opentelemetry.exporter.cloud_logging import ( CloudLoggingExporter, ) - from opentelemetry.sdk._logs._internal import LogRecord - from opentelemetry._logs.severity import SeverityNumber from opentelemetry.sdk.resources import Resource from opentelemetry._logs import set_logger_provider from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler @@ -57,7 +55,7 @@ Usage set_logger_provider(logger_provider) exporter = CloudLoggingExporter(default_log_name='my_log') logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter)) - handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider) + handler = LoggingHandler(level=logging.ERROR, logger_provider=logger_provider) # Attach OTLP handler to root logger logging.getLogger().addHandler(handler) @@ -67,25 +65,7 @@ Usage # so telemetry is collected only for the application logger1 = logging.getLogger("myapp.area1") - logger1.debug(LogRecord( - resource=Resource({}), - timestamp=1736976310997977393, - severity_number=SeverityNumber(20), - attributes={ - "gen_ai.system": "openai", - "event.name": "gen_ai.system.message", - }, - body={ - "kvlistValue": { - "values": [ - { - "key": "content", - "value": {"stringValue": "You're a helpful assistant."}, - } - ] - } - }, - )) + logger1.error({'structured_log_will_go_to_json_payload': 'value'}, extra={'this_will_go_to_LogEntry_labels_field': 'value'}) References ---------- diff --git a/opentelemetry-exporter-gcp-logging/setup.cfg b/opentelemetry-exporter-gcp-logging/setup.cfg index 7d8850bd..ff52b876 100644 --- a/opentelemetry-exporter-gcp-logging/setup.cfg +++ b/opentelemetry-exporter-gcp-logging/setup.cfg @@ -38,7 +38,7 @@ where = src test = [options.entry_points] -opentelemetry_logging_exporter = +opentelemetry_logs_exporter = gcp_logging = opentelemetry.exporter.cloud_logging:CloudLoggingExporter opentelemetry_environment_variables = gcp_logging = opentelemetry.exporter.cloud_logging.environment_variables diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py index c491b9c8..29d1114c 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py @@ -159,7 +159,6 @@ def export(self, batch: Sequence[LogData]): log_entry.log_name = log_name if monitored_resource: log_entry.resource = monitored_resource - attrs_map = {k: v for k, v in attributes.items()} log_entry.trace_sampled = bool(log_record.trace_flags) if log_record.trace_id: log_entry.trace = f"projects/{project_id}/traces/{format_trace_id(log_record.trace_id)}" @@ -172,7 +171,7 @@ def export(self, batch: Sequence[LogData]): log_entry.severity = SEVERITY_MAPPING[ # type: ignore[assignment] log_record.severity_number.value # type: ignore[index] ] - log_entry.labels = {k: str(v) for k, v in attrs_map.items()} + log_entry.labels = {k: str(v) for k, v in attributes.items()} if type(log_record.body) is dict: s = Struct() s.update(log_record.body) From 08a01de384e428a6b4be2a5e1026ebb144c8c3d5 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Thu, 30 Jan 2025 16:48:47 +0000 Subject: [PATCH 10/13] Support conversion of non-string LogRecord.attributes values and non-dictionary LogRecord.body. --- .../exporter/cloud_logging/__init__.py | 33 ++++- .../test_convert_non_json_dict_bytes.json | 18 +++ ....json => test_convert_otlp_dict_body.json} | 0 ...fferent_types_in_attrs_and_bytes_body.json | 35 +++++ ...convert_various_types_of_bodies[bool].json | 19 +++ ..._convert_various_types_of_bodies[str].json | 19 +++ .../tests/test_cloud_logging.py | 124 +++++++++++++----- 7 files changed, 216 insertions(+), 32 deletions(-) create mode 100644 opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes.json rename opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/{test_convert_otlp.json => test_convert_otlp_dict_body.json} (100%) create mode 100644 opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body.json create mode 100644 opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[bool].json create mode 100644 opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[str].json diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py index 29d1114c..4e9c680c 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py @@ -14,9 +14,10 @@ from __future__ import annotations import datetime +import json import logging import urllib.parse -from typing import Optional, Sequence +from typing import Any, Optional, Sequence import google.auth from google.api.monitored_resource_pb2 import MonitoredResource # type: ignore @@ -91,6 +92,16 @@ } +def convert_any_value_to_string(value: Any) -> str: + t = type(value) + if t is bool or t is int or t is float or t is str: + return str(value) + if t is list or t is tuple: + return json.dumps(value) + logging.warning(f"Unknown type {t} found, cannot convert to string.") + return "" + + class CloudLoggingExporter(LogExporter): def __init__( self, @@ -171,11 +182,29 @@ def export(self, batch: Sequence[LogData]): log_entry.severity = SEVERITY_MAPPING[ # type: ignore[assignment] log_record.severity_number.value # type: ignore[index] ] - log_entry.labels = {k: str(v) for k, v in attributes.items()} + log_entry.labels = { + k: convert_any_value_to_string(v) + for k, v in attributes.items() + } if type(log_record.body) is dict: s = Struct() s.update(log_record.body) log_entry.json_payload = s + elif type(log_record.body) is bytes: + json_str = log_record.body.decode("utf8") + json_dict = json.loads(json_str) + if type(json_dict) is dict: + s = Struct() + s.update(json_dict) + log_entry.json_payload = s + else: + logging.warning( + f"LogRecord.body was bytes type and json.loads turned body into type {type(json_dict)}, expected a dictionary." + ) + else: + log_entry.text_payload = convert_any_value_to_string( + log_record.body + ) log_entries.append(log_entry) self._write_log_entries(log_entries) diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes.json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes.json new file mode 100644 index 00000000..5e89d0f5 --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes.json @@ -0,0 +1,18 @@ +[ + { + "entries": [ + { + "logName": "projects/fakeproject/logs/test", + "resource": { + "labels": { + "location": "global", + "namespace": "", + "node_id": "" + }, + "type": "generic_node" + }, + "timestamp": "2025-01-15T21:25:10.997977393Z" + } + ] + } +] diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp.json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_dict_body.json similarity index 100% rename from opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp.json rename to opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_dict_body.json diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body.json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body.json new file mode 100644 index 00000000..d161211b --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body.json @@ -0,0 +1,35 @@ +[ + { + "entries": [ + { + "jsonPayload": { + "Classe": [ + "Email addresses", + "Passwords" + ], + "CreationDate": "2012-05-05", + "Date": "2016-05-21T21:35:40Z", + "Link": "http://some_link.com", + "LogoType": "png", + "Ref": 164611595.0 + }, + "labels": { + "boolArray": "[true, false, true, true]", + "float": "25.43231", + "int": "25", + "intArray": "[21, 18, 23, 17]" + }, + "logName": "projects/fakeproject/logs/test", + "resource": { + "labels": { + "location": "global", + "namespace": "", + "node_id": "" + }, + "type": "generic_node" + }, + "timestamp": "2025-01-15T21:25:10.997977393Z" + } + ] + } +] diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[bool].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[bool].json new file mode 100644 index 00000000..d29deae3 --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[bool].json @@ -0,0 +1,19 @@ +[ + { + "entries": [ + { + "logName": "projects/fakeproject/logs/test", + "resource": { + "labels": { + "location": "global", + "namespace": "", + "node_id": "" + }, + "type": "generic_node" + }, + "textPayload": "True", + "timestamp": "2025-01-15T21:25:10.997977393Z" + } + ] + } +] diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[str].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[str].json new file mode 100644 index 00000000..790b8a76 --- /dev/null +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[str].json @@ -0,0 +1,19 @@ +[ + { + "entries": [ + { + "logName": "projects/fakeproject/logs/test", + "resource": { + "labels": { + "location": "global", + "namespace": "", + "node_id": "" + }, + "type": "generic_node" + }, + "textPayload": "A text body", + "timestamp": "2025-01-15T21:25:10.997977393Z" + } + ] + } +] diff --git a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py index bc69e53e..fa556a1f 100644 --- a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py +++ b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py @@ -26,8 +26,9 @@ Be sure to review the changes. """ import re -from typing import List +from typing import List, Union +import pytest from fixtures.cloud_logging_fake import CloudLoggingFake, WriteLogEntriesCall from google.auth.credentials import AnonymousCredentials from google.cloud.logging_v2.services.logging_service_v2 import ( @@ -43,7 +44,7 @@ PROJECT_ID = "fakeproject" -def test_invalid_otlp_entries_raise_warnings(caplog) -> None: +def test_too_large_log_raises_warning(caplog) -> None: client = LoggingServiceV2Client(credentials=AnonymousCredentials()) no_default_logname = CloudLoggingExporter( project_id=PROJECT_ID, client=client @@ -52,6 +53,7 @@ def test_invalid_otlp_entries_raise_warnings(caplog) -> None: [ LogData( log_record=LogRecord( + body="abc", resource=Resource({}), attributes={str(i): "i" * 10000 for i in range(1000)}, ), @@ -63,39 +65,34 @@ def test_invalid_otlp_entries_raise_warnings(caplog) -> None: assert "exceeds Cloud Logging's maximum limit of 256000.\n" in caplog.text -def test_convert_otlp( +def test_convert_otlp_dict_body( cloudloggingfake: CloudLoggingFake, snapshot_writelogentrycalls: List[WriteLogEntriesCall], ) -> None: - # Create a new LogRecord object - log_record = LogRecord( - timestamp=1736976310997977393, - severity_number=SeverityNumber(20), - trace_id=25, - span_id=22, - attributes={ - "gen_ai.system": "openai", - "event.name": "gen_ai.system.message", - }, - body={ - "kvlistValue": { - "values": [ - { - "key": "content", - "value": { - "stringValue": "You're a helpful assistant." - }, - } - ] - } - }, - # Not sure why I'm getting AttributeError: 'NoneType' object has no attribute 'attributes' when unset. - resource=Resource({}), - ) - log_data = [ LogData( - log_record=log_record, + log_record=LogRecord( + timestamp=1736976310997977393, + severity_number=SeverityNumber(20), + trace_id=25, + span_id=22, + attributes={ + "gen_ai.system": "openai", + "event.name": "gen_ai.system.message", + }, + body={ + "kvlistValue": { + "values": [ + { + "key": "content", + "value": { + "stringValue": "You're a helpful assistant." + }, + } + ] + } + }, + ), instrumentation_scope=InstrumentationScope("test"), ) ] @@ -109,3 +106,70 @@ def test_convert_otlp( ) is not None ) + + +def test_convert_otlp_various_different_types_in_attrs_and_bytes_body( + cloudloggingfake: CloudLoggingFake, + snapshot_writelogentrycalls: List[WriteLogEntriesCall], +) -> None: + log_data = [ + LogData( + log_record=LogRecord( + timestamp=1736976310997977393, + attributes={ + "int": 25, + "float": 25.43231, + "intArray": [21, 18, 23, 17], + "boolArray": [True, False, True, True], + }, + body=b'{"Date": "2016-05-21T21:35:40Z", "CreationDate": "2012-05-05", "LogoType": "png", "Ref": 164611595, "Classe": ["Email addresses", "Passwords"],"Link":"http://some_link.com"}', + ), + instrumentation_scope=InstrumentationScope("test"), + ) + ] + cloudloggingfake.exporter.export(log_data) + assert cloudloggingfake.get_calls() == snapshot_writelogentrycalls + + +def test_convert_non_json_dict_bytes( + cloudloggingfake: CloudLoggingFake, + snapshot_writelogentrycalls: List[WriteLogEntriesCall], + caplog, +) -> None: + log_data = [ + LogData( + log_record=LogRecord( + timestamp=1736976310997977393, + body=b"123", + ), + instrumentation_scope=InstrumentationScope("test"), + ) + ] + cloudloggingfake.exporter.export(log_data) + assert cloudloggingfake.get_calls() == snapshot_writelogentrycalls + assert ( + "LogRecord.body was bytes type and json.loads turned body into type , expected a dictionary" + in caplog.text + ) + + +@pytest.mark.parametrize( + "body", + [pytest.param("A text body", id="str"), pytest.param(True, id="bool")], +) +def test_convert_various_types_of_bodies( + cloudloggingfake: CloudLoggingFake, + snapshot_writelogentrycalls: List[WriteLogEntriesCall], + body: Union[str, bool], +) -> None: + log_data = [ + LogData( + log_record=LogRecord( + timestamp=1736976310997977393, + body=body, + ), + instrumentation_scope=InstrumentationScope("test"), + ) + ] + cloudloggingfake.exporter.export(log_data) + assert cloudloggingfake.get_calls() == snapshot_writelogentrycalls From 4bfd7b1c690a01a7f9faa4ab4282783d31d29788 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Thu, 30 Jan 2025 22:04:01 +0000 Subject: [PATCH 11/13] Add tests, respond to comments on review. --- .../exporter/cloud_logging/__init__.py | 43 +++++++++++++++---- .../test_convert_non_json_dict_bytes.json | 3 +- .../test_convert_otlp_dict_body.json | 3 +- ...fferent_types_in_attrs_and_bytes_body.json | 3 +- ...convert_various_types_of_bodies[bool].json | 3 +- ..._convert_various_types_of_bodies[str].json | 3 +- .../tests/test_cloud_logging.py | 4 +- 7 files changed, 47 insertions(+), 15 deletions(-) diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py index 4e9c680c..829cd10a 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py @@ -17,7 +17,7 @@ import json import logging import urllib.parse -from typing import Any, Optional, Sequence +from typing import Any, Mapping, Optional, Sequence import google.auth from google.api.monitored_resource_pb2 import MonitoredResource # type: ignore @@ -28,6 +28,7 @@ LoggingServiceV2GrpcTransport, ) from google.cloud.logging_v2.types.log_entry import LogEntry +from google.cloud.logging_v2.types.logging import WriteLogEntriesRequest from google.logging.type.log_severity_pb2 import LogSeverity # type: ignore from google.protobuf.struct_pb2 import Struct from google.protobuf.timestamp_pb2 import Timestamp @@ -98,7 +99,7 @@ def convert_any_value_to_string(value: Any) -> str: return str(value) if t is list or t is tuple: return json.dumps(value) - logging.warning(f"Unknown type {t} found, cannot convert to string.") + logging.warning("Unknown type %s found, cannot convert to string.", t) return "" @@ -170,7 +171,10 @@ def export(self, batch: Sequence[LogData]): log_entry.log_name = log_name if monitored_resource: log_entry.resource = monitored_resource - log_entry.trace_sampled = bool(log_record.trace_flags) + log_entry.trace_sampled = ( + log_record.trace_flags is not None + and log_record.trace_flags.sampled + ) if log_record.trace_id: log_entry.trace = f"projects/{project_id}/traces/{format_trace_id(log_record.trace_id)}" if log_record.span_id: @@ -186,20 +190,21 @@ def export(self, batch: Sequence[LogData]): k: convert_any_value_to_string(v) for k, v in attributes.items() } - if type(log_record.body) is dict: + if isinstance(log_record.body, Mapping): s = Struct() s.update(log_record.body) log_entry.json_payload = s elif type(log_record.body) is bytes: json_str = log_record.body.decode("utf8") json_dict = json.loads(json_str) - if type(json_dict) is dict: + if isinstance(json_dict, Mapping): s = Struct() s.update(json_dict) log_entry.json_payload = s else: logging.warning( - f"LogRecord.body was bytes type and json.loads turned body into type {type(json_dict)}, expected a dictionary." + "LogRecord.body was bytes type and json.loads turned body into type %s, expected a dictionary.", + type(json_dict), ) else: log_entry.text_payload = convert_any_value_to_string( @@ -216,18 +221,38 @@ def _write_log_entries(self, log_entries: list[LogEntry]): msg_size = LogEntry.pb(entry).ByteSize() if msg_size > DEFAULT_MAX_ENTRY_SIZE: logging.warning( - f"Cannot write log that is {msg_size} bytes which exceeds Cloud Logging's maximum limit of {DEFAULT_MAX_ENTRY_SIZE}." + "Cannot write log that is %s bytes which exceeds Cloud Logging's maximum limit of %s bytes.", + msg_size, + DEFAULT_MAX_ENTRY_SIZE, ) continue if msg_size + batch_byte_size > DEFAULT_MAX_REQUEST_SIZE: - self.client.write_log_entries(entries=batch) + try: + self.client.write_log_entries( + WriteLogEntriesRequest( + entries=batch, partial_success=True + ) + ) + # pylint: disable=broad-except + except Exception as ex: + logging.error( + "Error while writing to Cloud Logging", exc_info=ex + ) batch = [entry] batch_byte_size = msg_size else: batch.append(entry) batch_byte_size += msg_size if batch: - self.client.write_log_entries(entries=batch) + try: + self.client.write_log_entries( + WriteLogEntriesRequest(entries=batch, partial_success=True) + ) + # pylint: disable=broad-except + except Exception as ex: + logging.error( + "Error while writing to Cloud Logging", exc_info=ex + ) def shutdown(self): pass diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes.json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes.json index 5e89d0f5..3a74737a 100644 --- a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes.json +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes.json @@ -13,6 +13,7 @@ }, "timestamp": "2025-01-15T21:25:10.997977393Z" } - ] + ], + "partialSuccess": true } ] diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_dict_body.json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_dict_body.json index a0788cd6..54368f8e 100644 --- a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_dict_body.json +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_dict_body.json @@ -32,6 +32,7 @@ "timestamp": "2025-01-15T21:25:10.997977393Z", "trace": "projects/fakeproject/traces/00000000000000000000000000000019" } - ] + ], + "partialSuccess": true } ] diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body.json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body.json index d161211b..bbf8dfcc 100644 --- a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body.json +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_various_different_types_in_attrs_and_bytes_body.json @@ -30,6 +30,7 @@ }, "timestamp": "2025-01-15T21:25:10.997977393Z" } - ] + ], + "partialSuccess": true } ] diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[bool].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[bool].json index d29deae3..4bdc6c1a 100644 --- a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[bool].json +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[bool].json @@ -14,6 +14,7 @@ "textPayload": "True", "timestamp": "2025-01-15T21:25:10.997977393Z" } - ] + ], + "partialSuccess": true } ] diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[str].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[str].json index 790b8a76..939d4784 100644 --- a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[str].json +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[str].json @@ -14,6 +14,7 @@ "textPayload": "A text body", "timestamp": "2025-01-15T21:25:10.997977393Z" } - ] + ], + "partialSuccess": true } ] diff --git a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py index fa556a1f..6a0f3bbd 100644 --- a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py +++ b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py @@ -62,7 +62,9 @@ def test_too_large_log_raises_warning(caplog) -> None: ] ) assert len(caplog.records) == 1 - assert "exceeds Cloud Logging's maximum limit of 256000.\n" in caplog.text + assert ( + "exceeds Cloud Logging's maximum limit of 256000 bytes" in caplog.text + ) def test_convert_otlp_dict_body( From ab18b30a91a6372e48ec54ea6da00f1dde7100fc Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Mon, 3 Feb 2025 15:22:05 +0000 Subject: [PATCH 12/13] Respond to comments. Handle case where bytes is set in LogRecord.body but is not a proto struct. --- opentelemetry-exporter-gcp-logging/README.rst | 2 +- .../exporter/cloud_logging/__init__.py | 98 ++++++++++--------- .../exporter/cloud_logging/version.py | 2 +- .../test_convert_non_json_dict_bytes.json | 1 + .../test_convert_otlp_dict_body.json | 3 +- ...convert_various_types_of_bodies[bool].json | 2 +- .../tests/test_cloud_logging.py | 8 +- 7 files changed, 61 insertions(+), 55 deletions(-) diff --git a/opentelemetry-exporter-gcp-logging/README.rst b/opentelemetry-exporter-gcp-logging/README.rst index 13608838..9fa6a4d9 100644 --- a/opentelemetry-exporter-gcp-logging/README.rst +++ b/opentelemetry-exporter-gcp-logging/README.rst @@ -65,7 +65,7 @@ Usage # so telemetry is collected only for the application logger1 = logging.getLogger("myapp.area1") - logger1.error({'structured_log_will_go_to_json_payload': 'value'}, extra={'this_will_go_to_LogEntry_labels_field': 'value'}) + logger1.warning("string log %s", "here") References ---------- diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py index 829cd10a..d627d3e4 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py @@ -13,6 +13,7 @@ # limitations under the License. from __future__ import annotations +import base64 import datetime import json import logging @@ -20,7 +21,9 @@ from typing import Any, Mapping, Optional, Sequence import google.auth -from google.api.monitored_resource_pb2 import MonitoredResource # type: ignore +from google.api.monitored_resource_pb2 import ( # pylint: disable = no-name-in-module + MonitoredResource, +) from google.cloud.logging_v2.services.logging_service_v2 import ( LoggingServiceV2Client, ) @@ -29,9 +32,15 @@ ) from google.cloud.logging_v2.types.log_entry import LogEntry from google.cloud.logging_v2.types.logging import WriteLogEntriesRequest -from google.logging.type.log_severity_pb2 import LogSeverity # type: ignore -from google.protobuf.struct_pb2 import Struct -from google.protobuf.timestamp_pb2 import Timestamp +from google.logging.type.log_severity_pb2 import ( # pylint: disable = no-name-in-module + LogSeverity, +) +from google.protobuf.struct_pb2 import ( # pylint: disable = no-name-in-module + Struct, +) +from google.protobuf.timestamp_pb2 import ( # pylint: disable = no-name-in-module + Timestamp, +) from opentelemetry.exporter.cloud_logging.version import __version__ from opentelemetry.resourcedetector.gcp_resource_detector._mapping import ( get_monitored_resource, @@ -94,12 +103,21 @@ def convert_any_value_to_string(value: Any) -> str: - t = type(value) - if t is bool or t is int or t is float or t is str: + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, bytes): + return base64.b64encode(value).decode() + if ( + isinstance(value, int) + or isinstance(value, float) + or isinstance(value, str) + ): return str(value) - if t is list or t is tuple: + if isinstance(value, list) or isinstance(value, tuple): return json.dumps(value) - logging.warning("Unknown type %s found, cannot convert to string.", t) + logging.warning( + "Unknown value %s found, cannot convert to string.", type(value) + ) return "" @@ -132,6 +150,7 @@ def export(self, batch: Sequence[LogData]): now = datetime.datetime.now() log_entries = [] for log_data in batch: + log_entry = LogEntry() log_record = log_data.log_record attributes = log_record.attributes or {} project_id = str( @@ -144,18 +163,7 @@ def export(self, batch: Sequence[LogData]): ) ) ) - monitored_resource_data = get_monitored_resource( - log_record.resource or Resource({}) - ) - # convert it to proto - monitored_resource: Optional[MonitoredResource] = ( - MonitoredResource( - type=monitored_resource_data.type, - labels=monitored_resource_data.labels, - ) - if monitored_resource_data - else None - ) + log_entry.log_name = f"projects/{project_id}/logs/{log_suffix}" # If timestamp is unset fall back to observed_time_unix_nano as recommended, # see https://github.com/open-telemetry/opentelemetry-proto/blob/4abbb78/opentelemetry/proto/logs/v1/logs.proto#L176-L179 ts = Timestamp() @@ -165,12 +173,15 @@ def export(self, batch: Sequence[LogData]): ) else: ts.FromDatetime(now) - log_name = f"projects/{project_id}/logs/{log_suffix}" - log_entry = LogEntry() log_entry.timestamp = ts - log_entry.log_name = log_name - if monitored_resource: - log_entry.resource = monitored_resource + monitored_resource_data = get_monitored_resource( + log_record.resource or Resource({}) + ) + if monitored_resource_data: + log_entry.resource = MonitoredResource( + type=monitored_resource_data.type, + labels=monitored_resource_data.labels, + ) log_entry.trace_sampled = ( log_record.trace_flags is not None and log_record.trace_flags.sampled @@ -190,30 +201,27 @@ def export(self, batch: Sequence[LogData]): k: convert_any_value_to_string(v) for k, v in attributes.items() } - if isinstance(log_record.body, Mapping): - s = Struct() - s.update(log_record.body) - log_entry.json_payload = s - elif type(log_record.body) is bytes: - json_str = log_record.body.decode("utf8") - json_dict = json.loads(json_str) - if isinstance(json_dict, Mapping): - s = Struct() - s.update(json_dict) - log_entry.json_payload = s - else: - logging.warning( - "LogRecord.body was bytes type and json.loads turned body into type %s, expected a dictionary.", - type(json_dict), - ) - else: - log_entry.text_payload = convert_any_value_to_string( - log_record.body - ) + self._set_payload_in_log_entry(log_entry, log_record.body) log_entries.append(log_entry) self._write_log_entries(log_entries) + def _set_payload_in_log_entry(self, log_entry: LogEntry, body: Any | None): + struct = Struct() + if isinstance(body, Mapping): + struct.update(body) + log_entry.json_payload = struct + elif isinstance(body, bytes): + json_str = body.decode("utf-8", errors="replace") + json_dict = json.loads(json_str) + if isinstance(json_dict, Mapping): + struct.update(json_dict) + log_entry.json_payload = struct + else: + log_entry.text_payload = base64.b64encode(body).decode() + else: + log_entry.text_payload = convert_any_value_to_string(body) + def _write_log_entries(self, log_entries: list[LogEntry]): batch: list[LogEntry] = [] batch_byte_size = 0 diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/version.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/version.py index 5464e913..bb01a21c 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/version.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.9b0" +__version__ = "1.9.0.dev0" diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes.json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes.json index 3a74737a..b658c503 100644 --- a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes.json +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_non_json_dict_bytes.json @@ -11,6 +11,7 @@ }, "type": "generic_node" }, + "textPayload": "MTIz", "timestamp": "2025-01-15T21:25:10.997977393Z" } ], diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_dict_body.json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_dict_body.json index 54368f8e..30a9ac22 100644 --- a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_dict_body.json +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_otlp_dict_body.json @@ -16,7 +16,8 @@ }, "labels": { "event.name": "gen_ai.system.message", - "gen_ai.system": "openai" + "gen_ai.system": "true", + "test": "23" }, "logName": "projects/fakeproject/logs/test", "resource": { diff --git a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[bool].json b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[bool].json index 4bdc6c1a..e0bc44ff 100644 --- a/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[bool].json +++ b/opentelemetry-exporter-gcp-logging/tests/__snapshots__/test_cloud_logging/test_convert_various_types_of_bodies[bool].json @@ -11,7 +11,7 @@ }, "type": "generic_node" }, - "textPayload": "True", + "textPayload": "true", "timestamp": "2025-01-15T21:25:10.997977393Z" } ], diff --git a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py index 6a0f3bbd..da1012e4 100644 --- a/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py +++ b/opentelemetry-exporter-gcp-logging/tests/test_cloud_logging.py @@ -79,7 +79,8 @@ def test_convert_otlp_dict_body( trace_id=25, span_id=22, attributes={ - "gen_ai.system": "openai", + "gen_ai.system": True, + "test": 23, "event.name": "gen_ai.system.message", }, body={ @@ -136,7 +137,6 @@ def test_convert_otlp_various_different_types_in_attrs_and_bytes_body( def test_convert_non_json_dict_bytes( cloudloggingfake: CloudLoggingFake, snapshot_writelogentrycalls: List[WriteLogEntriesCall], - caplog, ) -> None: log_data = [ LogData( @@ -149,10 +149,6 @@ def test_convert_non_json_dict_bytes( ] cloudloggingfake.exporter.export(log_data) assert cloudloggingfake.get_calls() == snapshot_writelogentrycalls - assert ( - "LogRecord.body was bytes type and json.loads turned body into type , expected a dictionary" - in caplog.text - ) @pytest.mark.parametrize( From 2e805745dc338dcb6ed8ee0d2b2b3c8f481b55f7 Mon Sep 17 00:00:00 2001 From: Dylan Russell Date: Mon, 3 Feb 2025 15:29:17 +0000 Subject: [PATCH 13/13] Fix lint checks. --- .../exporter/cloud_logging/__init__.py | 47 +++++++++---------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py index d627d3e4..d1094f7c 100644 --- a/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py +++ b/opentelemetry-exporter-gcp-logging/src/opentelemetry/exporter/cloud_logging/__init__.py @@ -102,18 +102,14 @@ } -def convert_any_value_to_string(value: Any) -> str: +def _convert_any_value_to_string(value: Any) -> str: if isinstance(value, bool): return "true" if value else "false" if isinstance(value, bytes): return base64.b64encode(value).decode() - if ( - isinstance(value, int) - or isinstance(value, float) - or isinstance(value, str) - ): + if isinstance(value, (int, float, str)): return str(value) - if isinstance(value, list) or isinstance(value, tuple): + if isinstance(value, (list, tuple)): return json.dumps(value) logging.warning( "Unknown value %s found, cannot convert to string.", type(value) @@ -121,6 +117,23 @@ def convert_any_value_to_string(value: Any) -> str: return "" +def _set_payload_in_log_entry(log_entry: LogEntry, body: Any | None): + struct = Struct() + if isinstance(body, Mapping): + struct.update(body) + log_entry.json_payload = struct + elif isinstance(body, bytes): + json_str = body.decode("utf-8", errors="replace") + json_dict = json.loads(json_str) + if isinstance(json_dict, Mapping): + struct.update(json_dict) + log_entry.json_payload = struct + else: + log_entry.text_payload = base64.b64encode(body).decode() + else: + log_entry.text_payload = _convert_any_value_to_string(body) + + class CloudLoggingExporter(LogExporter): def __init__( self, @@ -198,30 +211,14 @@ def export(self, batch: Sequence[LogData]): log_record.severity_number.value # type: ignore[index] ] log_entry.labels = { - k: convert_any_value_to_string(v) + k: _convert_any_value_to_string(v) for k, v in attributes.items() } - self._set_payload_in_log_entry(log_entry, log_record.body) + _set_payload_in_log_entry(log_entry, log_record.body) log_entries.append(log_entry) self._write_log_entries(log_entries) - def _set_payload_in_log_entry(self, log_entry: LogEntry, body: Any | None): - struct = Struct() - if isinstance(body, Mapping): - struct.update(body) - log_entry.json_payload = struct - elif isinstance(body, bytes): - json_str = body.decode("utf-8", errors="replace") - json_dict = json.loads(json_str) - if isinstance(json_dict, Mapping): - struct.update(json_dict) - log_entry.json_payload = struct - else: - log_entry.text_payload = base64.b64encode(body).decode() - else: - log_entry.text_payload = convert_any_value_to_string(body) - def _write_log_entries(self, log_entries: list[LogEntry]): batch: list[LogEntry] = [] batch_byte_size = 0