From fb5307c4d47c43d8fd6a016b0fef3aa6dbf9eeaa Mon Sep 17 00:00:00 2001 From: Adam Renberg Tamm Date: Mon, 23 Jun 2025 14:51:15 +0200 Subject: [PATCH 1/3] Add support for exponential histograms --- .../exporter/cloud_monitoring/__init__.py | 76 ++++-- .../test_exponential_histogram.json | 238 ++++++++++++++++++ .../tests/test_cloud_monitoring.py | 27 ++ 3 files changed, 325 insertions(+), 16 deletions(-) create mode 100644 opentelemetry-exporter-gcp-monitoring/tests/__snapshots__/test_cloud_monitoring/test_exponential_histogram.json diff --git a/opentelemetry-exporter-gcp-monitoring/src/opentelemetry/exporter/cloud_monitoring/__init__.py b/opentelemetry-exporter-gcp-monitoring/src/opentelemetry/exporter/cloud_monitoring/__init__.py index d3844644..ce789b44 100644 --- a/opentelemetry-exporter-gcp-monitoring/src/opentelemetry/exporter/cloud_monitoring/__init__.py +++ b/opentelemetry-exporter-gcp-monitoring/src/opentelemetry/exporter/cloud_monitoring/__init__.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +import math import random from dataclasses import replace from time import time_ns @@ -136,9 +137,7 @@ def __init__( self._metric_descriptors: Dict[str, MetricDescriptor] = {} self.unique_identifier = None if add_unique_identifier: - self.unique_identifier = "{:08x}".format( - random.randint(0, 16**8) - ) + self.unique_identifier = "{:08x}".format(random.randint(0, 16**8)) ( self._exporter_start_time_seconds, @@ -212,11 +211,7 @@ def _get_metric_descriptor( elif isinstance(data, Histogram): descriptor.metric_kind = MetricDescriptor.MetricKind.CUMULATIVE elif isinstance(data, ExponentialHistogram): - logger.warning( - "Unsupported metric data type %s, ignoring it", - type(data).__name__, - ) - return None + descriptor.metric_kind = MetricDescriptor.MetricKind.CUMULATIVE else: # Exhaustive check _: NoReturn = data @@ -235,6 +230,8 @@ def _get_metric_descriptor( ) elif isinstance(first_point, HistogramDataPoint): descriptor.value_type = MetricDescriptor.ValueType.DISTRIBUTION + elif isinstance(first_point, ExponentialHistogramDataPoint): + descriptor.value_type = MetricDescriptor.ValueType.DISTRIBUTION elif first_point is None: pass else: @@ -265,7 +262,9 @@ def _get_metric_descriptor( @staticmethod def _to_point( kind: "MetricDescriptor.MetricKind.V", - data_point: Union[NumberDataPoint, HistogramDataPoint], + data_point: Union[ + NumberDataPoint, HistogramDataPoint, ExponentialHistogramDataPoint + ], ) -> Point: if isinstance(data_point, HistogramDataPoint): mean = ( @@ -283,6 +282,55 @@ def _to_point( ), ) ) + elif isinstance(data_point, ExponentialHistogramDataPoint): + # Adapted from https://github.com/GoogleCloudPlatform/opentelemetry-operations-go/blob/v1.8.0/exporter/collector/metrics.go#L582 + mean = ( + data_point.sum / data_point.count if data_point.count else 0.0 + ) + + # Calculate underflow bucket (zero count + negative buckets) + underflow = data_point.zero_count + if data_point.negative.bucket_counts: + underflow += sum(data_point.negative.bucket_counts) + + # Create bucket counts array: [underflow, positive_buckets..., overflow=0] + bucket_counts = [underflow] + if data_point.positive.bucket_counts: + bucket_counts.extend(data_point.positive.bucket_counts) + bucket_counts.append(0) # overflow bucket is always empty + + # Determine bucket options + if not data_point.positive.bucket_counts: + # If no positive buckets, use explicit buckets with bounds=[0] + bucket_options = Distribution.BucketOptions( + explicit_buckets=Distribution.BucketOptions.Explicit( + bounds=[0.0], + ) + ) + else: + # Use exponential bucket options + # growth_factor = 2^(2^(-scale)) + growth_factor = math.pow(2, math.pow(2, -data_point.scale)) + # scale = growth_factor^(positive_bucket_offset) + scale = math.pow(growth_factor, data_point.positive.offset) + num_finite_buckets = len(bucket_counts) - 2 + + bucket_options = Distribution.BucketOptions( + exponential_buckets=Distribution.BucketOptions.Exponential( + num_finite_buckets=num_finite_buckets, + growth_factor=growth_factor, + scale=scale, + ) + ) + + point_value = TypedValue( + distribution_value=Distribution( + count=data_point.count, + mean=mean, + bucket_counts=bucket_counts, + bucket_options=bucket_options, + ) + ) else: if isinstance(data_point.value, int): point_value = TypedValue(int64_value=data_point.value) @@ -350,10 +398,6 @@ def export( continue for data_point in metric.data.data_points: - if isinstance( - data_point, ExponentialHistogramDataPoint - ): - continue labels = { _normalize_label_key(key): str(value) for key, value in ( @@ -361,9 +405,9 @@ def export( ).items() } if self.unique_identifier: - labels[ - UNIQUE_IDENTIFIER_KEY - ] = self.unique_identifier + labels[UNIQUE_IDENTIFIER_KEY] = ( + self.unique_identifier + ) point = self._to_point( descriptor.metric_kind, data_point ) diff --git a/opentelemetry-exporter-gcp-monitoring/tests/__snapshots__/test_cloud_monitoring/test_exponential_histogram.json b/opentelemetry-exporter-gcp-monitoring/tests/__snapshots__/test_cloud_monitoring/test_exponential_histogram.json new file mode 100644 index 00000000..a7ef2c5b --- /dev/null +++ b/opentelemetry-exporter-gcp-monitoring/tests/__snapshots__/test_cloud_monitoring/test_exponential_histogram.json @@ -0,0 +1,238 @@ +{ + "/google.monitoring.v3.MetricService/CreateMetricDescriptor": [ + { + "metricDescriptor": { + "description": "foo", + "displayName": "myexponentialhistogram", + "labels": [ + { + "key": "string" + }, + { + "key": "int" + }, + { + "key": "float" + } + ], + "metricKind": "CUMULATIVE", + "type": "workload.googleapis.com/myexponentialhistogram", + "unit": "{myunit}", + "valueType": "DISTRIBUTION" + }, + "name": "projects/fakeproject" + } + ], + "/google.monitoring.v3.MetricService/CreateTimeSeries": [ + { + "name": "projects/fakeproject", + "timeSeries": [ + { + "metric": { + "labels": { + "float": "123.4", + "int": "123", + "string": "string" + }, + "type": "workload.googleapis.com/myexponentialhistogram" + }, + "metricKind": "CUMULATIVE", + "points": [ + { + "interval": { + "endTime": "str", + "startTime": "str" + }, + "value": { + "distributionValue": { + "bucketCounts": [ + "0", + "1", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "1", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "1", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "1", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "1", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "1", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "1", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0", + "0" + ], + "bucketOptions": { + "exponentialBuckets": { + "growthFactor": 1.0218971486541166, + "numFiniteBuckets": 160, + "scale": 24.67537320652687 + } + }, + "count": "7", + "mean": 128.57142857142858 + } + } + } + ], + "resource": { + "labels": { + "location": "global", + "namespace": "", + "node_id": "" + }, + "type": "generic_node" + }, + "unit": "{myunit}" + } + ] + } + ] +} diff --git a/opentelemetry-exporter-gcp-monitoring/tests/test_cloud_monitoring.py b/opentelemetry-exporter-gcp-monitoring/tests/test_cloud_monitoring.py index 3fb09bb2..aeec456b 100644 --- a/opentelemetry-exporter-gcp-monitoring/tests/test_cloud_monitoring.py +++ b/opentelemetry-exporter-gcp-monitoring/tests/test_cloud_monitoring.py @@ -40,6 +40,7 @@ from opentelemetry.metrics import CallbackOptions, Observation from opentelemetry.sdk.metrics.view import ( ExplicitBucketHistogramAggregation, + ExponentialBucketHistogramAggregation, View, ) from opentelemetry.sdk.resources import Resource @@ -140,6 +141,32 @@ def test_histogram_single_bucket( assert gcmfake.get_calls() == snapshot_gcmcalls +def test_exponential_histogram( + gcmfake_meter_provider: GcmFakeMeterProvider, + gcmfake: GcmFake, + snapshot_gcmcalls, +) -> None: + meter_provider = gcmfake_meter_provider( + views=[ + View( + instrument_name="myexponentialhistogram", + aggregation=ExponentialBucketHistogramAggregation( + max_size=160, max_scale=20 + ), + ) + ] + ) + histogram = meter_provider.get_meter(__name__).create_histogram( + "myexponentialhistogram", description="foo", unit="{myunit}" + ) + + for value in [100, 50, 200, 25, 300, 75, 150]: + histogram.record(value, LABELS) + + meter_provider.force_flush() + assert gcmfake.get_calls() == snapshot_gcmcalls + + @pytest.mark.parametrize( "value", [pytest.param(123, id="int"), pytest.param(45.6, id="float")] ) From bd06767103655306991b048fa9ec452e12bb7745 Mon Sep 17 00:00:00 2001 From: Adam Renberg Tamm Date: Mon, 23 Jun 2025 16:15:35 +0200 Subject: [PATCH 2/3] Fix linting errors --- .../exporter/cloud_monitoring/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/opentelemetry-exporter-gcp-monitoring/src/opentelemetry/exporter/cloud_monitoring/__init__.py b/opentelemetry-exporter-gcp-monitoring/src/opentelemetry/exporter/cloud_monitoring/__init__.py index ce789b44..174795e3 100644 --- a/opentelemetry-exporter-gcp-monitoring/src/opentelemetry/exporter/cloud_monitoring/__init__.py +++ b/opentelemetry-exporter-gcp-monitoring/src/opentelemetry/exporter/cloud_monitoring/__init__.py @@ -137,7 +137,9 @@ def __init__( self._metric_descriptors: Dict[str, MetricDescriptor] = {} self.unique_identifier = None if add_unique_identifier: - self.unique_identifier = "{:08x}".format(random.randint(0, 16**8)) + self.unique_identifier = "{:08x}".format( + random.randint(0, 16**8) + ) ( self._exporter_start_time_seconds, @@ -405,9 +407,9 @@ def export( ).items() } if self.unique_identifier: - labels[UNIQUE_IDENTIFIER_KEY] = ( - self.unique_identifier - ) + labels[ + UNIQUE_IDENTIFIER_KEY + ] = self.unique_identifier point = self._to_point( descriptor.metric_kind, data_point ) From 7a46aefb2fcb29e5b5a0467ffcf488955da4df6c Mon Sep 17 00:00:00 2001 From: Adam Renberg Tamm Date: Thu, 10 Jul 2025 23:43:12 +0200 Subject: [PATCH 3/3] Move mean calculation to where it's used --- .../src/opentelemetry/exporter/cloud_monitoring/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opentelemetry-exporter-gcp-monitoring/src/opentelemetry/exporter/cloud_monitoring/__init__.py b/opentelemetry-exporter-gcp-monitoring/src/opentelemetry/exporter/cloud_monitoring/__init__.py index 174795e3..c5dff791 100644 --- a/opentelemetry-exporter-gcp-monitoring/src/opentelemetry/exporter/cloud_monitoring/__init__.py +++ b/opentelemetry-exporter-gcp-monitoring/src/opentelemetry/exporter/cloud_monitoring/__init__.py @@ -286,9 +286,6 @@ def _to_point( ) elif isinstance(data_point, ExponentialHistogramDataPoint): # Adapted from https://github.com/GoogleCloudPlatform/opentelemetry-operations-go/blob/v1.8.0/exporter/collector/metrics.go#L582 - mean = ( - data_point.sum / data_point.count if data_point.count else 0.0 - ) # Calculate underflow bucket (zero count + negative buckets) underflow = data_point.zero_count @@ -325,6 +322,9 @@ def _to_point( ) ) + mean = ( + data_point.sum / data_point.count if data_point.count else 0.0 + ) point_value = TypedValue( distribution_value=Distribution( count=data_point.count,