Skip to content

Commit b7584ad

Browse files
felixbarnyfelixbarny
committed
OTLP: add support for histograms
1 parent f7dd604 commit b7584ad

File tree

9 files changed

+652
-6
lines changed

9 files changed

+652
-6
lines changed

‎x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/datapoint/DataPoint.java‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@
88
package org.elasticsearch.xpack.oteldata.otlp.datapoint;
99

1010
import io.opentelemetry.proto.common.v1.KeyValue;
11+
import io.opentelemetry.proto.metrics.v1.AggregationTemporality;
12+
import io.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint;
13+
import io.opentelemetry.proto.metrics.v1.HistogramDataPoint;
1114
import io.opentelemetry.proto.metrics.v1.Metric;
1215
import io.opentelemetry.proto.metrics.v1.NumberDataPoint;
1316

@@ -134,4 +137,92 @@ public boolean isValid(Set<String> errors) {
134137
return true;
135138
}
136139
}
140+
141+
record ExponentialHistogram(ExponentialHistogramDataPoint dataPoint, Metric metric) implements DataPoint {
142+
143+
@Override
144+
public long getTimestampUnixNano() {
145+
return dataPoint.getTimeUnixNano();
146+
}
147+
148+
@Override
149+
public List<KeyValue> getAttributes() {
150+
return dataPoint.getAttributesList();
151+
}
152+
153+
@Override
154+
public long getStartTimestampUnixNano() {
155+
return dataPoint.getStartTimeUnixNano();
156+
}
157+
158+
@Override
159+
public String getUnit() {
160+
return metric.getUnit();
161+
}
162+
163+
@Override
164+
public String getMetricName() {
165+
return metric.getName();
166+
}
167+
168+
@Override
169+
public String getDynamicTemplate() {
170+
return "histogram";
171+
}
172+
173+
@Override
174+
public boolean isValid(Set<String> errors) {
175+
if (metric.getExponentialHistogram()
176+
.getAggregationTemporality() == AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA == false) {
177+
errors.add("cumulative exponential histogram metrics are not supported, ignoring " + metric.getName());
178+
return false;
179+
}
180+
return true;
181+
}
182+
}
183+
184+
record Histogram(HistogramDataPoint dataPoint, Metric metric) implements DataPoint {
185+
@Override
186+
public long getTimestampUnixNano() {
187+
return dataPoint.getTimeUnixNano();
188+
}
189+
190+
@Override
191+
public List<KeyValue> getAttributes() {
192+
return dataPoint.getAttributesList();
193+
}
194+
195+
@Override
196+
public long getStartTimestampUnixNano() {
197+
return dataPoint.getStartTimeUnixNano();
198+
}
199+
200+
@Override
201+
public String getUnit() {
202+
return metric.getUnit();
203+
}
204+
205+
@Override
206+
public String getMetricName() {
207+
return metric.getName();
208+
}
209+
210+
@Override
211+
public String getDynamicTemplate() {
212+
return "histogram";
213+
}
214+
215+
@Override
216+
public boolean isValid(Set<String> errors) {
217+
if (metric.getHistogram().getAggregationTemporality() == AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA == false) {
218+
errors.add("cumulative histogram metrics are not supported, ignoring " + metric.getName());
219+
return false;
220+
}
221+
if (dataPoint.getBucketCountsCount() == 1 && dataPoint.getExplicitBoundsCount() == 0) {
222+
errors.add("histogram with a single bucket and no explicit bounds is not supported, ignoring " + metric.getName());
223+
return false;
224+
}
225+
return true;
226+
}
227+
}
137228
}

‎x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/datapoint/DataPointGroupingContext.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
import org.elasticsearch.xpack.oteldata.otlp.tsid.ResourceTsidFunnel;
2727
import org.elasticsearch.xpack.oteldata.otlp.tsid.ScopeTsidFunnel;
2828

29-
import java.util.ArrayList;
29+
import java.util.Collection;
3030
import java.util.HashMap;
3131
import java.util.HashSet;
3232
import java.util.Iterator;
@@ -68,12 +68,14 @@ public void groupDataPoints(ExportMetricsServiceRequest exportMetricsServiceRequ
6868
scopeGroup.addDataPoints(metric, metric.getGauge().getDataPointsList(), DataPoint.Number::new);
6969
break;
7070
case EXPONENTIAL_HISTOGRAM:
71-
ignoredDataPoints += metric.getExponentialHistogram().getDataPointsCount();
72-
ignoredDataPointMessages.add("Exponential histogram is not supported yet. Dropping " + metric.getName());
71+
scopeGroup.addDataPoints(
72+
metric,
73+
metric.getExponentialHistogram().getDataPointsList(),
74+
DataPoint.ExponentialHistogram::new
75+
);
7376
break;
7477
case HISTOGRAM:
75-
ignoredDataPoints += metric.getHistogram().getDataPointsCount();
76-
ignoredDataPointMessages.add("Histogram is not supported yet. Dropping " + metric.getName());
78+
scopeGroup.addDataPoints(metric, metric.getHistogram().getDataPointsList(), DataPoint.Histogram::new);
7779
break;
7880
case SUMMARY:
7981
ignoredDataPoints += metric.getSummary().getDataPointsList().size();

‎x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/datapoint/HistogramConverter.java‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.oteldata.otlp.datapoint;
9+
10+
import io.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint;
11+
import io.opentelemetry.proto.metrics.v1.HistogramDataPoint;
12+
13+
import org.elasticsearch.exponentialhistogram.ExponentialScaleUtils;
14+
15+
/**
16+
* Utility class to convert OpenTelemetry histogram data points into counts and centroid values
17+
* so that we can use it with the {@code histogram} field type.
18+
* This class provides methods to extract counts and centroid values from both
19+
* {@link ExponentialHistogramDataPoint} and {@link HistogramDataPoint}.
20+
* The algorithm is ported over from the OpenTelemetry collector's Elasticsearch exporter.
21+
* @see <a href="https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.132.0/exporter/elasticsearchexporter">
22+
* Elasticsearch exporter on GitHub
23+
* </a>
24+
*/
25+
class HistogramConverter {
26+
27+
static <E extends Exception> void counts(ExponentialHistogramDataPoint dp, CheckedLongConsumer<E> counts) throws E {
28+
ExponentialHistogramDataPoint.Buckets negative = dp.getNegative();
29+
30+
for (int i = negative.getBucketCountsCount() - 1; i >= 0; i--) {
31+
long count = negative.getBucketCounts(i);
32+
if (count != 0) {
33+
counts.accept(count);
34+
}
35+
}
36+
37+
long zeroCount = dp.getZeroCount();
38+
if (zeroCount > 0) {
39+
counts.accept(zeroCount);
40+
}
41+
42+
ExponentialHistogramDataPoint.Buckets positive = dp.getPositive();
43+
for (int i = 0; i < positive.getBucketCountsCount(); i++) {
44+
long count = positive.getBucketCounts(i);
45+
if (count != 0) {
46+
counts.accept(count);
47+
}
48+
}
49+
}
50+
51+
/**
52+
* @see <a
53+
* href="https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.132.0/exporter/elasticsearchexporter/internal/exphistogram/exphistogram.go">
54+
* <code>ToTDigest</code> function
55+
* </a>
56+
*/
57+
static <E extends Exception> void centroidValues(ExponentialHistogramDataPoint dp, CheckedDoubleConsumer<E> values) throws E {
58+
int scale = dp.getScale();
59+
ExponentialHistogramDataPoint.Buckets negative = dp.getNegative();
60+
61+
int offset = negative.getOffset();
62+
for (int i = negative.getBucketCountsCount() - 1; i >= 0; i--) {
63+
long count = negative.getBucketCounts(i);
64+
if (count != 0) {
65+
double lb = -ExponentialScaleUtils.getUpperBucketBoundary(offset + i, scale);
66+
double ub = -ExponentialScaleUtils.getLowerBucketBoundary(offset + i, scale);
67+
values.accept(lb + (ub - lb) / 2);
68+
}
69+
}
70+
71+
long zeroCount = dp.getZeroCount();
72+
if (zeroCount > 0) {
73+
values.accept(0.0);
74+
}
75+
76+
ExponentialHistogramDataPoint.Buckets positive = dp.getPositive();
77+
offset = positive.getOffset();
78+
for (int i = 0; i < positive.getBucketCountsCount(); i++) {
79+
long count = positive.getBucketCounts(i);
80+
if (count != 0) {
81+
double lb = ExponentialScaleUtils.getLowerBucketBoundary(offset + i, scale);
82+
double ub = ExponentialScaleUtils.getUpperBucketBoundary(offset + i, scale);
83+
values.accept(lb + (ub - lb) / 2);
84+
}
85+
}
86+
}
87+
88+
static <E extends Exception> void counts(HistogramDataPoint dp, CheckedLongConsumer<E> counts) throws E {
89+
for (int i = 0; i < dp.getBucketCountsCount(); i++) {
90+
long count = dp.getBucketCounts(i);
91+
if (count != 0) {
92+
counts.accept(count);
93+
}
94+
}
95+
}
96+
97+
/**
98+
* @see <a
99+
* href="https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.132.0/exporter/elasticsearchexporter/internal/datapoints/histogram.go">
100+
* <code>histogramToValue</code> function
101+
* </a>
102+
*/
103+
static <E extends Exception> void centroidValues(HistogramDataPoint dp, CheckedDoubleConsumer<E> values) throws E {
104+
int size = dp.getBucketCountsCount();
105+
for (int i = 0; i < size; i++) {
106+
long count = dp.getBucketCounts(i);
107+
if (count != 0) {
108+
double value;
109+
if (i == 0) {
110+
// (-infinity, explicit_bounds[i]]
111+
value = dp.getExplicitBounds(i);
112+
if (value > 0) {
113+
value /= 2;
114+
}
115+
} else if (i == size - 1) {
116+
// (explicit_bounds[i], +infinity)
117+
value = dp.getExplicitBounds(i - 1);
118+
} else {
119+
// [explicit_bounds[i-1], explicit_bounds[i])
120+
// Use the midpoint between the boundaries.
121+
value = dp.getExplicitBounds(i - 1) + (dp.getExplicitBounds(i) - dp.getExplicitBounds(i - 1)) / 2.0;
122+
}
123+
values.accept(value);
124+
}
125+
}
126+
}
127+
128+
interface CheckedLongConsumer<E extends Exception> {
129+
void accept(long value) throws E;
130+
}
131+
132+
interface CheckedDoubleConsumer<E extends Exception> {
133+
void accept(double value) throws E;
134+
}
135+
}

‎x-pack/plugin/otel-data/src/test/java/org/elasticsearch/xpack/oteldata/otlp/OtlpUtils.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,11 @@
1313
import io.opentelemetry.proto.common.v1.KeyValue;
1414
import io.opentelemetry.proto.common.v1.KeyValueList;
1515
import io.opentelemetry.proto.metrics.v1.AggregationTemporality;
16+
import io.opentelemetry.proto.metrics.v1.ExponentialHistogram;
17+
import io.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint;
1618
import io.opentelemetry.proto.metrics.v1.Gauge;
19+
import io.opentelemetry.proto.metrics.v1.Histogram;
20+
import io.opentelemetry.proto.metrics.v1.HistogramDataPoint;
1721
import io.opentelemetry.proto.metrics.v1.Metric;
1822
import io.opentelemetry.proto.metrics.v1.NumberDataPoint;
1923
import io.opentelemetry.proto.metrics.v1.ResourceMetrics;
@@ -89,6 +93,36 @@ public static Metric createGaugeMetric(String name, String unit, List<NumberData
8993
return Metric.newBuilder().setName(name).setUnit(unit).setGauge(Gauge.newBuilder().addAllDataPoints(dataPoints).build()).build();
9094
}
9195

96+
public static Metric createExponentialHistogramMetric(
97+
String name,
98+
String unit,
99+
List<ExponentialHistogramDataPoint> dataPoints,
100+
AggregationTemporality temporality
101+
) {
102+
return Metric.newBuilder()
103+
.setName(name)
104+
.setUnit(unit)
105+
.setExponentialHistogram(
106+
ExponentialHistogram.newBuilder().setAggregationTemporality(temporality).addAllDataPoints(dataPoints).build()
107+
)
108+
.build();
109+
}
110+
111+
public static Metric createHistogramMetric(
112+
String name,
113+
String unit,
114+
List<HistogramDataPoint> dataPoints,
115+
AggregationTemporality temporality
116+
) {
117+
return Metric.newBuilder()
118+
.setName(name)
119+
.setUnit(unit)
120+
.setHistogram(
121+
Histogram.newBuilder().setAggregationTemporality(temporality).addAllDataPoints(dataPoints).build()
122+
)
123+
.build();
124+
}
125+
92126
public static Metric createSumMetric(
93127
String name,
94128
String unit,

‎x-pack/plugin/otel-data/src/test/java/org/elasticsearch/xpack/oteldata/otlp/datapoint/DataPointExponentialHistogramTests.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.oteldata.otlp.datapoint;
9+
10+
import io.opentelemetry.proto.metrics.v1.ExponentialHistogram;
11+
import io.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint;
12+
import io.opentelemetry.proto.metrics.v1.Metric;
13+
14+
import org.elasticsearch.test.ESTestCase;
15+
16+
import java.util.HashSet;
17+
18+
import static io.opentelemetry.proto.metrics.v1.AggregationTemporality.AGGREGATION_TEMPORALITY_CUMULATIVE;
19+
import static io.opentelemetry.proto.metrics.v1.AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA;
20+
import static org.hamcrest.Matchers.contains;
21+
import static org.hamcrest.Matchers.containsString;
22+
import static org.hamcrest.Matchers.empty;
23+
import static org.hamcrest.Matchers.equalTo;
24+
25+
public class DataPointExponentialHistogramTests extends ESTestCase {
26+
27+
private final HashSet<String> validationErrors = new HashSet<>();
28+
29+
public void testExponentialHistogram() {
30+
DataPoint.ExponentialHistogram doubleGauge = new DataPoint.ExponentialHistogram(
31+
ExponentialHistogramDataPoint.newBuilder().build(),
32+
Metric.newBuilder()
33+
.setExponentialHistogram(ExponentialHistogram.newBuilder().setAggregationTemporality(AGGREGATION_TEMPORALITY_DELTA).build())
34+
.build()
35+
);
36+
assertThat(doubleGauge.getDynamicTemplate(), equalTo("histogram"));
37+
assertThat(doubleGauge.isValid(validationErrors), equalTo(true));
38+
assertThat(validationErrors, empty());
39+
}
40+
41+
public void testExponentialHistogramUnsupportedTemporality() {
42+
DataPoint.ExponentialHistogram doubleGauge = new DataPoint.ExponentialHistogram(
43+
ExponentialHistogramDataPoint.newBuilder().build(),
44+
Metric.newBuilder()
45+
.setExponentialHistogram(
46+
ExponentialHistogram.newBuilder().setAggregationTemporality(AGGREGATION_TEMPORALITY_CUMULATIVE).build()
47+
)
48+
.build()
49+
);
50+
assertThat(doubleGauge.getDynamicTemplate(), equalTo("histogram"));
51+
assertThat(doubleGauge.isValid(validationErrors), equalTo(false));
52+
assertThat(validationErrors, contains(containsString("cumulative exponential histogram metrics are not supported")));
53+
}
54+
}

0 commit comments

Comments
 (0)