Skip to content

Commit 5449919

Browse files
felixbarnyfelixbarny
authored
OTLP: optimize _tsid creation (elastic#134982)
1 parent c543631 commit 5449919

File tree

9 files changed

+279
-302
lines changed

9 files changed

+279
-302
lines changed

‎server/src/main/java/org/elasticsearch/cluster/routing/TsidBuilder.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,6 @@ public TsidBuilder addAll(TsidBuilder other) {
207207
* @throws IllegalArgumentException if no dimensions have been added
208208
*/
209209
public MurmurHash3.Hash128 hash() {
210-
throwIfEmpty();
211210
Collections.sort(dimensions);
212211
murmur3Hasher.reset();
213212
for (Dimension dim : dimensions) {

‎server/src/test/java/org/elasticsearch/cluster/routing/TsidBuilderTests.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
package org.elasticsearch.cluster.routing;
1111

1212
import org.apache.lucene.util.BytesRef;
13+
import org.elasticsearch.common.hash.MurmurHash3;
1314
import org.elasticsearch.test.ESTestCase;
1415
import org.elasticsearch.xcontent.Text;
1516

@@ -115,11 +116,9 @@ public void testAddAllWithNullOrEmpty() {
115116
}
116117

117118
public void testExceptionWhenNoDimensions() {
118-
// Test that exception is thrown when no dimensions are added
119119
TsidBuilder builder = TsidBuilder.newBuilder();
120120

121-
IllegalArgumentException hashException = expectThrows(IllegalArgumentException.class, builder::hash);
122-
assertTrue(hashException.getMessage().contains("Dimensions are empty"));
121+
assertThat(builder.hash(), equalTo(new MurmurHash3.Hash128()));
123122

124123
IllegalArgumentException tsidException = expectThrows(IllegalArgumentException.class, builder::buildTsid);
125124
assertTrue(tsidException.getMessage().contains("Dimensions are empty"));

‎x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportAction.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
import org.apache.logging.log4j.LogManager;
1717
import org.apache.logging.log4j.Logger;
18+
import org.apache.lucene.util.BytesRef;
1819
import org.elasticsearch.ExceptionsHelper;
1920
import org.elasticsearch.action.ActionListener;
2021
import org.elasticsearch.action.ActionRequest;
@@ -35,6 +36,7 @@
3536
import org.elasticsearch.common.io.stream.BytesStreamOutput;
3637
import org.elasticsearch.common.io.stream.StreamInput;
3738
import org.elasticsearch.common.io.stream.StreamOutput;
39+
import org.elasticsearch.common.util.Maps;
3840
import org.elasticsearch.injection.guice.Inject;
3941
import org.elasticsearch.rest.RestStatus;
4042
import org.elasticsearch.tasks.Task;
@@ -47,6 +49,7 @@
4749
import org.elasticsearch.xpack.oteldata.otlp.proto.BufferedByteStringAccessor;
4850

4951
import java.io.IOException;
52+
import java.util.Map;
5053

5154
/**
5255
* Transport action for handling OpenTelemetry Protocol (OTLP) Metrics requests.
@@ -126,11 +129,13 @@ private void addIndexRequest(
126129
DataPointGroupingContext.DataPointGroup dataPointGroup
127130
) throws IOException {
128131
try (XContentBuilder xContentBuilder = XContentFactory.cborBuilder(new BytesStreamOutput())) {
129-
var dynamicTemplates = metricDocumentBuilder.buildMetricDocument(xContentBuilder, dataPointGroup);
132+
Map<String, String> dynamicTemplates = Maps.newHashMapWithExpectedSize(dataPointGroup.dataPoints().size());
133+
BytesRef tsid = metricDocumentBuilder.buildMetricDocument(xContentBuilder, dynamicTemplates, dataPointGroup);
130134
bulkRequestBuilder.add(
131135
new IndexRequest(dataPointGroup.targetIndex().index()).opType(DocWriteRequest.OpType.CREATE)
132136
.setRequireDataStream(true)
133137
.source(xContentBuilder)
138+
.tsid(tsid)
134139
.setDynamicTemplates(dynamicTemplates)
135140
);
136141
}

‎x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/datapoint/DataPointGroupingContext.java‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ private ResourceGroup getOrCreateResourceGroup(ResourceMetrics resourceMetrics)
128128
Hash128 resourceHash = resourceTsidBuilder.hash();
129129
ResourceGroup resourceGroup = resourceGroups.get(resourceHash);
130130
if (resourceGroup == null) {
131-
resourceGroup = new ResourceGroup(resourceMetrics.getResource(), resourceMetrics.getSchemaUrlBytes());
131+
resourceGroup = new ResourceGroup(resourceMetrics.getResource(), resourceMetrics.getSchemaUrlBytes(), resourceTsidBuilder);
132132
resourceGroups.put(resourceHash, resourceGroup);
133133
}
134134
return resourceGroup;
@@ -137,20 +137,23 @@ private ResourceGroup getOrCreateResourceGroup(ResourceMetrics resourceMetrics)
137137
class ResourceGroup {
138138
private final Resource resource;
139139
private final ByteString resourceSchemaUrl;
140+
private final TsidBuilder resourceTsidBuilder;
140141
private final Map<Hash128, ScopeGroup> scopes;
141142

142-
ResourceGroup(Resource resource, ByteString resourceSchemaUrl) {
143+
ResourceGroup(Resource resource, ByteString resourceSchemaUrl, TsidBuilder resourceTsidBuilder) {
143144
this.resource = resource;
144145
this.resourceSchemaUrl = resourceSchemaUrl;
146+
this.resourceTsidBuilder = resourceTsidBuilder;
145147
this.scopes = new HashMap<>();
146148
}
147149

148150
public ScopeGroup getOrCreateScope(ScopeMetrics scopeMetrics) {
149151
TsidBuilder scopeTsidBuilder = ScopeTsidFunnel.forScope(byteStringAccessor, scopeMetrics);
150152
Hash128 scopeHash = scopeTsidBuilder.hash();
153+
scopeTsidBuilder.addAll(resourceTsidBuilder);
151154
ScopeGroup scopeGroup = scopes.get(scopeHash);
152155
if (scopeGroup == null) {
153-
scopeGroup = new ScopeGroup(this, scopeMetrics.getScope(), scopeMetrics.getSchemaUrlBytes());
156+
scopeGroup = new ScopeGroup(this, scopeMetrics.getScope(), scopeMetrics.getSchemaUrlBytes(), scopeTsidBuilder);
154157
scopes.put(scopeHash, scopeGroup);
155158
}
156159
return scopeGroup;
@@ -169,15 +172,17 @@ class ScopeGroup {
169172
private final ResourceGroup resourceGroup;
170173
private final InstrumentationScope scope;
171174
private final ByteString scopeSchemaUrl;
175+
private final TsidBuilder scopeTsidBuilder;
172176
@Nullable
173177
private final String receiverName;
174178
// index -> timestamp -> dataPointGroupHash -> DataPointGroup
175179
private final Map<TargetIndex, Map<Hash128, Map<Hash128, DataPointGroup>>> dataPointGroupsByIndexAndTimestamp;
176180

177-
ScopeGroup(ResourceGroup resourceGroup, InstrumentationScope scope, ByteString scopeSchemaUrl) {
181+
ScopeGroup(ResourceGroup resourceGroup, InstrumentationScope scope, ByteString scopeSchemaUrl, TsidBuilder scopeTsidBuilder) {
178182
this.resourceGroup = resourceGroup;
179183
this.scope = scope;
180184
this.scopeSchemaUrl = scopeSchemaUrl;
185+
this.scopeTsidBuilder = scopeTsidBuilder;
181186
this.dataPointGroupsByIndexAndTimestamp = new HashMap<>();
182187
this.receiverName = extractReceiverName(scope);
183188
}
@@ -216,8 +221,13 @@ public void addDataPoint(DataPoint dataPoint) {
216221
}
217222

218223
private DataPointGroup getOrCreateDataPointGroup(DataPoint dataPoint) {
219-
TsidBuilder dataPointGroupTsidBuilder = DataPointTsidFunnel.forDataPoint(byteStringAccessor, dataPoint);
224+
TsidBuilder dataPointGroupTsidBuilder = DataPointTsidFunnel.forDataPoint(
225+
byteStringAccessor,
226+
dataPoint,
227+
scopeTsidBuilder.size()
228+
);
220229
Hash128 dataPointGroupHash = dataPointGroupTsidBuilder.hash();
230+
dataPointGroupTsidBuilder.addAll(scopeTsidBuilder);
221231
// in addition to the fields that go into the _tsid, we also need to group by timestamp and start timestamp
222232
Hash128 timestamp = new Hash128(dataPoint.getTimestampUnixNano(), dataPoint.getStartTimestampUnixNano());
223233
TargetIndex targetIndex = TargetIndex.evaluate(
@@ -236,6 +246,7 @@ private DataPointGroup getOrCreateDataPointGroup(DataPoint dataPoint) {
236246
resourceGroup.resourceSchemaUrl,
237247
scope,
238248
scopeSchemaUrl,
249+
dataPointGroupTsidBuilder,
239250
dataPoint.getAttributes(),
240251
dataPoint.getUnit(),
241252
targetIndex
@@ -261,6 +272,7 @@ public static final class DataPointGroup {
261272
private final ByteString resourceSchemaUrl;
262273
private final InstrumentationScope scope;
263274
private final ByteString scopeSchemaUrl;
275+
private final TsidBuilder tsidBuilder;
264276
private final List<KeyValue> dataPointAttributes;
265277
private final String unit;
266278
private final Set<String> metricNames = new HashSet<>();
@@ -273,6 +285,7 @@ public DataPointGroup(
273285
ByteString resourceSchemaUrl,
274286
InstrumentationScope scope,
275287
ByteString scopeSchemaUrl,
288+
TsidBuilder tsidBuilder,
276289
List<KeyValue> dataPointAttributes,
277290
String unit,
278291
TargetIndex targetIndex
@@ -281,6 +294,7 @@ public DataPointGroup(
281294
this.resourceSchemaUrl = resourceSchemaUrl;
282295
this.scope = scope;
283296
this.scopeSchemaUrl = scopeSchemaUrl;
297+
this.tsidBuilder = tsidBuilder;
284298
this.dataPointAttributes = dataPointAttributes;
285299
this.unit = unit;
286300
this.targetIndex = targetIndex;
@@ -334,6 +348,10 @@ public ByteString scopeSchemaUrl() {
334348
return scopeSchemaUrl;
335349
}
336350

351+
public TsidBuilder tsidBuilder() {
352+
return tsidBuilder;
353+
}
354+
337355
public List<KeyValue> dataPointAttributes() {
338356
return dataPointAttributes;
339357
}

‎x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/docbuilder/MetricDocumentBuilder.java‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414

1515
import com.google.protobuf.ByteString;
1616

17+
import org.apache.lucene.util.BytesRef;
18+
import org.elasticsearch.cluster.routing.TsidBuilder;
1719
import org.elasticsearch.common.Strings;
1820
import org.elasticsearch.common.hash.BufferedMurmur3Hasher;
1921
import org.elasticsearch.xcontent.XContentBuilder;
@@ -23,8 +25,8 @@
2325
import org.elasticsearch.xpack.oteldata.otlp.proto.BufferedByteStringAccessor;
2426

2527
import java.io.IOException;
26-
import java.util.HashMap;
2728
import java.util.List;
29+
import java.util.Map;
2830
import java.util.concurrent.TimeUnit;
2931

3032
/**
@@ -40,9 +42,11 @@ public MetricDocumentBuilder(BufferedByteStringAccessor byteStringAccessor) {
4042
this.byteStringAccessor = byteStringAccessor;
4143
}
4244

43-
public HashMap<String, String> buildMetricDocument(XContentBuilder builder, DataPointGroupingContext.DataPointGroup dataPointGroup)
44-
throws IOException {
45-
HashMap<String, String> dynamicTemplates = new HashMap<>();
45+
public BytesRef buildMetricDocument(
46+
XContentBuilder builder,
47+
Map<String, String> dynamicTemplates,
48+
DataPointGroupingContext.DataPointGroup dataPointGroup
49+
) throws IOException {
4650
List<DataPoint> dataPoints = dataPointGroup.dataPoints();
4751
builder.startObject();
4852
builder.field("@timestamp", TimeUnit.NANOSECONDS.toMillis(dataPointGroup.getTimestampUnixNano()));
@@ -53,7 +57,8 @@ public HashMap<String, String> buildMetricDocument(XContentBuilder builder, Data
5357
buildDataStream(builder, dataPointGroup.targetIndex());
5458
buildScope(builder, dataPointGroup.scopeSchemaUrl(), dataPointGroup.scope());
5559
buildDataPointAttributes(builder, dataPointGroup.dataPointAttributes(), dataPointGroup.unit());
56-
builder.field("_metric_names_hash", dataPointGroup.getMetricNamesHash(hasher));
60+
String metricNamesHash = dataPointGroup.getMetricNamesHash(hasher);
61+
builder.field("_metric_names_hash", metricNamesHash);
5762

5863
long docCount = 0;
5964
builder.startObject("metrics");
@@ -75,7 +80,9 @@ public HashMap<String, String> buildMetricDocument(XContentBuilder builder, Data
7580
builder.field("_doc_count", docCount);
7681
}
7782
builder.endObject();
78-
return dynamicTemplates;
83+
TsidBuilder tsidBuilder = dataPointGroup.tsidBuilder();
84+
tsidBuilder.addStringDimension("_metric_names_hash", metricNamesHash);
85+
return tsidBuilder.buildTsid();
7986
}
8087

8188
private void buildResource(Resource resource, ByteString schemaUrl, XContentBuilder builder) throws IOException {

‎x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/tsid/DataPointTsidFunnel.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,17 @@
1414

1515
public class DataPointTsidFunnel implements TsidFunnel<DataPoint> {
1616

17+
// for "unit" and "_metric_names_hash" that will be added in
18+
// MetricDocumentBuilder once the data point group is complete
19+
private static final int EXTRA_DIMENSIONS_SIZE = 2;
1720
private final BufferedByteStringAccessor byteStringAccessor;
1821

1922
private DataPointTsidFunnel(BufferedByteStringAccessor byteStringAccessor) {
2023
this.byteStringAccessor = byteStringAccessor;
2124
}
2225

23-
public static TsidBuilder forDataPoint(BufferedByteStringAccessor byteStringAccessor, DataPoint dataPoint) {
24-
TsidBuilder tsidBuilder = new TsidBuilder(dataPoint.getAttributes().size());
26+
public static TsidBuilder forDataPoint(BufferedByteStringAccessor byteStringAccessor, DataPoint dataPoint, int scopeTsidBuilderSize) {
27+
TsidBuilder tsidBuilder = new TsidBuilder(dataPoint.getAttributes().size() + scopeTsidBuilderSize + EXTRA_DIMENSIONS_SIZE);
2528
new DataPointTsidFunnel(byteStringAccessor).add(dataPoint, tsidBuilder);
2629
return tsidBuilder;
2730
}

‎x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/tsid/ResourceTsidFunnel.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ public ResourceTsidFunnel(BufferedByteStringAccessor byteStringAccessor) {
2525
}
2626

2727
public static TsidBuilder forResource(BufferedByteStringAccessor byteStringAccessor, ResourceMetrics resourceMetrics) {
28-
TsidBuilder tsidBuilder = new TsidBuilder(resourceMetrics.getResource().getAttributesCount() + 1);
28+
TsidBuilder tsidBuilder = new TsidBuilder(resourceMetrics.getResource().getAttributesCount());
2929
new ResourceTsidFunnel(byteStringAccessor).add(resourceMetrics, tsidBuilder);
3030
return tsidBuilder;
3131
}
@@ -34,6 +34,5 @@ public static TsidBuilder forResource(BufferedByteStringAccessor byteStringAcces
3434
public void add(ResourceMetrics resourceMetrics, TsidBuilder tsidBuilder) {
3535
List<KeyValue> resourceAttributes = resourceMetrics.getResource().getAttributesList();
3636
tsidBuilder.add(resourceAttributes, AttributeListTsidFunnel.get(byteStringAccessor, "resource.attributes."));
37-
byteStringAccessor.addStringDimension(tsidBuilder, "schema_url", resourceMetrics.getSchemaUrlBytes());
3837
}
3938
}

‎x-pack/plugin/otel-data/src/test/java/org/elasticsearch/xpack/oteldata/otlp/OtlpUtils.java‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -190,15 +190,13 @@ public static SummaryDataPoint createSummaryDataPoint(long timestamp, List<KeyVa
190190
}
191191

192192
public static ExportMetricsServiceRequest createMetricsRequest(List<Metric> metrics) {
193+
return createMetricsRequest(List.of(keyValue("service.name", "test-service")), metrics);
194+
}
193195

196+
public static ExportMetricsServiceRequest createMetricsRequest(List<KeyValue> resourceAttributes, List<Metric> metrics) {
194197
List<ResourceMetrics> resourceMetrics = new ArrayList<>();
195198
for (Metric metric : metrics) {
196-
resourceMetrics.add(
197-
createResourceMetrics(
198-
List.of(keyValue("service.name", "test-service")),
199-
List.of(createScopeMetrics("test", "1.0.0", List.of(metric)))
200-
)
201-
);
199+
resourceMetrics.add(createResourceMetrics(resourceAttributes, List.of(createScopeMetrics("test", "1.0.0", List.of(metric)))));
202200
}
203201

204202
return ExportMetricsServiceRequest.newBuilder().addAllResourceMetrics(resourceMetrics).build();

0 commit comments

Comments
 (0)