Skip to content

Commit fbeebad

Browse files
craigtavernercraigtaverner
committed
Merge branch 'fix_release_tests_geogrid' of github.com:craigtaverner/elasticsearch into fix_release_tests_geogrid
2 parents 424c1cd + eaa9c3b commit fbeebad

File tree

7 files changed

+227
-15
lines changed

7 files changed

+227
-15
lines changed

‎x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/datapoint/DataPoint.java‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import io.opentelemetry.proto.metrics.v1.NumberDataPoint;
1313

1414
import org.elasticsearch.xcontent.XContentBuilder;
15+
import org.elasticsearch.xpack.oteldata.otlp.docbuilder.MappingHints;
1516

1617
import java.io.IOException;
1718
import java.util.List;
@@ -80,9 +81,10 @@ public interface DataPoint {
8081
* Returns the dynamic template name for the data point based on its type and value.
8182
* This is used to dynamically map the appropriate field type according to the data point's characteristics.
8283
*
84+
* @param mappingHints hints for building the dynamic template
8385
* @return the dynamic template name as a string
8486
*/
85-
String getDynamicTemplate();
87+
String getDynamicTemplate(MappingHints mappingHints);
8688

8789
/**
8890
* Validates whether the data point can be indexed into Elasticsearch.
@@ -92,6 +94,14 @@ public interface DataPoint {
9294
*/
9395
boolean isValid(Set<String> errors);
9496

97+
/**
98+
* Returns the {@code _doc_count} for the data point.
99+
* This is used when {@link MappingHints#docCount()} is true.
100+
*
101+
* @return the {@code _doc_count}
102+
*/
103+
long getDocCount();
104+
95105
record Number(NumberDataPoint dataPoint, Metric metric) implements DataPoint {
96106

97107
@Override
@@ -128,7 +138,12 @@ public void buildMetricValue(XContentBuilder builder) throws IOException {
128138
}
129139

130140
@Override
131-
public String getDynamicTemplate() {
141+
public long getDocCount() {
142+
return 1;
143+
}
144+
145+
@Override
146+
public String getDynamicTemplate(MappingHints mappingHints) {
132147
String type;
133148
if (metric.hasSum()
134149
// TODO add support for delta counters - for now we represent them as gauges

‎x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/docbuilder/MappingHints.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.oteldata.otlp.docbuilder;
9+
10+
import io.opentelemetry.proto.common.v1.AnyValue;
11+
import io.opentelemetry.proto.common.v1.KeyValue;
12+
13+
import java.util.List;
14+
15+
/**
16+
* Represents mapping hints that can be used to influence how data is indexed in Elasticsearch.
17+
* These hints can be provided by users via data point attributes.
18+
*
19+
* @param aggregateMetricDouble Indicates that the metric should be mapped as an aggregate_metric_double.
20+
* This hint is available for histogram and exponential histogram metrics.
21+
* @param docCount Indicates that the metric should be mapped with a _doc_count field.
22+
* This hint is available for all metric types.
23+
* When used for a histogram, exponential histogram, or summary metric,
24+
* the _doc_count field will be populated with the number of total counts.
25+
* It is not recommended to use this hint for multiple metrics that are grouped together
26+
* into the same document.
27+
* In these cases, the behavior is undefined but does not lead to data loss.
28+
*/
29+
public record MappingHints(boolean aggregateMetricDouble, boolean docCount) {
30+
public static final String MAPPING_HINTS = "elasticsearch.mapping.hints";
31+
32+
private static final MappingHints EMPTY = new MappingHints(false, false);
33+
private static final String AGGREGATE_METRIC_DOUBLE = "aggregate_metric_double";
34+
private static final String DOC_COUNT = "_doc_count";
35+
36+
public static MappingHints fromAttributes(List<KeyValue> attributes) {
37+
boolean aggregateMetricDouble = false;
38+
boolean docCount = false;
39+
for (int i = 0, attributesSize = attributes.size(); i < attributesSize; i++) {
40+
KeyValue attribute = attributes.get(i);
41+
if (attribute.getKey().equals(MAPPING_HINTS)) {
42+
if (attribute.getValue().hasArrayValue()) {
43+
List<AnyValue> valuesList = attribute.getValue().getArrayValue().getValuesList();
44+
for (int j = 0, valuesListSize = valuesList.size(); j < valuesListSize; j++) {
45+
AnyValue hint = valuesList.get(j);
46+
if (hint.hasStringValue()) {
47+
String value = hint.getStringValue();
48+
if (value.equals(AGGREGATE_METRIC_DOUBLE)) {
49+
aggregateMetricDouble = true;
50+
} else if (value.equals(DOC_COUNT)) {
51+
docCount = true;
52+
}
53+
}
54+
}
55+
}
56+
return new MappingHints(aggregateMetricDouble, docCount);
57+
}
58+
}
59+
return EMPTY;
60+
}
61+
62+
public static MappingHints empty() {
63+
return EMPTY;
64+
}
65+
}

‎x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/docbuilder/MetricDocumentBuilder.java‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ public HashMap<String, String> buildMetricDocument(XContentBuilder builder, Data
5656
DataPoint dataPoint = dataPoints.get(i);
5757
builder.field(dataPoint.getMetricName());
5858
dataPoint.buildMetricValue(builder);
59-
String dynamicTemplate = dataPoint.getDynamicTemplate();
59+
String dynamicTemplate = dataPoint.getDynamicTemplate(MappingHints.empty());
6060
if (dynamicTemplate != null) {
6161
dynamicTemplates.put("metrics." + dataPoint.getMetricName(), dynamicTemplate);
6262
}
@@ -111,11 +111,26 @@ private void buildDataPointAttributes(XContentBuilder builder, List<KeyValue> at
111111
private void buildAttributes(XContentBuilder builder, List<KeyValue> attributes) throws IOException {
112112
for (int i = 0, size = attributes.size(); i < size; i++) {
113113
KeyValue attribute = attributes.get(i);
114-
builder.field(attribute.getKey());
115-
attributeValue(builder, attribute.getValue());
114+
String key = attribute.getKey();
115+
if (isIgnoredAttribute(key) == false) {
116+
builder.field(key);
117+
attributeValue(builder, attribute.getValue());
118+
}
116119
}
117120
}
118121

122+
/**
123+
* Checks if the given attribute key is an ignored attribute.
124+
* Ignored attributes are well-known Elastic-specific attributes
125+
* that influence how the documents are indexed but are not stored themselves.
126+
*
127+
* @param attributeKey the attribute key to check
128+
* @return true if the attribute is ignored, false otherwise
129+
*/
130+
public static boolean isIgnoredAttribute(String attributeKey) {
131+
return attributeKey.equals(MappingHints.MAPPING_HINTS);
132+
}
133+
119134
private void attributeValue(XContentBuilder builder, AnyValue value) throws IOException {
120135
switch (value.getValueCase()) {
121136
case STRING_VALUE -> byteStringAccessor.utf8Value(builder, value.getStringValueBytes());

‎x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/tsid/AttributeListTsidFunnel.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212

1313
import org.elasticsearch.cluster.routing.TsidBuilder;
1414
import org.elasticsearch.cluster.routing.TsidBuilder.TsidFunnel;
15+
import org.elasticsearch.xpack.oteldata.otlp.docbuilder.MetricDocumentBuilder;
1516
import org.elasticsearch.xpack.oteldata.otlp.proto.BufferedByteStringAccessor;
1617

1718
import java.util.List;
@@ -35,7 +36,9 @@ public void add(List<KeyValue> attributesList, TsidBuilder tsidBuilder) {
3536
for (int i = 0; i < attributesList.size(); i++) {
3637
KeyValue keyValue = attributesList.get(i);
3738
String attributeKey = keyValue.getKey();
38-
hashValue(tsidBuilder, prefix + attributeKey, keyValue.getValue());
39+
if (MetricDocumentBuilder.isIgnoredAttribute(attributeKey) == false) {
40+
hashValue(tsidBuilder, prefix + attributeKey, keyValue.getValue());
41+
}
3942
}
4043
}
4144

‎x-pack/plugin/otel-data/src/test/java/org/elasticsearch/xpack/oteldata/otlp/datapoint/DataPointNumberTests.java‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
package org.elasticsearch.xpack.oteldata.otlp.datapoint;
99

1010
import org.elasticsearch.test.ESTestCase;
11+
import org.elasticsearch.xpack.oteldata.otlp.docbuilder.MappingHints;
1112

1213
import java.util.List;
1314

@@ -28,48 +29,48 @@ public void testGauge() {
2829
createDoubleDataPoint(nowUnixNanos),
2930
createGaugeMetric("system.cpu.usage", "", List.of())
3031
);
31-
assertThat(doubleGauge.getDynamicTemplate(), equalTo("gauge_double"));
32+
assertThat(doubleGauge.getDynamicTemplate(MappingHints.empty()), equalTo("gauge_double"));
3233
DataPoint.Number longGauge = new DataPoint.Number(
3334
createLongDataPoint(nowUnixNanos),
3435
createGaugeMetric("system.cpu.usage", "", List.of())
3536
);
36-
assertThat(longGauge.getDynamicTemplate(), equalTo("gauge_long"));
37+
assertThat(longGauge.getDynamicTemplate(MappingHints.empty()), equalTo("gauge_long"));
3738
}
3839

3940
public void testCounterTemporality() {
4041
DataPoint.Number doubleCumulative = new DataPoint.Number(
4142
createDoubleDataPoint(nowUnixNanos),
4243
createSumMetric("http.requests.count", "", List.of(), true, AGGREGATION_TEMPORALITY_CUMULATIVE)
4344
);
44-
assertThat(doubleCumulative.getDynamicTemplate(), equalTo("counter_double"));
45+
assertThat(doubleCumulative.getDynamicTemplate(MappingHints.empty()), equalTo("counter_double"));
4546
DataPoint.Number longCumulative = new DataPoint.Number(
4647
createLongDataPoint(nowUnixNanos),
4748
createSumMetric("http.requests.count", "", List.of(), true, AGGREGATION_TEMPORALITY_CUMULATIVE)
4849
);
49-
assertThat(longCumulative.getDynamicTemplate(), equalTo("counter_long"));
50+
assertThat(longCumulative.getDynamicTemplate(MappingHints.empty()), equalTo("counter_long"));
5051
DataPoint.Number doubleDelta = new DataPoint.Number(
5152
createDoubleDataPoint(nowUnixNanos),
5253
createSumMetric("http.requests.count", "", List.of(), true, AGGREGATION_TEMPORALITY_DELTA)
5354
);
54-
assertThat(doubleDelta.getDynamicTemplate(), equalTo("gauge_double"));
55+
assertThat(doubleDelta.getDynamicTemplate(MappingHints.empty()), equalTo("gauge_double"));
5556
DataPoint.Number longDelta = new DataPoint.Number(
5657
createLongDataPoint(nowUnixNanos),
5758
createSumMetric("http.requests.count", "", List.of(), true, AGGREGATION_TEMPORALITY_DELTA)
5859
);
59-
assertThat(longDelta.getDynamicTemplate(), equalTo("gauge_long"));
60+
assertThat(longDelta.getDynamicTemplate(MappingHints.empty()), equalTo("gauge_long"));
6061
}
6162

6263
public void testCounterNonMonotonic() {
6364
DataPoint.Number doubleNonMonotonic = new DataPoint.Number(
6465
createDoubleDataPoint(nowUnixNanos),
6566
createSumMetric("http.requests.count", "", List.of(), false, AGGREGATION_TEMPORALITY_CUMULATIVE)
6667
);
67-
assertThat(doubleNonMonotonic.getDynamicTemplate(), equalTo("gauge_double"));
68+
assertThat(doubleNonMonotonic.getDynamicTemplate(MappingHints.empty()), equalTo("gauge_double"));
6869
DataPoint.Number longNonMonotonic = new DataPoint.Number(
6970
createLongDataPoint(nowUnixNanos),
7071
createSumMetric("http.requests.count", "", List.of(), false, AGGREGATION_TEMPORALITY_DELTA)
7172
);
72-
assertThat(longNonMonotonic.getDynamicTemplate(), equalTo("gauge_long"));
73+
assertThat(longNonMonotonic.getDynamicTemplate(MappingHints.empty()), equalTo("gauge_long"));
7374
}
7475

7576
}

‎x-pack/plugin/otel-data/src/test/java/org/elasticsearch/xpack/oteldata/otlp/datapoint/MappingHintsTests.java‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.oteldata.otlp.datapoint;
9+
10+
import io.opentelemetry.proto.common.v1.AnyValue;
11+
import io.opentelemetry.proto.common.v1.ArrayValue;
12+
import io.opentelemetry.proto.common.v1.KeyValue;
13+
14+
import org.elasticsearch.test.ESTestCase;
15+
import org.elasticsearch.xpack.oteldata.otlp.docbuilder.MappingHints;
16+
17+
import java.util.List;
18+
19+
public class MappingHintsTests extends ESTestCase {
20+
21+
public void testEmptyAttributes() {
22+
MappingHints hints = MappingHints.fromAttributes(List.of());
23+
assertFalse(hints.aggregateMetricDouble());
24+
assertFalse(hints.docCount());
25+
}
26+
27+
public void testNoMappingHints() {
28+
KeyValue kv = KeyValue.newBuilder()
29+
.setKey("some.other.key")
30+
.setValue(AnyValue.newBuilder().setStringValue("some_value").build())
31+
.build();
32+
MappingHints hints = MappingHints.fromAttributes(List.of(kv));
33+
assertFalse(hints.aggregateMetricDouble());
34+
assertFalse(hints.docCount());
35+
}
36+
37+
public void testSingleMappingHint() {
38+
// Test with just aggregate_metric_double hint
39+
KeyValue aggregateMetricHint = createMappingHint("aggregate_metric_double");
40+
MappingHints hints = MappingHints.fromAttributes(List.of(aggregateMetricHint));
41+
assertTrue(hints.aggregateMetricDouble());
42+
assertFalse(hints.docCount());
43+
44+
// Test with just _doc_count hint
45+
KeyValue docCountHint = createMappingHint("_doc_count");
46+
hints = MappingHints.fromAttributes(List.of(docCountHint));
47+
assertFalse(hints.aggregateMetricDouble());
48+
assertTrue(hints.docCount());
49+
}
50+
51+
public void testMultipleMappingHints() {
52+
// Test with both hints
53+
KeyValue bothHints = createMappingHint("aggregate_metric_double", "_doc_count");
54+
MappingHints hints = MappingHints.fromAttributes(List.of(bothHints));
55+
assertTrue(hints.aggregateMetricDouble());
56+
assertTrue(hints.docCount());
57+
}
58+
59+
public void testInvalidHints() {
60+
// Test with invalid hint
61+
KeyValue invalidHint = createMappingHint("invalid_hint");
62+
MappingHints hints = MappingHints.fromAttributes(List.of(invalidHint));
63+
assertFalse(hints.aggregateMetricDouble());
64+
assertFalse(hints.docCount());
65+
66+
// Test with mix of valid and invalid hints
67+
KeyValue mixedHints = createMappingHint("aggregate_metric_double", "invalid_hint", "_doc_count");
68+
hints = MappingHints.fromAttributes(List.of(mixedHints));
69+
assertTrue(hints.aggregateMetricDouble());
70+
assertTrue(hints.docCount());
71+
}
72+
73+
public void testNonArrayValue() {
74+
// Test with non-array value
75+
KeyValue nonArrayHint = KeyValue.newBuilder()
76+
.setKey("elasticsearch.mapping.hints")
77+
.setValue(AnyValue.newBuilder().setStringValue("aggregate_metric_double").build())
78+
.build();
79+
MappingHints hints = MappingHints.fromAttributes(List.of(nonArrayHint));
80+
assertFalse(hints.aggregateMetricDouble());
81+
assertFalse(hints.docCount());
82+
}
83+
84+
public void testNonStringArrayValues() {
85+
// Test with non-string array values
86+
AnyValue numberValue = AnyValue.newBuilder().setIntValue(42).build();
87+
AnyValue boolValue = AnyValue.newBuilder().setBoolValue(true).build();
88+
89+
ArrayValue.Builder arrayBuilder = ArrayValue.newBuilder();
90+
arrayBuilder.addValues(numberValue);
91+
arrayBuilder.addValues(boolValue);
92+
93+
KeyValue invalidTypeHints = KeyValue.newBuilder()
94+
.setKey("elasticsearch.mapping.hints")
95+
.setValue(AnyValue.newBuilder().setArrayValue(arrayBuilder).build())
96+
.build();
97+
98+
MappingHints hints = MappingHints.fromAttributes(List.of(invalidTypeHints));
99+
assertFalse(hints.aggregateMetricDouble());
100+
assertFalse(hints.docCount());
101+
}
102+
103+
private KeyValue createMappingHint(String... hintValues) {
104+
ArrayValue.Builder arrayBuilder = ArrayValue.newBuilder();
105+
for (String hint : hintValues) {
106+
arrayBuilder.addValues(AnyValue.newBuilder().setStringValue(hint));
107+
}
108+
109+
return KeyValue.newBuilder()
110+
.setKey("elasticsearch.mapping.hints")
111+
.setValue(AnyValue.newBuilder().setArrayValue(arrayBuilder))
112+
.build();
113+
}
114+
}

‎x-pack/plugin/security/src/main/java/module-info.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
requires org.apache.httpcomponents.httpclient;
2626
requires org.apache.httpcomponents.httpasyncclient;
2727
requires org.apache.httpcomponents.httpcore.nio;
28-
requires org.apache.log4j;
2928
requires org.apache.logging.log4j;
3029
requires org.apache.logging.log4j.core;
3130
requires org.apache.lucene.core;

0 commit comments

Comments
 (0)