Skip to content

Commit dff5196

Browse files
gmarouligmarouli
authored
Add histogram as a metric type of an exponential histogram (#138366)
This PR introduces the `time_series_metric` parameter to the `exponential_histogram` mapper to allow a user to define it as a metric. Furthermore, we introduce a new non scalar time series metric: `histogram`. Currently, the `histogram` metric type is not protected by a feature flag but it's only used by the `exponential_histogram` type which is behind a feature flag. Resolves: #138161
1 parent 0fb42ed commit dff5196

File tree

10 files changed

+174
-9
lines changed

10 files changed

+174
-9
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ private TimeSeriesParams() {}
3636
public enum MetricType {
3737
GAUGE(new String[] { "max", "min", "value_count", "sum" }),
3838
COUNTER(new String[] { "last_value" }),
39+
HISTOGRAM(new String[] {}, false),
3940
POSITION(new String[] {}, false);
4041

4142
private final String[] supportedAggs;

‎x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ private AbstractDownsampleFieldProducer createFieldProducer(DownsampleConfig.Sam
7272
return switch (fieldType.getMetricType()) {
7373
case GAUGE -> MetricFieldProducer.createFieldProducerForGauge(name(), samplingMethod);
7474
case COUNTER -> MetricFieldProducer.createFieldProducerForCounter(name());
75+
case HISTOGRAM -> throw new IllegalArgumentException("Unsupported metric type [histogram] for downsampling, coming soon");
7576
// TODO: Support POSITION in downsampling
7677
case POSITION -> throw new IllegalArgumentException("Unsupported metric type [position] for down-sampling");
7778
};

‎x-pack/plugin/mapper-exponential-histogram/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ base {
2020

2121
restResources {
2222
restApi {
23-
include 'bulk', 'search'
23+
include 'bulk', 'search', 'indices', 'cluster'
2424
}
2525
}
2626
dependencies {

‎x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
import org.elasticsearch.index.mapper.NumberFieldMapper;
4747
import org.elasticsearch.index.mapper.SourceLoader;
4848
import org.elasticsearch.index.mapper.SourceValueFetcher;
49+
import org.elasticsearch.index.mapper.TimeSeriesParams;
4950
import org.elasticsearch.index.mapper.ValueFetcher;
5051
import org.elasticsearch.index.mapper.blockloader.docvalues.BlockDocValuesReader;
5152
import org.elasticsearch.index.mapper.blockloader.docvalues.BytesRefsFromBinaryBlockLoader;
@@ -151,6 +152,11 @@ static class Builder extends FieldMapper.Builder {
151152
private final FieldMapper.Parameter<Map<String, String>> meta = FieldMapper.Parameter.metaParam();
152153
private final FieldMapper.Parameter<Explicit<Boolean>> ignoreMalformed;
153154
private final Parameter<Explicit<Boolean>> coerce;
155+
/**
156+
* Parameter that marks this field as a time series metric defining its time series metric type.
157+
* Only the metric type histogram is supported.
158+
*/
159+
private final Parameter<TimeSeriesParams.MetricType> metric;
154160

155161
Builder(String name, boolean ignoreMalformedByDefault, boolean coerceByDefault) {
156162
super(name);
@@ -161,18 +167,24 @@ static class Builder extends FieldMapper.Builder {
161167
ignoreMalformedByDefault
162168
);
163169
this.coerce = Parameter.explicitBoolParam("coerce", true, m -> toType(m).coerce, coerceByDefault);
170+
this.metric = TimeSeriesParams.metricParam(m -> toType(m).metricType, TimeSeriesParams.MetricType.HISTOGRAM);
171+
}
172+
173+
public Builder metric(TimeSeriesParams.MetricType metric) {
174+
this.metric.setValue(metric);
175+
return this;
164176
}
165177

166178
@Override
167179
protected FieldMapper.Parameter<?>[] getParameters() {
168-
return new FieldMapper.Parameter<?>[] { ignoreMalformed, coerce, meta };
180+
return new FieldMapper.Parameter<?>[] { ignoreMalformed, coerce, meta, metric };
169181
}
170182

171183
@Override
172184
public ExponentialHistogramFieldMapper build(MapperBuilderContext context) {
173185
return new ExponentialHistogramFieldMapper(
174186
leafName(),
175-
new ExponentialHistogramFieldType(context.buildFullName(leafName()), meta.getValue()),
187+
new ExponentialHistogramFieldType(context.buildFullName(leafName()), meta.getValue(), metric.getValue()),
176188
builderParams(this, context),
177189
this
178190
);
@@ -189,6 +201,7 @@ public ExponentialHistogramFieldMapper build(MapperBuilderContext context) {
189201

190202
private final Explicit<Boolean> coerce;
191203
private final boolean coerceByDefault;
204+
private final TimeSeriesParams.MetricType metricType;
192205

193206
ExponentialHistogramFieldMapper(
194207
String simpleName,
@@ -201,6 +214,7 @@ public ExponentialHistogramFieldMapper build(MapperBuilderContext context) {
201214
this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue().value();
202215
this.coerce = builder.coerce.getValue();
203216
this.coerceByDefault = builder.coerce.getDefaultValue().value();
217+
this.metricType = builder.metric.getValue();
204218
}
205219

206220
@Override
@@ -219,7 +233,7 @@ protected String contentType() {
219233

220234
@Override
221235
public FieldMapper.Builder getMergeBuilder() {
222-
return new Builder(leafName(), ignoreMalformedByDefault, coerceByDefault).init(this);
236+
return new Builder(leafName(), ignoreMalformedByDefault, coerceByDefault).metric(metricType).init(this);
223237
}
224238

225239
@Override
@@ -229,9 +243,12 @@ protected void parseCreateField(DocumentParserContext context) {
229243

230244
public static final class ExponentialHistogramFieldType extends MappedFieldType {
231245

246+
private final TimeSeriesParams.MetricType metricType;
247+
232248
// Visible for testing
233-
public ExponentialHistogramFieldType(String name, Map<String, String> meta) {
249+
public ExponentialHistogramFieldType(String name, Map<String, String> meta, TimeSeriesParams.MetricType metricType) {
234250
super(name, IndexType.docValuesOnly(), false, meta);
251+
this.metricType = metricType;
235252
}
236253

237254
@Override
@@ -254,6 +271,11 @@ public boolean isAggregatable() {
254271
return true;
255272
}
256273

274+
@Override
275+
public TimeSeriesParams.MetricType getMetricType() {
276+
return metricType;
277+
}
278+
257279
@Override
258280
public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
259281
return (cache, breakerService) -> new IndexExponentialHistogramFieldData(name()) {

‎x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.elasticsearch.index.mapper.DocumentParsingException;
3232
import org.elasticsearch.index.mapper.MappedFieldType;
3333
import org.elasticsearch.index.mapper.MapperParsingException;
34+
import org.elasticsearch.index.mapper.MapperService;
3435
import org.elasticsearch.index.mapper.MapperTestCase;
3536
import org.elasticsearch.index.mapper.ParsedDocument;
3637
import org.elasticsearch.index.mapper.SourceToParse;
@@ -743,6 +744,42 @@ public void testFormattedDocValues() throws IOException {
743744
}
744745
}
745746

747+
public void testMetricType() throws IOException {
748+
// Test default setting
749+
MapperService mapperService = createMapperService(fieldMapping(this::minimalMapping));
750+
ExponentialHistogramFieldMapper.ExponentialHistogramFieldType ft =
751+
(ExponentialHistogramFieldMapper.ExponentialHistogramFieldType) mapperService.fieldType("field");
752+
assertNull(ft.getMetricType());
753+
754+
assertMetricType("histogram", ExponentialHistogramFieldMapper.ExponentialHistogramFieldType::getMetricType);
755+
756+
{
757+
String unsupportedMetricTypes = randomFrom("counter", "gauge", "position");
758+
// Test invalid metric type for this field type
759+
Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> {
760+
minimalMapping(b);
761+
b.field("time_series_metric", unsupportedMetricTypes);
762+
})));
763+
assertThat(
764+
e.getCause().getMessage(),
765+
containsString(
766+
"Unknown value [" + unsupportedMetricTypes + "] for field [time_series_metric] - accepted values are [histogram]"
767+
)
768+
);
769+
}
770+
{
771+
// Test invalid metric type
772+
Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> {
773+
minimalMapping(b);
774+
b.field("time_series_metric", "unknown");
775+
})));
776+
assertThat(
777+
e.getCause().getMessage(),
778+
containsString("Unknown value [unknown] for field [time_series_metric] - accepted values are [histogram]")
779+
);
780+
}
781+
}
782+
746783
@Override
747784
public void testSyntheticSourceKeepArrays() {
748785
// exponential_histogram can't be used within an array

‎x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/aggregations/bucket/histogram/ExponentialHistogramBackedHistogramAggregatorTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ private void testCase(
194194
}
195195

196196
private MappedFieldType defaultFieldType(String fieldName) {
197-
return new ExponentialHistogramFieldMapper.ExponentialHistogramFieldType(fieldName, Collections.emptyMap());
197+
return new ExponentialHistogramFieldMapper.ExponentialHistogramFieldType(fieldName, Collections.emptyMap(), null);
198198
}
199199

200200
private Map<Double, Long> computeExpectedHistogram(List<ExponentialHistogram> histograms, double interval, double offset) {

‎x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/aggregations/metrics/ExponentialHistogramAvgAggregatorTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ public void testQueryFiltering() throws IOException {
109109

110110
private void testCase(Query query, CheckedConsumer<RandomIndexWriter, IOException> buildIndex, Consumer<InternalAvg> verify)
111111
throws IOException {
112-
var fieldType = new ExponentialHistogramFieldMapper.ExponentialHistogramFieldType(FIELD_NAME, Collections.emptyMap());
112+
var fieldType = new ExponentialHistogramFieldMapper.ExponentialHistogramFieldType(FIELD_NAME, Collections.emptyMap(), null);
113113
AggregationBuilder aggregationBuilder = createAggBuilderForTypeTest(fieldType, FIELD_NAME);
114114
testCase(buildIndex, verify, new AggTestConfig(aggregationBuilder, fieldType).withQuery(query));
115115
}

‎x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/aggregations/metrics/ExponentialHistogramSumAggregatorTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ public void testQueryFiltering() throws IOException {
9696

9797
private void testCase(Query query, CheckedConsumer<RandomIndexWriter, IOException> buildIndex, Consumer<Sum> verify)
9898
throws IOException {
99-
var fieldType = new ExponentialHistogramFieldMapper.ExponentialHistogramFieldType(FIELD_NAME, Collections.emptyMap());
99+
var fieldType = new ExponentialHistogramFieldMapper.ExponentialHistogramFieldType(FIELD_NAME, Collections.emptyMap(), null);
100100
AggregationBuilder aggregationBuilder = createAggBuilderForTypeTest(fieldType, FIELD_NAME);
101101
testCase(buildIndex, verify, new AggTestConfig(aggregationBuilder, fieldType).withQuery(query));
102102
}

‎x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/aggregations/metrics/ExponentialHistogramValueCountAggregatorTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ public void testQueryFiltering() throws IOException {
9696

9797
private void testCase(Query query, CheckedConsumer<RandomIndexWriter, IOException> buildIndex, Consumer<InternalValueCount> verify)
9898
throws IOException {
99-
var fieldType = new ExponentialHistogramFieldMapper.ExponentialHistogramFieldType(FIELD_NAME, Collections.emptyMap());
99+
var fieldType = new ExponentialHistogramFieldMapper.ExponentialHistogramFieldType(FIELD_NAME, Collections.emptyMap(), null);
100100
AggregationBuilder aggregationBuilder = createAggBuilderForTypeTest(fieldType, FIELD_NAME);
101101
testCase(buildIndex, verify, new AggTestConfig(aggregationBuilder, fieldType).withQuery(query));
102102
}

‎x-pack/plugin/mapper-exponential-histogram/src/yamlRestTest/resources/rest-api-spec/test/50_time_series_metric.yml‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
# TODO we need to add "requires capability" when the feature flag is removed
2+
---
3+
"Create time series metric with exponential histogram":
4+
- do:
5+
indices.create:
6+
index: test_exponential_histogram_metric
7+
body:
8+
settings:
9+
index:
10+
mapping:
11+
source:
12+
mode: synthetic
13+
mappings:
14+
properties:
15+
my_metric:
16+
type: exponential_histogram
17+
time_series_metric: histogram
18+
- is_true: shards_acknowledged
19+
20+
- do:
21+
indices.get_mapping:
22+
index: test_exponential_histogram_metric
23+
- match: { test_exponential_histogram_metric.mappings.properties.my_metric.type: exponential_histogram }
24+
- match: { test_exponential_histogram_metric.mappings.properties.my_metric.time_series_metric: histogram }
25+
26+
---
27+
"Merging templates with time series metric":
28+
- do:
29+
cluster.put_component_template:
30+
name: metric-histogram
31+
body:
32+
template:
33+
mappings:
34+
properties:
35+
my_histogram:
36+
type: exponential_histogram
37+
time_series_metric: histogram
38+
39+
40+
- is_true: acknowledged
41+
42+
- do:
43+
indices.put_index_template:
44+
name: only-histogram
45+
body:
46+
index_patterns: [ my-templated-index ]
47+
composed_of: [ metric-histogram ]
48+
template:
49+
mappings:
50+
properties:
51+
my_counter:
52+
type: long
53+
time_series_metric: counter
54+
55+
- is_true: acknowledged
56+
57+
- do:
58+
indices.create:
59+
index: my-templated-index
60+
- is_true: shards_acknowledged
61+
62+
- do:
63+
indices.get_mapping:
64+
index: my-templated-index
65+
- match: { my-templated-index.mappings.properties.my_histogram.type: exponential_histogram }
66+
- match: { my-templated-index.mappings.properties.my_histogram.time_series_metric: histogram }
67+
- match: { my-templated-index.mappings.properties.my_counter.type: long }
68+
- match: { my-templated-index.mappings.properties.my_counter.time_series_metric: counter }
69+
70+
---
71+
"Exponential histogram cannot be a gauge":
72+
- do:
73+
catch: /Unknown value \[gauge\] for field \[time_series_metric\] - accepted values are \[histogram\]/
74+
indices.create:
75+
index: invalid_exponential_histogram_metric
76+
body:
77+
settings:
78+
index:
79+
mapping:
80+
source:
81+
mode: synthetic
82+
mappings:
83+
properties:
84+
my_metric:
85+
type: exponential_histogram
86+
time_series_metric: gauge
87+
88+
---
89+
"Exponential histogram cannot be a counter":
90+
- do:
91+
catch: /Unknown value \[counter\] for field \[time_series_metric\] - accepted values are \[histogram\]/
92+
indices.create:
93+
index: invalid_exponential_histogram_metric
94+
body:
95+
settings:
96+
index:
97+
mapping:
98+
source:
99+
mode: synthetic
100+
mappings:
101+
properties:
102+
my_metric:
103+
type: exponential_histogram
104+
time_series_metric: counter

0 commit comments

Comments
 (0)