Skip to content

Commit 1ac8ce9

Browse files
limotovalimotova
committed
Set default_metric on index-level
1 parent 31088f5 commit 1ac8ce9

File tree

8 files changed

+150
-8
lines changed

8 files changed

+150
-8
lines changed

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ public class DataStreamFeatures implements FeatureSpecification {
3030

3131
public static final NodeFeature FAILURE_STORE_IN_LOG_DATA_STREAMS = new NodeFeature("logs_data_streams.failure_store.enabled");
3232

33+
public static final NodeFeature DOWNSAMPLE_INDEX_LEVEL_DEFAULT_METRIC = new NodeFeature(
34+
"data_stream.downsample.index_level_default_metric"
35+
);
36+
3337
@Override
3438
public Set<NodeFeature> getFeatures() {
3539
return Set.of(DataStream.DATA_STREAM_FAILURE_STORE_FEATURE);
@@ -41,7 +45,8 @@ public Set<NodeFeature> getTestFeatures() {
4145
DATA_STREAM_FAILURE_STORE_TSDB_FIX,
4246
DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX,
4347
LOGS_STREAM_FEATURE,
44-
FAILURE_STORE_IN_LOG_DATA_STREAMS
48+
FAILURE_STORE_IN_LOG_DATA_STREAMS,
49+
DOWNSAMPLE_INDEX_LEVEL_DEFAULT_METRIC
4550
);
4651
}
4752
}

‎server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
223223
// TSDB index settings
224224
IndexSettings.MODE,
225225
IndexMetadata.INDEX_ROUTING_PATH,
226+
IndexSettings.TIME_SERIES_DEFAULT_METRIC,
226227
IndexSettings.TIME_SERIES_START_TIME,
227228
IndexSettings.TIME_SERIES_END_TIME,
228229
IndexSettings.SEQ_NO_INDEX_OPTIONS_SETTING,

‎server/src/main/java/org/elasticsearch/index/IndexSettings.java

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -647,6 +647,26 @@ public Iterator<Setting<?>> settings() {
647647
Property.Final
648648
);
649649

650+
public enum AggregateMetricDoubleDefaultMetric {
651+
MIN,
652+
MAX,
653+
SUM,
654+
VALUE_COUNT;
655+
}
656+
657+
public static final Setting<AggregateMetricDoubleDefaultMetric> TIME_SERIES_DEFAULT_METRIC = Setting.enumSetting(
658+
AggregateMetricDoubleDefaultMetric.class,
659+
"index.time_series.default_metric",
660+
AggregateMetricDoubleDefaultMetric.MAX,
661+
Property.IndexScope,
662+
Property.Final,
663+
Property.ServerlessPublic
664+
);
665+
666+
public AggregateMetricDoubleDefaultMetric getDefaultMetric() {
667+
return defaultMetric;
668+
}
669+
650670
/**
651671
* Returns <code>true</code> if TSDB encoding is enabled. The default is <code>true</code>
652672
*/
@@ -896,6 +916,7 @@ private static String getIgnoreAboveDefaultValue(final Settings settings) {
896916
private final boolean softDeleteEnabled;
897917
private volatile long softDeleteRetentionOperations;
898918
private final boolean es87TSDBCodecEnabled;
919+
private final AggregateMetricDoubleDefaultMetric defaultMetric;
899920
private final boolean logsdbRouteOnSortFields;
900921
private final boolean logsdbSortOnHostName;
901922
private final boolean logsdbAddHostNameField;
@@ -1116,6 +1137,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
11161137
indexRouting = IndexRouting.fromIndexMetadata(indexMetadata);
11171138
sourceKeepMode = scopedSettings.get(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING);
11181139
es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING);
1140+
defaultMetric = scopedSettings.get(TIME_SERIES_DEFAULT_METRIC);
11191141
logsdbRouteOnSortFields = scopedSettings.get(LOGSDB_ROUTE_ON_SORT_FIELDS);
11201142
logsdbSortOnHostName = scopedSettings.get(LOGSDB_SORT_ON_HOST_NAME);
11211143
logsdbAddHostNameField = scopedSettings.get(LOGSDB_ADD_HOST_NAME_FIELD);

‎server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ public final class TimeSeriesParams {
2121

2222
public static final String TIME_SERIES_METRIC_PARAM = "time_series_metric";
2323
public static final String TIME_SERIES_DIMENSION_PARAM = "time_series_dimension";
24+
public static final String TIME_SERIES_DEFAULT_METRIC_PARAM = "default_metric";
2425

2526
private TimeSeriesParams() {}
2627

‎x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/80_downsample_aggregate.yml

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,3 +77,91 @@
7777
metrics: [min, sum, value_count]
7878
default_metric: sum
7979
time_series_metric: gauge
80+
81+
---
82+
"downsample numeric field":
83+
- requires:
84+
cluster_features: ["data_stream.downsample.index_level_default_metric"]
85+
reason: "Specify default metric to be used in an index when downsampling numerics"
86+
87+
- do:
88+
indices.create:
89+
index: test
90+
body:
91+
settings:
92+
number_of_shards: 1
93+
index:
94+
mode: time_series
95+
routing_path: [sensor_id]
96+
time_series:
97+
start_time: 2021-04-28T00:00:00Z
98+
end_time: 2021-04-29T00:00:00Z
99+
default_metric: value_count
100+
mappings:
101+
properties:
102+
"@timestamp":
103+
type: date
104+
sensor_id:
105+
type: keyword
106+
time_series_dimension: true
107+
temperature:
108+
type: double
109+
time_series_metric: gauge
110+
agg_metric:
111+
type: aggregate_metric_double
112+
metrics: [ min, sum, value_count ]
113+
default_metric: sum
114+
time_series_metric: gauge
115+
116+
- do:
117+
bulk:
118+
refresh: true
119+
index: test
120+
body:
121+
- '{"index": {}}'
122+
- '{"@timestamp": "2021-04-28T17:34:00Z", "sensor_id": "1", "temperature": 24.1, "agg_metric": {"max": 10, "min": -1, "sum": 20, "value_count": 5}}'
123+
- '{"index": {}}'
124+
- '{"@timestamp": "2021-04-28T17:39:00Z", "sensor_id": "1", "temperature": 25.3, "agg_metric": {"max": 20, "min": 3, "sum": 50, "value_count": 7}}'
125+
- '{"index": {}}'
126+
- '{"@timestamp": "2021-04-28T17:45:00Z", "sensor_id": "1", "temperature": 24.8, "agg_metric": {"max": 17, "min": -5, "sum": 33, "value_count": 9}}'
127+
128+
- do:
129+
indices.put_settings:
130+
index: test
131+
body:
132+
index.blocks.write: true
133+
134+
- do:
135+
indices.downsample:
136+
index: test
137+
target_index: test-downsample
138+
body: >
139+
{
140+
"fixed_interval": "30m"
141+
}
142+
- is_true: acknowledged
143+
144+
- do:
145+
search:
146+
index: test-downsample
147+
body:
148+
size: 0
149+
150+
- match:
151+
hits.total.value: 1
152+
153+
- do:
154+
indices.get_mapping:
155+
index: test-downsample
156+
- match:
157+
test-downsample.mappings.properties.temperature:
158+
type: aggregate_metric_double
159+
metrics: [min, max, sum, value_count]
160+
default_metric: value_count
161+
time_series_metric: gauge
162+
- match:
163+
test-downsample.mappings.properties.agg_metric:
164+
type: aggregate_metric_double
165+
metrics: [min, max, sum, value_count]
166+
default_metric: sum
167+
time_series_metric: gauge

‎x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
package org.elasticsearch.xpack.downsample;
99

10+
import org.elasticsearch.index.IndexSettings;
1011
import org.elasticsearch.index.mapper.MappedFieldType;
1112
import org.elasticsearch.index.mapper.MapperService;
1213
import org.elasticsearch.index.mapper.MappingLookup;
@@ -56,6 +57,10 @@ public static boolean isPassthroughField(final Map<String, ?> fieldMapping) {
5657
return PassThroughObjectMapper.CONTENT_TYPE.equals(fieldMapping.get(ContextMapping.FIELD_TYPE));
5758
}
5859

60+
public IndexSettings.AggregateMetricDoubleDefaultMetric getDefaultMetric() {
61+
return mapperService.getIndexSettings().getDefaultMetric();
62+
}
63+
5964
public List<String> extractFlattenedDimensions(final String field, final Map<String, ?> fieldMapping) {
6065
var mapper = mapperService.mappingLookup().getMapper(field);
6166
if (mapper instanceof FlattenedFieldMapper == false) {

‎x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@
9292
import java.time.format.DateTimeFormatter;
9393
import java.util.ArrayList;
9494
import java.util.List;
95+
import java.util.Locale;
9596
import java.util.Map;
9697
import java.util.Objects;
9798
import java.util.Set;
@@ -712,7 +713,7 @@ private static void addMetricFields(
712713
MappingVisitor.visitMapping(sourceIndexMappings, (field, mapping) -> {
713714
if (helper.isTimeSeriesMetric(field, mapping)) {
714715
try {
715-
addMetricFieldMapping(builder, field, mapping);
716+
addMetricFieldMapping(builder, field, mapping, helper.getDefaultMetric());
716717
} catch (IOException e) {
717718
throw new ElasticsearchException("Error while adding metric for field [" + field + "]");
718719
}
@@ -761,7 +762,8 @@ public record AggregateMetricDoubleFieldSupportedMetrics(String defaultMetric, L
761762
// public for testing
762763
public static AggregateMetricDoubleFieldSupportedMetrics getSupportedMetrics(
763764
final TimeSeriesParams.MetricType metricType,
764-
final Map<String, ?> fieldProperties
765+
final Map<String, ?> fieldProperties,
766+
final IndexSettings.AggregateMetricDoubleDefaultMetric indexLevelDefaultMetric
765767
) {
766768
boolean sourceIsAggregate = fieldProperties.get("type").equals(AggregateMetricDoubleFieldMapper.CONTENT_TYPE);
767769
List<String> supportedAggs = List.of(metricType.supportedAggs());
@@ -783,13 +785,19 @@ public static AggregateMetricDoubleFieldSupportedMetrics getSupportedMetrics(
783785
(String) fieldProperties.get(AggregateMetricDoubleFieldMapper.Names.DEFAULT_METRIC),
784786
defaultMetric
785787
);
788+
} else {
789+
defaultMetric = indexLevelDefaultMetric.name().toLowerCase(Locale.ROOT);
786790
}
787791

788792
return new AggregateMetricDoubleFieldSupportedMetrics(defaultMetric, supportedAggs);
789793
}
790794

791-
private static void addMetricFieldMapping(final XContentBuilder builder, final String field, final Map<String, ?> fieldProperties)
792-
throws IOException {
795+
private static void addMetricFieldMapping(
796+
final XContentBuilder builder,
797+
final String field,
798+
final Map<String, ?> fieldProperties,
799+
IndexSettings.AggregateMetricDoubleDefaultMetric defaultMetric
800+
) throws IOException {
793801
final TimeSeriesParams.MetricType metricType = TimeSeriesParams.MetricType.fromString(
794802
fieldProperties.get(TIME_SERIES_METRIC_PARAM).toString()
795803
);
@@ -801,7 +809,7 @@ private static void addMetricFieldMapping(final XContentBuilder builder, final S
801809
builder.field(fieldProperty, fieldProperties.get(fieldProperty));
802810
}
803811
} else {
804-
var supported = getSupportedMetrics(metricType, fieldProperties);
812+
var supported = getSupportedMetrics(metricType, fieldProperties, defaultMetric);
805813

806814
builder.field("type", AggregateMetricDoubleFieldMapper.CONTENT_TYPE)
807815
.stringListField(AggregateMetricDoubleFieldMapper.Names.METRICS, supported.supportedMetrics)
@@ -942,6 +950,10 @@ private void createDownsampleIndex(
942950
.put(
943951
IndexSettings.TIME_SERIES_END_TIME.getKey(),
944952
sourceIndexMetadata.getSettings().get(IndexSettings.TIME_SERIES_END_TIME.getKey())
953+
)
954+
.put(
955+
IndexSettings.TIME_SERIES_DEFAULT_METRIC.getKey(),
956+
sourceIndexMetadata.getSettings().get(IndexSettings.TIME_SERIES_DEFAULT_METRIC.getKey())
945957
);
946958
if (sourceIndexMetadata.getSettings().hasValue(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())) {
947959
builder.put(

‎x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,12 +123,20 @@ public void testGetSupportedMetrics() {
123123
"sum"
124124
);
125125

126-
var supported = TransportDownsampleAction.getSupportedMetrics(metricType, fieldProperties);
126+
var supported = TransportDownsampleAction.getSupportedMetrics(
127+
metricType,
128+
fieldProperties,
129+
IndexSettings.AggregateMetricDoubleDefaultMetric.MAX
130+
);
127131
assertThat(supported.defaultMetric(), is("sum"));
128132
assertThat(supported.supportedMetrics(), is(List.of("max", "sum")));
129133

130134
fieldProperties = Map.of("type", "integer");
131-
supported = TransportDownsampleAction.getSupportedMetrics(metricType, fieldProperties);
135+
supported = TransportDownsampleAction.getSupportedMetrics(
136+
metricType,
137+
fieldProperties,
138+
IndexSettings.AggregateMetricDoubleDefaultMetric.MAX
139+
);
132140
assertThat(supported.defaultMetric(), is("max"));
133141
assertThat(supported.supportedMetrics(), is(List.of(metricType.supportedAggs())));
134142
}

0 commit comments

Comments
 (0)