From c871bbd0c981f910c2cd0a18e87b15ee6be3c94f Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Sun, 10 Aug 2025 15:50:21 -1000 Subject: [PATCH] Set default_metric on index-level --- .../datastreams/DataStreamFeatures.java | 7 +- .../common/settings/IndexScopedSettings.java | 1 + .../elasticsearch/index/IndexSettings.java | 22 +++++ .../index/mapper/TimeSeriesParams.java | 1 + .../downsample/80_downsample_aggregate.yml | 88 +++++++++++++++++++ .../downsample/TimeseriesFieldTypeHelper.java | 5 ++ .../downsample/TransportDownsampleAction.java | 22 +++-- .../TransportDownsampleActionTests.java | 12 ++- 8 files changed, 150 insertions(+), 8 deletions(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java index 38f03557ca9c7..a6e7db66e8c78 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java @@ -30,6 +30,10 @@ public class DataStreamFeatures implements FeatureSpecification { public static final NodeFeature FAILURE_STORE_IN_LOG_DATA_STREAMS = new NodeFeature("logs_data_streams.failure_store.enabled"); + public static final NodeFeature DOWNSAMPLE_INDEX_LEVEL_DEFAULT_METRIC = new NodeFeature( + "data_stream.downsample.index_level_default_metric" + ); + @Override public Set getFeatures() { return Set.of(DataStream.DATA_STREAM_FAILURE_STORE_FEATURE); @@ -41,7 +45,8 @@ public Set getTestFeatures() { DATA_STREAM_FAILURE_STORE_TSDB_FIX, DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX, LOGS_STREAM_FEATURE, - FAILURE_STORE_IN_LOG_DATA_STREAMS + FAILURE_STORE_IN_LOG_DATA_STREAMS, + DOWNSAMPLE_INDEX_LEVEL_DEFAULT_METRIC ); } } diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 93ddb5d3fc485..7aea6df75b4b0 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -223,6 +223,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { // TSDB index settings IndexSettings.MODE, IndexMetadata.INDEX_ROUTING_PATH, + IndexSettings.TIME_SERIES_DEFAULT_METRIC, IndexSettings.TIME_SERIES_START_TIME, IndexSettings.TIME_SERIES_END_TIME, IndexSettings.SEQ_NO_INDEX_OPTIONS_SETTING, diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index a6335ca6666b0..ae4245da7c9e9 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -647,6 +647,26 @@ public Iterator> settings() { Property.Final ); + public enum AggregateMetricDoubleDefaultMetric { + MIN, + MAX, + SUM, + VALUE_COUNT; + } + + public static final Setting TIME_SERIES_DEFAULT_METRIC = Setting.enumSetting( + AggregateMetricDoubleDefaultMetric.class, + "index.time_series.default_metric", + AggregateMetricDoubleDefaultMetric.MAX, + Property.IndexScope, + Property.Final, + Property.ServerlessPublic + ); + + public AggregateMetricDoubleDefaultMetric getDefaultMetric() { + return defaultMetric; + } + /** * Returns true if TSDB encoding is enabled. The default is true */ @@ -896,6 +916,7 @@ private static String getIgnoreAboveDefaultValue(final Settings settings) { private final boolean softDeleteEnabled; private volatile long softDeleteRetentionOperations; private final boolean es87TSDBCodecEnabled; + private final AggregateMetricDoubleDefaultMetric defaultMetric; private final boolean logsdbRouteOnSortFields; private final boolean logsdbSortOnHostName; private final boolean logsdbAddHostNameField; @@ -1116,6 +1137,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti indexRouting = IndexRouting.fromIndexMetadata(indexMetadata); sourceKeepMode = scopedSettings.get(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING); es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING); + defaultMetric = scopedSettings.get(TIME_SERIES_DEFAULT_METRIC); logsdbRouteOnSortFields = scopedSettings.get(LOGSDB_ROUTE_ON_SORT_FIELDS); logsdbSortOnHostName = scopedSettings.get(LOGSDB_SORT_ON_HOST_NAME); logsdbAddHostNameField = scopedSettings.get(LOGSDB_ADD_HOST_NAME_FIELD); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java index 1f4676d60814f..395c78812e491 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesParams.java @@ -21,6 +21,7 @@ public final class TimeSeriesParams { public static final String TIME_SERIES_METRIC_PARAM = "time_series_metric"; public static final String TIME_SERIES_DIMENSION_PARAM = "time_series_dimension"; + public static final String TIME_SERIES_DEFAULT_METRIC_PARAM = "default_metric"; private TimeSeriesParams() {} diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/80_downsample_aggregate.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/80_downsample_aggregate.yml index 991aa3858d8bc..5716bc2341a16 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/80_downsample_aggregate.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/80_downsample_aggregate.yml @@ -77,3 +77,91 @@ metrics: [min, sum, value_count] default_metric: sum time_series_metric: gauge + +--- +"downsample numeric field": + - requires: + cluster_features: ["data_stream.downsample.index_level_default_metric"] + reason: "Specify default metric to be used in an index when downsampling numerics" + + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 1 + index: + mode: time_series + routing_path: [sensor_id] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + default_metric: value_count + mappings: + properties: + "@timestamp": + type: date + sensor_id: + type: keyword + time_series_dimension: true + temperature: + type: double + time_series_metric: gauge + agg_metric: + type: aggregate_metric_double + metrics: [ min, sum, value_count ] + default_metric: sum + time_series_metric: gauge + + - do: + bulk: + refresh: true + index: test + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T17:34:00Z", "sensor_id": "1", "temperature": 24.1, "agg_metric": {"min": -1, "sum": 20, "value_count": 5}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T17:39:00Z", "sensor_id": "1", "temperature": 25.3, "agg_metric": {"min": 3, "sum": 50, "value_count": 7}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T17:45:00Z", "sensor_id": "1", "temperature": 24.8, "agg_metric": {"min": -5, "sum": 33, "value_count": 9}}' + + - do: + indices.put_settings: + index: test + body: + index.blocks.write: true + + - do: + indices.downsample: + index: test + target_index: test-downsample + body: > + { + "fixed_interval": "30m" + } + - is_true: acknowledged + + - do: + search: + index: test-downsample + body: + size: 0 + + - match: + hits.total.value: 1 + + - do: + indices.get_mapping: + index: test-downsample + - match: + test-downsample.mappings.properties.temperature: + type: aggregate_metric_double + metrics: [min, max, sum, value_count] + default_metric: value_count + time_series_metric: gauge + - match: + test-downsample.mappings.properties.agg_metric: + type: aggregate_metric_double + metrics: [min, sum, value_count] + default_metric: sum + time_series_metric: gauge diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java index 93425be0b7d40..b0f7ee9f297b6 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TimeseriesFieldTypeHelper.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.downsample; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; @@ -56,6 +57,10 @@ public static boolean isPassthroughField(final Map fieldMapping) { return PassThroughObjectMapper.CONTENT_TYPE.equals(fieldMapping.get(ContextMapping.FIELD_TYPE)); } + public IndexSettings.AggregateMetricDoubleDefaultMetric getDefaultMetric() { + return mapperService.getIndexSettings().getDefaultMetric(); + } + public List extractFlattenedDimensions(final String field, final Map fieldMapping) { var mapper = mapperService.mappingLookup().getMapper(field); if (mapper instanceof FlattenedFieldMapper == false) { diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index 8486963a5daee..ae491b6fdcb1d 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -92,6 +92,7 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -712,7 +713,7 @@ private static void addMetricFields( MappingVisitor.visitMapping(sourceIndexMappings, (field, mapping) -> { if (helper.isTimeSeriesMetric(field, mapping)) { try { - addMetricFieldMapping(builder, field, mapping); + addMetricFieldMapping(builder, field, mapping, helper.getDefaultMetric()); } catch (IOException e) { throw new ElasticsearchException("Error while adding metric for field [" + field + "]"); } @@ -761,7 +762,8 @@ public record AggregateMetricDoubleFieldSupportedMetrics(String defaultMetric, L // public for testing public static AggregateMetricDoubleFieldSupportedMetrics getSupportedMetrics( final TimeSeriesParams.MetricType metricType, - final Map fieldProperties + final Map fieldProperties, + final IndexSettings.AggregateMetricDoubleDefaultMetric indexLevelDefaultMetric ) { boolean sourceIsAggregate = fieldProperties.get("type").equals(AggregateMetricDoubleFieldMapper.CONTENT_TYPE); List supportedAggs = List.of(metricType.supportedAggs()); @@ -783,13 +785,19 @@ public static AggregateMetricDoubleFieldSupportedMetrics getSupportedMetrics( (String) fieldProperties.get(AggregateMetricDoubleFieldMapper.Names.DEFAULT_METRIC), defaultMetric ); + } else { + defaultMetric = indexLevelDefaultMetric.name().toLowerCase(Locale.ROOT); } return new AggregateMetricDoubleFieldSupportedMetrics(defaultMetric, supportedAggs); } - private static void addMetricFieldMapping(final XContentBuilder builder, final String field, final Map fieldProperties) - throws IOException { + private static void addMetricFieldMapping( + final XContentBuilder builder, + final String field, + final Map fieldProperties, + IndexSettings.AggregateMetricDoubleDefaultMetric defaultMetric + ) throws IOException { final TimeSeriesParams.MetricType metricType = TimeSeriesParams.MetricType.fromString( fieldProperties.get(TIME_SERIES_METRIC_PARAM).toString() ); @@ -801,7 +809,7 @@ private static void addMetricFieldMapping(final XContentBuilder builder, final S builder.field(fieldProperty, fieldProperties.get(fieldProperty)); } } else { - var supported = getSupportedMetrics(metricType, fieldProperties); + var supported = getSupportedMetrics(metricType, fieldProperties, defaultMetric); builder.field("type", AggregateMetricDoubleFieldMapper.CONTENT_TYPE) .stringListField(AggregateMetricDoubleFieldMapper.Names.METRICS, supported.supportedMetrics) @@ -942,6 +950,10 @@ private void createDownsampleIndex( .put( IndexSettings.TIME_SERIES_END_TIME.getKey(), sourceIndexMetadata.getSettings().get(IndexSettings.TIME_SERIES_END_TIME.getKey()) + ) + .put( + IndexSettings.TIME_SERIES_DEFAULT_METRIC.getKey(), + sourceIndexMetadata.getSettings().get(IndexSettings.TIME_SERIES_DEFAULT_METRIC.getKey()) ); if (sourceIndexMetadata.getSettings().hasValue(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey())) { builder.put( diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java index 1b2cc32e12a65..60dab7ff52af7 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/TransportDownsampleActionTests.java @@ -123,12 +123,20 @@ public void testGetSupportedMetrics() { "sum" ); - var supported = TransportDownsampleAction.getSupportedMetrics(metricType, fieldProperties); + var supported = TransportDownsampleAction.getSupportedMetrics( + metricType, + fieldProperties, + IndexSettings.AggregateMetricDoubleDefaultMetric.MAX + ); assertThat(supported.defaultMetric(), is("sum")); assertThat(supported.supportedMetrics(), is(List.of("max", "sum"))); fieldProperties = Map.of("type", "integer"); - supported = TransportDownsampleAction.getSupportedMetrics(metricType, fieldProperties); + supported = TransportDownsampleAction.getSupportedMetrics( + metricType, + fieldProperties, + IndexSettings.AggregateMetricDoubleDefaultMetric.MAX + ); assertThat(supported.defaultMetric(), is("max")); assertThat(supported.supportedMetrics(), is(List.of(metricType.supportedAggs()))); }