diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml index f76b4eb7bc32f..0bb334b3c53b3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml @@ -276,7 +276,7 @@ override sort order settings: type: text - match: { error.type: "illegal_argument_exception" } - - match: { error.reason: "index.sort.fields:[] index.sort.order:[asc, asc], size mismatch" } + - match: { error.reason: "index.sort.field:[] index.sort.order:[asc, asc], size mismatch" } --- override sort missing settings: @@ -312,7 +312,7 @@ override sort missing settings: type: text - match: { error.type: "illegal_argument_exception" } - - match: { error.reason: "index.sort.fields:[] index.sort.missing:[_last, _first], size mismatch" } + - match: { error.reason: "index.sort.field:[] index.sort.missing:[_last, _first], size mismatch" } --- override sort mode settings: @@ -348,7 +348,7 @@ override sort mode settings: type: text - match: { error.type: "illegal_argument_exception" } - - match: { error.reason: "index.sort.fields:[] index.sort.mode:[MAX, MAX], size mismatch" } + - match: { error.reason: "index.sort.field:[] index.sort.mode:[MAX, MAX], size mismatch" } --- override sort field using nested field type in sorting: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml index c54556e99bee9..3798be1bc611c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml @@ -576,3 +576,29 @@ check time_series empty time bound value: "@timestamp": "2021-09-26T03:09:52.123456789Z", "metricset": "pod" } + +--- +default sort field: + - requires: + cluster_features: ["gte_v8.1.0"] + reason: introduced in 8.1.0 + + - do: + indices.create: + index: test_index + body: + settings: + index: + mode: time_series + routing_path: foo + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + + - do: + indices.get_settings: + index: test_index + include_defaults: true + - match: { .test_index.settings.index.mode: time_series } + - match: { .test_index.settings.index.sort.field: [ "_tsid", "@timestamp" ] } + - match: { .test_index.settings.index.sort.order: [ "asc", "desc" ] } diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 46c790a4b5b70..151c7f2efb690 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -38,6 +38,7 @@ import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; +import org.elasticsearch.search.sort.SortOrder; import java.io.IOException; import java.time.Instant; @@ -402,7 +403,7 @@ public SourceFieldMapper.Mode defaultSourceMode() { } }; - static final String HOST_NAME = "host.name"; + public static final String HOST_NAME = "host.name"; private static void validateRoutingPathSettings(Map, Object> settings) { settingRequiresTimeSeries(settings, IndexMetadata.INDEX_ROUTING_PATH); @@ -451,8 +452,6 @@ private static CompressedXContent createDefaultMapping(boolean includeHostName) } private static final List> TIME_SERIES_UNSUPPORTED = List.of( - IndexSortConfig.INDEX_SORT_FIELD_SETTING, - IndexSortConfig.INDEX_SORT_ORDER_SETTING, IndexSortConfig.INDEX_SORT_MODE_SETTING, IndexSortConfig.INDEX_SORT_MISSING_SETTING ); @@ -639,6 +638,32 @@ public void provideAdditionalSettings( } if (indexMode == LOOKUP) { additionalSettings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1); + } else if (indexMode == TIME_SERIES) { + var sortFields = List.of(TimeSeriesIdFieldMapper.NAME, DataStreamTimestampFieldMapper.DEFAULT_PATH); + var sortOrder = List.of(SortOrder.ASC, SortOrder.DESC); + + if (IndexSortConfig.INDEX_SORT_FIELD_SETTING.exists(indexTemplateAndCreateRequestSettings)) { + if (IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(indexTemplateAndCreateRequestSettings).equals(sortFields) == false) { + throw new IllegalArgumentException( + tsdbMode() + " is incompatible with [" + IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey() + "]" + ); + } + } else { + additionalSettings.putList(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), sortFields); + } + + if (IndexSortConfig.INDEX_SORT_ORDER_SETTING.exists(indexTemplateAndCreateRequestSettings)) { + if (IndexSortConfig.INDEX_SORT_ORDER_SETTING.get(indexTemplateAndCreateRequestSettings).equals(sortOrder) == false) { + throw new IllegalArgumentException( + tsdbMode() + " is incompatible with [" + IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey() + "]" + ); + } + } else { + additionalSettings.putList( + IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), + sortOrder.stream().map(Object::toString).toList() + ); + } } } } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index a3ed94416b7e8..66be825fae4c7 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -102,25 +102,6 @@ public final class IndexSortConfig { Setting.Property.ServerlessPublic ); - public static final FieldSortSpec[] TIME_SERIES_SORT, TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_BWC_SORT; - static { - FieldSortSpec timeStampSpec = new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH); - timeStampSpec.order = SortOrder.DESC; - TIME_SERIES_SORT = new FieldSortSpec[] { new FieldSortSpec(TimeSeriesIdFieldMapper.NAME), timeStampSpec }; - TIMESTAMP_SORT = new FieldSortSpec[] { timeStampSpec }; - - FieldSortSpec hostnameSpec = new FieldSortSpec(IndexMode.HOST_NAME); - hostnameSpec.order = SortOrder.ASC; - hostnameSpec.missingValue = "_last"; - hostnameSpec.mode = MultiValueMode.MIN; - HOSTNAME_TIMESTAMP_SORT = new FieldSortSpec[] { hostnameSpec, timeStampSpec }; - - // Older indexes use ascending ordering for host name and timestamp. - HOSTNAME_TIMESTAMP_BWC_SORT = new FieldSortSpec[] { - new FieldSortSpec(IndexMode.HOST_NAME), - new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH) }; - } - private static String validateMissingValue(String missing) { if ("_last".equals(missing) == false && "_first".equals(missing) == false) { throw new IllegalArgumentException("Illegal missing value:[" + missing + "], " + "must be one of [_last, _first]"); @@ -146,6 +127,85 @@ private static MultiValueMode parseMultiValueMode(String value) { return mode; } + private static void checkSizeMismatch(String firstKey, List first, String secondKey, List second) { + if (first.size() != second.size()) { + throw new IllegalArgumentException(firstKey + ":" + first + " " + secondKey + ":" + second + ", size mismatch"); + } + } + + public static void validateSortSettings(Settings settings) { + List fields = INDEX_SORT_FIELD_SETTING.get(settings); + + if (INDEX_SORT_ORDER_SETTING.exists(settings)) { + var order = INDEX_SORT_ORDER_SETTING.get(settings); + checkSizeMismatch(INDEX_SORT_FIELD_SETTING.getKey(), fields, INDEX_SORT_ORDER_SETTING.getKey(), order); + } + + if (INDEX_SORT_MODE_SETTING.exists(settings)) { + var mode = INDEX_SORT_MODE_SETTING.get(settings); + checkSizeMismatch(INDEX_SORT_FIELD_SETTING.getKey(), fields, INDEX_SORT_MODE_SETTING.getKey(), mode); + } + + if (INDEX_SORT_MISSING_SETTING.exists(settings)) { + var missing = INDEX_SORT_MISSING_SETTING.get(settings); + checkSizeMismatch(INDEX_SORT_FIELD_SETTING.getKey(), fields, INDEX_SORT_MISSING_SETTING.getKey(), missing); + } + } + + public static class LegacyIndexSortConfigDefaults { + public static final FieldSortSpec[] TIME_SERIES_SORT, TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_BWC_SORT; + + static { + FieldSortSpec timeStampSpec = new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH); + timeStampSpec.order = SortOrder.DESC; + TIME_SERIES_SORT = new FieldSortSpec[] { new FieldSortSpec(TimeSeriesIdFieldMapper.NAME), timeStampSpec }; + TIMESTAMP_SORT = new FieldSortSpec[] { timeStampSpec }; + + FieldSortSpec hostnameSpec = new FieldSortSpec(IndexMode.HOST_NAME); + hostnameSpec.order = SortOrder.ASC; + hostnameSpec.missingValue = "_last"; + hostnameSpec.mode = MultiValueMode.MIN; + HOSTNAME_TIMESTAMP_SORT = new FieldSortSpec[] { hostnameSpec, timeStampSpec }; + + // Older indexes use ascending ordering for host name and timestamp. + HOSTNAME_TIMESTAMP_BWC_SORT = new FieldSortSpec[] { + new FieldSortSpec(IndexMode.HOST_NAME), + new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH) }; + } + + public static FieldSortSpec[] getLegacyDefaultSortSpecs(IndexSettings indexSettings) { + var version = indexSettings.getIndexVersionCreated(); + if (version.onOrAfter(IndexVersions.EXPLICIT_INDEX_SORTING_DEFAULTS)) { + return null; + } + + final Settings settings = indexSettings.getSettings(); + + IndexMode indexMode = indexSettings.getMode(); + if (indexMode == IndexMode.TIME_SERIES) { + return TIME_SERIES_SORT; + } else if (indexMode == IndexMode.LOGSDB) { + if (INDEX_SORT_FIELD_SETTING.exists(settings)) { + return null; + } + + validateSortSettings(settings); + + if (version.onOrAfter(IndexVersions.LOGSB_OPTIONAL_SORTING_ON_HOST_NAME) + || version.between( + IndexVersions.LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT, + IndexVersions.UPGRADE_TO_LUCENE_10_0_0 + )) { + return (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(settings)) ? HOSTNAME_TIMESTAMP_SORT : TIMESTAMP_SORT; + } else { + return HOSTNAME_TIMESTAMP_BWC_SORT; + } + } + + return null; + } + } + // visible for tests final FieldSortSpec[] sortSpecs; private final IndexVersion indexCreatedVersion; @@ -158,34 +218,13 @@ public IndexSortConfig(IndexSettings indexSettings) { this.indexName = indexSettings.getIndex().getName(); this.indexMode = indexSettings.getMode(); - if (indexMode == IndexMode.TIME_SERIES) { - sortSpecs = TIME_SERIES_SORT; + var legacyDefaultSortSpecs = LegacyIndexSortConfigDefaults.getLegacyDefaultSortSpecs(indexSettings); + if (legacyDefaultSortSpecs != null) { + this.sortSpecs = legacyDefaultSortSpecs; return; } List fields = INDEX_SORT_FIELD_SETTING.get(settings); - if (indexMode == IndexMode.LOGSDB && INDEX_SORT_FIELD_SETTING.exists(settings) == false) { - if (INDEX_SORT_ORDER_SETTING.exists(settings)) { - var order = INDEX_SORT_ORDER_SETTING.get(settings); - throw new IllegalArgumentException("index.sort.fields:" + fields + " index.sort.order:" + order + ", size mismatch"); - } - if (INDEX_SORT_MODE_SETTING.exists(settings)) { - var mode = INDEX_SORT_MODE_SETTING.get(settings); - throw new IllegalArgumentException("index.sort.fields:" + fields + " index.sort.mode:" + mode + ", size mismatch"); - } - if (INDEX_SORT_MISSING_SETTING.exists(settings)) { - var missing = INDEX_SORT_MISSING_SETTING.get(settings); - throw new IllegalArgumentException("index.sort.fields:" + fields + " index.sort.missing:" + missing + ", size mismatch"); - } - var version = indexSettings.getIndexVersionCreated(); - if (version.onOrAfter(IndexVersions.LOGSB_OPTIONAL_SORTING_ON_HOST_NAME) - || version.between(IndexVersions.LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0)) { - sortSpecs = (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(settings)) ? HOSTNAME_TIMESTAMP_SORT : TIMESTAMP_SORT; - } else { - sortSpecs = HOSTNAME_TIMESTAMP_BWC_SORT; - } - return; - } sortSpecs = fields.stream().map(FieldSortSpec::new).toArray(FieldSortSpec[]::new); if (INDEX_SORT_ORDER_SETTING.exists(settings)) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index de561ec82a53b..49dd76163582d 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -186,6 +186,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion TSID_CREATED_DURING_ROUTING = def(9_037_0_00, Version.LUCENE_10_2_2); public static final IndexVersion UPGRADE_TO_LUCENE_10_3_0 = def(9_038_0_00, Version.LUCENE_10_3_0); public static final IndexVersion IGNORED_SOURCE_COALESCED_ENTRIES = def(9_039_0_00, Version.LUCENE_10_3_0); + public static final IndexVersion EXPLICIT_INDEX_SORTING_DEFAULTS = def(9_040_0_00, Version.LUCENE_10_3_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 4fe064b1af067..ed17cbd4b0c5e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -52,6 +52,7 @@ public class MapperFeatures implements FeatureSpecification { static final NodeFeature IGNORED_SOURCE_FIELDS_PER_ENTRY = new NodeFeature("mapper.ignored_source_fields_per_entry"); public static final NodeFeature MULTI_FIELD_UNICODE_OPTIMISATION_FIX = new NodeFeature("mapper.multi_field.unicode_optimisation_fix"); static final NodeFeature PATTERN_TEXT_RENAME = new NodeFeature("mapper.pattern_text_rename"); + static final NodeFeature PROVIDE_INDEX_SORT_SETTING_DEFAULTS = new NodeFeature("mapper.provide_index_sort_setting_defaults"); @Override public Set getTestFeatures() { @@ -89,7 +90,8 @@ public Set getTestFeatures() { IGNORED_SOURCE_FIELDS_PER_ENTRY, MULTI_FIELD_UNICODE_OPTIMISATION_FIX, MATCH_ONLY_TEXT_BLOCK_LOADER_FIX, - PATTERN_TEXT_RENAME + PATTERN_TEXT_RENAME, + PROVIDE_INDEX_SORT_SETTING_DEFAULTS ); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java index 313f8e43014d0..67c12713b26a0 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java @@ -39,7 +39,7 @@ import java.util.List; import java.util.function.IntSupplier; -import static org.elasticsearch.index.IndexSortConfig.TIME_SERIES_SORT; +import static org.elasticsearch.index.IndexSortConfig.LegacyIndexSortConfigDefaults.TIME_SERIES_SORT; /** * An IndexSearcher wrapper that executes the searches in time-series indices by traversing them by tsid and timestamp diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index ea18734190d7f..7b5ea2137300e 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -268,6 +268,10 @@ public static IndexMetadata newIndexMeta(String name, Settings indexSettings) { return IndexMetadata.builder(name).settings(indexSettings(IndexVersion.current(), 1, 1).put(indexSettings)).build(); } + public static IndexMetadata newIndexMeta(String name, Settings indexSettings, IndexVersion indexVersion) { + return IndexMetadata.builder(name).settings(indexSettings(indexVersion, 1, 1).put(indexSettings)).build(); + } + public void testUpdateDurability() { IndexMetadata metadata = newIndexMeta( "index", diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index 478996be08e93..be209c51f85c5 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.index.IndexVersionUtils; import java.util.ArrayList; import java.util.Collections; @@ -49,6 +50,10 @@ private static IndexSettings indexSettings(Settings settings) { return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); } + private static IndexSettings indexSettings(Settings settings, IndexVersion indexVersion) { + return new IndexSettings(newIndexMeta("test", settings, indexVersion), Settings.EMPTY); + } + public void testNoIndexSort() { IndexSettings indexSettings = indexSettings(Settings.EMPTY); assertFalse(indexSettings.getIndexSortConfig().hasIndexSort()); @@ -229,14 +234,15 @@ public void testSortMissingValueDateNanoFieldPre714() { } } - public void testTimeSeriesMode() { + public void testLegacyTimeSeriesMode() { IndexSettings indexSettings = indexSettings( Settings.builder() .put(IndexSettings.MODE.getKey(), "time_series") .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some_dimension") .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") - .build() + .build(), + IndexVersionUtils.getPreviousVersion(IndexVersions.EXPLICIT_INDEX_SORTING_DEFAULTS) ); Sort sort = buildIndexSort(indexSettings, TimeSeriesIdFieldMapper.FIELD_TYPE, new DateFieldMapper.DateFieldType("@timestamp")); assertThat(sort.getSort(), arrayWithSize(2)); @@ -244,19 +250,137 @@ public void testTimeSeriesMode() { assertThat(sort.getSort()[1].getField(), equalTo("@timestamp")); } - public void testTimeSeriesModeNoTimestamp() { + public void testLegacyTimeSeriesModeNoTimestamp() { IndexSettings indexSettings = indexSettings( Settings.builder() .put(IndexSettings.MODE.getKey(), "time_series") .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some_dimension") .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") - .build() + .build(), + IndexVersionUtils.getPreviousVersion(IndexVersions.EXPLICIT_INDEX_SORTING_DEFAULTS) ); Exception e = expectThrows(IllegalArgumentException.class, () -> buildIndexSort(indexSettings, TimeSeriesIdFieldMapper.FIELD_TYPE)); assertThat(e.getMessage(), equalTo("unknown index sort field:[@timestamp] required by [index.mode=time_series]")); } + public void testTimeSeriesMode() { + IndexSettings indexSettings = indexSettings( + Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some_dimension") + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") + .build() + ); + assertFalse(indexSettings.getIndexSortConfig().hasIndexSort()); + } + + public void testLegacyLogsdbIndexSortWithArrays() { + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "logsdb") + .putList("index.sort.field", "field1", "field2") + .putList("index.sort.order", "asc", "desc") + .putList("index.sort.missing", "_last", "_first") + .build(); + IndexSettings indexSettings = indexSettings( + settings, + IndexVersionUtils.getPreviousVersion(IndexVersions.EXPLICIT_INDEX_SORTING_DEFAULTS) + ); + IndexSortConfig config = indexSettings.getIndexSortConfig(); + assertTrue(config.hasIndexSort()); + assertThat(config.sortSpecs.length, equalTo(2)); + + assertThat(config.sortSpecs[0].field, equalTo("field1")); + assertThat(config.sortSpecs[1].field, equalTo("field2")); + assertThat(config.sortSpecs[0].order, equalTo(SortOrder.ASC)); + assertThat(config.sortSpecs[1].order, equalTo(SortOrder.DESC)); + assertThat(config.sortSpecs[0].missingValue, equalTo("_last")); + assertThat(config.sortSpecs[1].missingValue, equalTo("_first")); + assertNull(config.sortSpecs[0].mode); + assertNull(config.sortSpecs[1].mode); + } + + public void testLegacyLogsdbInvalidIndexSortOrder() { + final Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "logsdb") + .putList("index.sort.order", new String[] { "asc", "desc" }) + .build(); + IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + assertThat(exc.getMessage(), containsString("index.sort.field:[] index.sort.order:[asc, desc], size mismatch")); + } + + public void testLegacyLogsdbInvalidIndexSortMode() { + final Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "logsdb") + .putList("index.sort.mode", new String[] { "max" }) + .build(); + IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + assertThat(exc.getMessage(), containsString("index.sort.field:[] index.sort.mode:[MAX], size mismatch")); + } + + public void testLegacyLogsdbInvalidIndexSortMissing() { + final Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "logsdb") + .putList("index.sort.missing", new String[] { "_last", "_last" }) + .build(); + IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + assertThat(exc.getMessage(), containsString("index.sort.field:[] index.sort.missing:[_last, _last], size mismatch")); + } + + public void testLegacyLogsdbIndexSortWithHostname() { + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "logsdb") + .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true) + .build(); + IndexSettings indexSettings = indexSettings(settings, IndexVersions.UPGRADE_TO_LUCENE_10_2_2); + IndexSortConfig config = indexSettings.getIndexSortConfig(); + assertTrue(config.hasIndexSort()); + assertThat(config.sortSpecs.length, equalTo(2)); + + assertThat(config.sortSpecs[0].field, equalTo("host.name")); + assertThat(config.sortSpecs[1].field, equalTo("@timestamp")); + assertThat(config.sortSpecs[0].order, equalTo(SortOrder.ASC)); + assertThat(config.sortSpecs[1].order, equalTo(SortOrder.DESC)); + assertThat(config.sortSpecs[0].missingValue, equalTo("_last")); + assertNull(config.sortSpecs[1].missingValue); + assertThat(config.sortSpecs[0].mode, equalTo(MultiValueMode.MIN)); + assertNull(config.sortSpecs[1].mode); + } + + public void testLegacyLogsdbIndexSortTimestampOnly() { + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "logsdb") + .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), false) + .build(); + IndexSettings indexSettings = indexSettings(settings, IndexVersions.UPGRADE_TO_LUCENE_10_2_2); + IndexSortConfig config = indexSettings.getIndexSortConfig(); + assertTrue(config.hasIndexSort()); + assertThat(config.sortSpecs.length, equalTo(1)); + + assertThat(config.sortSpecs[0].field, equalTo("@timestamp")); + assertThat(config.sortSpecs[0].order, equalTo(SortOrder.DESC)); + assertNull(config.sortSpecs[0].missingValue); + assertNull(config.sortSpecs[0].mode); + } + + public void testLegacyLogsdbIndexSortTimestampBWC() { + Settings settings = Settings.builder().put(IndexSettings.MODE.getKey(), "logsdb").build(); + IndexSettings indexSettings = indexSettings(settings, IndexVersions.UPGRADE_TO_LUCENE_10_0_0); + IndexSortConfig config = indexSettings.getIndexSortConfig(); + assertTrue(config.hasIndexSort()); + assertThat(config.sortSpecs.length, equalTo(2)); + + assertThat(config.sortSpecs[0].field, equalTo("host.name")); + assertThat(config.sortSpecs[1].field, equalTo("@timestamp")); + assertNull(config.sortSpecs[0].order); + assertNull(config.sortSpecs[1].order); + assertNull(config.sortSpecs[0].missingValue); + assertNull(config.sortSpecs[1].missingValue); + assertNull(config.sortSpecs[0].mode); + assertNull(config.sortSpecs[1].mode); + } + private Sort buildIndexSort(IndexSettings indexSettings, MappedFieldType... mfts) { Map lookup = Maps.newMapWithExpectedSize(mfts.length); for (MappedFieldType mft : mfts) { diff --git a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java index 6f3077defa4a3..060e8d563d866 100644 --- a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java @@ -24,7 +24,11 @@ public void testLogsIndexModeSetting() { } public void testDefaultHostNameSortField() { - final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", buildSettings()); + final IndexMetadata metadata = IndexSettingsTests.newIndexMeta( + "test", + buildSettings(), + IndexVersionUtils.getPreviousVersion(IndexVersions.EXPLICIT_INDEX_SORTING_DEFAULTS) + ); assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB)); boolean sortOnHostName = randomBoolean(); final IndexSettings settings = new IndexSettings( @@ -35,7 +39,11 @@ public void testDefaultHostNameSortField() { } public void testDefaultHostNameSortFieldAndMapping() { - final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", buildSettings()); + final IndexMetadata metadata = IndexSettingsTests.newIndexMeta( + "test", + buildSettings(), + IndexVersionUtils.getPreviousVersion(IndexVersions.EXPLICIT_INDEX_SORTING_DEFAULTS) + ); assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB)); final IndexSettings settings = new IndexSettings( metadata, @@ -74,7 +82,7 @@ public void testDefaultHostNameSortWithOrder() { .build() ) ); - assertEquals("index.sort.fields:[] index.sort.order:[desc], size mismatch", exception.getMessage()); + assertEquals("index.sort.field:[] index.sort.order:[desc], size mismatch", exception.getMessage()); } public void testDefaultHostNameSortWithMode() { @@ -90,7 +98,7 @@ public void testDefaultHostNameSortWithMode() { .build() ) ); - assertEquals("index.sort.fields:[] index.sort.mode:[MAX], size mismatch", exception.getMessage()); + assertEquals("index.sort.field:[] index.sort.mode:[MAX], size mismatch", exception.getMessage()); } public void testDefaultHostNameSortWithMissing() { @@ -106,7 +114,7 @@ public void testDefaultHostNameSortWithMissing() { .build() ) ); - assertEquals("index.sort.fields:[] index.sort.missing:[_first], size mismatch", exception.getMessage()); + assertEquals("index.sort.field:[] index.sort.missing:[_first], size mismatch", exception.getMessage()); } public void testCustomSortField() { diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index 7cbc391ceda2f..bb0fa8326a565 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.time.Instant; +import java.util.List; import java.util.Map; import static org.elasticsearch.index.IndexSettings.TIME_SERIES_END_TIME; @@ -48,13 +49,6 @@ public void testPartitioned() { assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.routing_partition_size]")); } - public void testSortField() { - Settings s = Settings.builder().put(getSettings()).put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "a").build(); - IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); - Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); - assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.field]")); - } - public void testSortMode() { Settings s = Settings.builder().put(getSettings()).put(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_last").build(); IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); @@ -62,10 +56,43 @@ public void testSortMode() { assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.missing]")); } + public void testSortField() { + Settings s = Settings.builder().put(getSettings()).put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "a").build(); + var settingsProvider = new IndexMode.IndexModeSettingsProvider(); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> settingsProvider.provideAdditionalSettings( + "test", + null, + IndexMode.TIME_SERIES, + null, + null, + s, + List.of(), + null, + Settings.builder() + ) + ); + assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.field]")); + } + public void testSortOrder() { Settings s = Settings.builder().put(getSettings()).put(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "desc").build(); - IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", s); - Exception e = expectThrows(IllegalArgumentException.class, () -> new IndexSettings(metadata, Settings.EMPTY)); + var settingsProvider = new IndexMode.IndexModeSettingsProvider(); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> settingsProvider.provideAdditionalSettings( + "test", + null, + IndexMode.TIME_SERIES, + null, + null, + s, + List.of(), + null, + Settings.builder() + ) + ); assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.order]")); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java index 71fd3a4761cbe..f9cd2a6c29982 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java @@ -43,7 +43,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; -import static org.elasticsearch.index.IndexSortConfig.TIME_SERIES_SORT; +import static org.elasticsearch.index.IndexSortConfig.LegacyIndexSortConfigDefaults.TIME_SERIES_SORT; import static org.hamcrest.Matchers.greaterThan; public class TimeSeriesIndexSearcherTests extends ESTestCase { diff --git a/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java b/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java new file mode 100644 index 0000000000000..e17551fa90897 --- /dev/null +++ b/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java @@ -0,0 +1,274 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import org.apache.lucene.index.DirectoryReader; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; +import org.elasticsearch.action.datastreams.GetDataStreamAction; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexSortConfig; +import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.license.LicenseSettings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.XPackPlugin; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.function.Consumer; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; + +public class LogsdbSortConfigIT extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + return List.of(InternalSettingsPlugin.class, XPackPlugin.class, LogsDBPlugin.class, DataStreamsPlugin.class); + } + + @Override + protected Settings nodeSettings() { + return Settings.builder() + .put(super.nodeSettings()) + .put("cluster.logsdb.enabled", "true") + .put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial") + .build(); + } + + private record DocWithId(String id, String source) { + public String source() { + return source.replace("%id%", id); + } + } + + private int id = 0; + + private DocWithId doc(String source) { + return new DocWithId(Integer.toString(id++), source); + } + + public void testHostnameTimestampSortConfig() throws IOException { + final String dataStreamName = "test-logsdb-sort-hostname-timestamp"; + + final String MAPPING = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "keyword" + }, + "test_id": { + "type": "text" + } + } + } + } + """; + + final DocWithId[] orderedDocs = { + doc("{\"@timestamp\": \"2025-01-01T15:00:00\", \"host.name\": \"aaa\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T14:00:00\", \"host.name\": [\"aaa\", \"bbb\"], \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T12:30:00\", \"host.name\": [\"aaa\", \"bbb\"], \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T12:00:00\", \"host.name\": \"aaa\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T12:00:00\", \"host.name\": \"bbb\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T11:00:00\", \"host.name\": \"bbb\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T16:00:00\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T11:00:00\", \"test_id\": \"%id%\"}") }; + + createDataStream(dataStreamName, MAPPING); + + List shuffledDocs = shuffledList(Arrays.asList(orderedDocs)); + indexDocuments(dataStreamName, shuffledDocs); + + Index backingIndex = getBackingIndex(dataStreamName); + + var featureService = getInstanceFromNode(FeatureService.class); + if (featureService.getNodeFeatures().containsKey("mapper.provide_index_sort_setting_defaults")) { + assertSettings(backingIndex, settings -> { + assertThat(IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(settings), equalTo(List.of("host.name", "@timestamp"))); + assertThat(IndexSortConfig.INDEX_SORT_ORDER_SETTING.get(settings), equalTo(List.of(SortOrder.ASC, SortOrder.DESC))); + assertThat(IndexSortConfig.INDEX_SORT_MODE_SETTING.get(settings), equalTo(List.of(MultiValueMode.MIN, MultiValueMode.MAX))); + assertThat(IndexSortConfig.INDEX_SORT_MISSING_SETTING.get(settings), equalTo(List.of("_last", "_last"))); + }); + } + + assertOrder(backingIndex, orderedDocs); + } + + public void testTimestampOnlySortConfig() throws IOException { + final String dataStreamName = "test-logsdb-sort-timestamp-only"; + + final String MAPPING = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "type": "keyword" + }, + "test_id": { + "type": "text" + } + } + } + } + """; + + final DocWithId[] orderedDocs = { + doc("{\"@timestamp\": \"2025-01-01T15:00:00\", \"host\": \"aaa\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T14:00:00\", \"host\": [\"aaa\", \"bbb\"], \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T12:30:00\", \"host\": [\"aaa\", \"bbb\"], \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T12:15:00\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T12:00:00\", \"host\": \"aaa\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T11:45:00\", \"host\": \"bbb\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T11:15:00\", \"host\": \"bbb\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T11:00:00\", \"test_id\": \"%id%\"}") }; + + createDataStream(dataStreamName, MAPPING); + + List shuffledDocs = shuffledList(Arrays.asList(orderedDocs)); + indexDocuments(dataStreamName, shuffledDocs); + + Index backingIndex = getBackingIndex(dataStreamName); + + var featureService = getInstanceFromNode(FeatureService.class); + if (featureService.getNodeFeatures().containsKey("mapper.provide_index_sort_setting_defaults")) { + assertSettings(backingIndex, settings -> { + assertThat(IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(settings), equalTo(List.of("@timestamp"))); + assertThat(IndexSortConfig.INDEX_SORT_ORDER_SETTING.get(settings), equalTo(List.of(SortOrder.DESC))); + assertThat(IndexSortConfig.INDEX_SORT_MODE_SETTING.get(settings), equalTo(List.of(MultiValueMode.MAX))); + assertThat(IndexSortConfig.INDEX_SORT_MISSING_SETTING.get(settings), equalTo(List.of("_last"))); + }); + } + + assertOrder(backingIndex, orderedDocs); + } + + private void createDataStream(String dataStreamName, String mapping) throws IOException { + var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); + putTemplateRequest.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName + "*")) + .template(new Template(indexSettings(1, 0).put("index.mode", "logsdb").build(), new CompressedXContent(mapping), null)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest).actionGet(); + + client().execute( + CreateDataStreamAction.INSTANCE, + new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName) + ).actionGet(); + } + + private List indexDocuments(String dataStreamName, List docs) { + BulkRequest bulkRequest = new BulkRequest(dataStreamName); + for (DocWithId doc : docs) { + var indexRequest = new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.CREATE); + indexRequest.source(doc.source(), XContentType.JSON); + bulkRequest.add(indexRequest); + } + var bulkResponse = client().bulk(bulkRequest).actionGet(); + assertFalse(bulkResponse.hasFailures()); + + return Arrays.stream(bulkResponse.getItems()).map(item -> item.getId()).toList(); + } + + private Index getBackingIndex(String dataStreamName) { + var getDataStreamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName }); + getDataStreamRequest.indicesOptions( + IndicesOptions.builder() + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .includeHidden(true) + .matchOpen(true) + .matchClosed(true) + .allowEmptyExpressions(true) + .resolveAliases(false) + .build() + ) + .build() + ); + + var response = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet(); + var dataStreams = response.getDataStreams(); + assertThat(dataStreams, hasSize(1)); + + var indices = dataStreams.getFirst().getDataStream().getIndices(); + assertThat(indices, hasSize(1)); + + return indices.getFirst(); + } + + private void assertSettings(Index backingIndex, Consumer settingsTest) { + final GetSettingsResponse getSettingsResponse = indicesAdmin().getSettings( + new GetSettingsRequest(TEST_REQUEST_TIMEOUT).indices(backingIndex.getName()).includeDefaults(false) + ).actionGet(); + final Settings settings = getSettingsResponse.getIndexToSettings().get(backingIndex.getName()); + assertNotNull(settings); + settingsTest.accept(settings); + } + + private void assertOrder(Index backingIndex, DocWithId[] orderedDocs) throws IOException { + indicesAdmin().forceMerge(new ForceMergeRequest(backingIndex.getName()).maxNumSegments(1).flush(true)).actionGet(); + + IndexService indexService = getInstanceFromNode(IndicesService.class).indexServiceSafe(backingIndex); + assertThat(indexService.numberOfShards(), equalTo(1)); + IndexShard shard = indexService.getShard(0); + try (Engine.Searcher searcher = shard.acquireSearcher("test")) { + DirectoryReader directoryReader = searcher.getDirectoryReader(); + + assertThat(directoryReader.numDocs(), equalTo(orderedDocs.length)); + + var segments = directoryReader.leaves(); + assertThat(segments, hasSize(1)); + + var segment = segments.getFirst(); + var reader = segment.reader(); + var storedFields = reader.storedFields(); + + int expectedDocIdx = 0; + + for (int docId = 0; docId < reader.maxDoc(); docId++) { + String expectedId = orderedDocs[expectedDocIdx++].id; + String actualId = storedFields.document(docId).get("test_id"); + assertEquals(expectedId, actualId); + } + } + } + +} diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeEnabledRestTestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeEnabledRestTestIT.java index 63094852c3626..206fa1f8a9d75 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeEnabledRestTestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeEnabledRestTestIT.java @@ -257,9 +257,7 @@ public void testLogsAtSettingWithTimeSeriesOverride() throws IOException { "settings": { "index": { "routing_path": [ "hostname" ], - "mode": "time_series", - "sort.field": [], - "sort.order": [] + "mode": "time_series" } }, "mappings": { diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index ed9bd43262f0c..6d6d4b078a9f6 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -200,6 +200,33 @@ && matchesLogsPattern(dataStreamName)) { if (licenseService.allowPatternTextTemplating(isTemplateValidation) == false && mappingHints.maybeUsesPatternText) { additionalSettings.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true); } + + if (isLogsDB && isTemplateValidation == false && IndexSortConfig.INDEX_SORT_FIELD_SETTING.exists(settings) == false) { + IndexSortConfig.validateSortSettings(settings); + if (mappingHints.sortOnHostName) { + applyHostnameTimestampSort(additionalSettings); + } else { + applyTimestampSort(additionalSettings); + } + } + } + + private static void applyHostnameTimestampSort(Settings.Builder builder) { + builder.putList( + IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), + IndexMode.HOST_NAME, + DataStreamTimestampFieldMapper.DEFAULT_PATH + ); + builder.putList(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "asc", "desc"); + builder.putList(IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey(), "min", "max"); + builder.putList(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_last", "_last"); + } + + private static void applyTimestampSort(Settings.Builder builder) { + builder.putList(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataStreamTimestampFieldMapper.DEFAULT_PATH); + builder.putList(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "desc"); + builder.putList(IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey(), "max"); + builder.putList(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_last"); } record MappingHints(boolean hasSyntheticSourceUsage, boolean sortOnHostName, boolean addHostNameField, boolean maybeUsesPatternText) { diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index 92448722a01a1..cc11bb8d06079 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -33,6 +33,8 @@ import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.license.internal.XPackLicenseStatus; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.logsdb.patterntext.PatternTextFieldMapper; import org.junit.Before; @@ -784,10 +786,14 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() { settingsBuilder ); result = settingsBuilder.build(); - assertThat(result.size(), equalTo(3)); + assertThat(result.size(), equalTo(7)); assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertThat(IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(result), contains("host.name", "@timestamp")); + assertThat(IndexSortConfig.INDEX_SORT_ORDER_SETTING.get(result), contains(SortOrder.ASC, SortOrder.DESC)); + assertThat(IndexSortConfig.INDEX_SORT_MODE_SETTING.get(result), contains(MultiValueMode.MIN, MultiValueMode.MAX)); + assertThat(IndexSortConfig.INDEX_SORT_MISSING_SETTING.get(result), contains("_last", "_last")); assertThat(newMapperServiceCounter.get(), equalTo(4)); } @@ -869,11 +875,15 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch( settingsBuilder ); result = settingsBuilder.build(); - assertThat(result.size(), equalTo(4)); + assertThat(result.size(), equalTo(8)); assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertEquals(IndexMode.LOGSDB, IndexSettings.MODE.get(result)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertThat(IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(result), contains("host.name", "@timestamp")); + assertThat(IndexSortConfig.INDEX_SORT_ORDER_SETTING.get(result), contains(SortOrder.ASC, SortOrder.DESC)); + assertThat(IndexSortConfig.INDEX_SORT_MODE_SETTING.get(result), contains(MultiValueMode.MIN, MultiValueMode.MAX)); + assertThat(IndexSortConfig.INDEX_SORT_MISSING_SETTING.get(result), contains("_last", "_last")); settingsBuilder = builder(); provider.provideAdditionalSettings( @@ -992,6 +1002,10 @@ public void testPatternTextNotAllowedByLicense() throws IOException { .put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true) .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true) .put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true) + .putList(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name", "@timestamp") + .putList(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "asc", "desc") + .putList(IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey(), "min", "max") + .putList(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_last", "_last") .build(); for (String mapping : patternTextLicenceCheckedFieldMappings) { @@ -1007,6 +1021,10 @@ public void testPatternTextNotAllowedByLicenseAlreadyDisallowed() throws IOExcep .put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true) .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true) .put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true) + .putList(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name", "@timestamp") + .putList(IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), "asc", "desc") + .putList(IndexSortConfig.INDEX_SORT_MODE_SETTING.getKey(), "min", "max") + .putList(IndexSortConfig.INDEX_SORT_MISSING_SETTING.getKey(), "_last", "_last") .build(); assertEquals(expected, result); } @@ -1048,8 +1066,13 @@ public void testSortAndHostNoHost() throws Exception { } """; Settings result = generateLogsdbSettings(settings, mappings); + assertThat(result.size(), equalTo(6)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertThat(IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(result), contains("host.name", "@timestamp")); + assertThat(IndexSortConfig.INDEX_SORT_ORDER_SETTING.get(result), contains(SortOrder.ASC, SortOrder.DESC)); + assertThat(IndexSortConfig.INDEX_SORT_MODE_SETTING.get(result), contains(MultiValueMode.MIN, MultiValueMode.MAX)); + assertThat(IndexSortConfig.INDEX_SORT_MISSING_SETTING.get(result), contains("_last", "_last")); assertEquals(1, newMapperServiceCounter.get()); } @@ -1067,7 +1090,11 @@ public void testSortAndHostNoHostOldNode() throws Exception { } """; Settings result = generateLogsdbSettings(settings, mappings, Version.V_8_17_0); - assertTrue(result.isEmpty()); + assertThat(result.size(), equalTo(4)); + assertThat(IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(result), contains("host.name", "@timestamp")); + assertThat(IndexSortConfig.INDEX_SORT_ORDER_SETTING.get(result), contains(SortOrder.ASC, SortOrder.DESC)); + assertThat(IndexSortConfig.INDEX_SORT_MODE_SETTING.get(result), contains(MultiValueMode.MIN, MultiValueMode.MAX)); + assertThat(IndexSortConfig.INDEX_SORT_MISSING_SETTING.get(result), contains("_last", "_last")); } public void testSortAndHostNameKeyword() throws Exception { diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/80_index_sort_defaults.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/80_index_sort_defaults.yml new file mode 100644 index 0000000000000..a0756d79cbfdf --- /dev/null +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/80_index_sort_defaults.yml @@ -0,0 +1,510 @@ +setup: + - requires: + cluster_features: [ "mapper.provide_index_sort_setting_defaults" ] + reason: "testing index sort setting defaults" + - requires: + test_runner_features: [ "allowed_warnings" ] + +--- +"test standard mode": + - do: + indices.create: + index: test-standard + body: + mappings: + properties: + "@timestamp": + type: date + host.name: + type: keyword + + - do: + indices.get_settings: + index: test-standard + include_defaults: true + + - match: { test-standard.defaults.index.mode: standard } + - match: { test-standard.defaults.index.sort.field: [] } + - match: { test-standard.defaults.index.sort.order: [] } + - match: { test-standard.defaults.index.sort.mode: [] } + - match: { test-standard.defaults.index.sort.missing: [] } + +--- +create logsdb data stream with host.name as keyword: + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host.name: + type: "keyword" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: "true" } + - match: { .$backing_index.settings.index.sort.field: [ "host.name", "@timestamp" ] } + - match: { .$backing_index.settings.index.sort.order: [ "asc", "desc" ] } + - match: { .$backing_index.settings.index.sort.mode: [ "min", "max" ] } + - match: { .$backing_index.settings.index.sort.missing: [ "_last", "_last" ] } + +--- +create logsdb data stream with host.name as keyword and timestamp as date: + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host.name: + type: "keyword" + "@timestamp": + type: "date" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: "true" } + - match: { .$backing_index.settings.index.sort.field: [ "host.name", "@timestamp" ] } + - match: { .$backing_index.settings.index.sort.order: [ "asc", "desc" ] } + - match: { .$backing_index.settings.index.sort.mode: [ "min", "max" ] } + - match: { .$backing_index.settings.index.sort.missing: [ "_last", "_last" ] } + +--- +create logsdb data stream with host.name as integer and timestamp as date: + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host.name: + type: "integer" + "@timestamp": + type: "date" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: "true" } + - match: { .$backing_index.settings.index.sort.field: [ "host.name", "@timestamp" ] } + - match: { .$backing_index.settings.index.sort.order: [ "asc", "desc" ] } + - match: { .$backing_index.settings.index.sort.mode: [ "min", "max" ] } + - match: { .$backing_index.settings.index.sort.missing: [ "_last", "_last" ] } +--- + +create logsdb data stream with no host.name and timestamp as date: + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + "@timestamp": + type: "date" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: [ "logsdb" ] + data_stream: { } + composed_of: [ "logsdb-mappings" ] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: "true" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: "true" } + - match: { .$backing_index.settings.index.sort.field: [ "host.name", "@timestamp" ] } + - match: { .$backing_index.settings.index.sort.order: [ "asc", "desc" ] } + - match: { .$backing_index.settings.index.sort.mode: [ "min", "max" ] } + - match: { .$backing_index.settings.index.sort.missing: [ "_last", "_last" ] } + +--- +create logsdb data stream with host as keyword and timestamp as date: + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host: + type: "keyword" + "@timestamp": + type: "date" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: [ "logsdb" ] + data_stream: { } + composed_of: [ "logsdb-mappings" ] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: null } + - match: { .$backing_index.defaults.index.logsdb.sort_on_host_name: "false" } + - match: { .$backing_index.settings.index.sort.field: [ "@timestamp" ] } + - match: { .$backing_index.settings.index.sort.order: [ "desc" ] } + - match: { .$backing_index.settings.index.sort.mode: [ "max" ] } + - match: { .$backing_index.settings.index.sort.missing: [ "_last" ] } + +--- +create logsdb data stream with host as keyword: + - requires: + test_runner_features: [ "allowed_warnings" ] + + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host: + type: "keyword" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: null } + - match: { .$backing_index.defaults.index.logsdb.sort_on_host_name: "false" } + - match: { .$backing_index.settings.index.sort.field: [ "@timestamp" ] } + - match: { .$backing_index.settings.index.sort.order: [ "desc" ] } + - match: { .$backing_index.settings.index.sort.mode: [ "max" ] } + - match: { .$backing_index.settings.index.sort.missing: [ "_last" ] } +--- +create logsdb data stream with host as text and multi fields: + - requires: + test_runner_features: [ "allowed_warnings" ] + + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host: + type: "text" + fields: + keyword: + ignore_above: 256 + type: "keyword" + "@timestamp": + type: "date" + format: "strict_date_optional_time" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: null } + - match: { .$backing_index.defaults.index.logsdb.sort_on_host_name: "false" } + - match: { .$backing_index.settings.index.sort.field: [ "@timestamp" ] } + - match: { .$backing_index.settings.index.sort.order: [ "desc" ] } + - match: { .$backing_index.settings.index.sort.mode: [ "max" ] } + - match: { .$backing_index.settings.index.sort.missing: [ "_last" ] } + +--- +create logsdb data stream with host as text: + - requires: + test_runner_features: [ "allowed_warnings" ] + + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host: + type: "text" + "@timestamp": + type: "date" + format: "strict_date_optional_time" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: null } + - match: { .$backing_index.defaults.index.logsdb.sort_on_host_name: "false" } + - match: { .$backing_index.settings.index.sort.field: [ "@timestamp" ] } + - match: { .$backing_index.settings.index.sort.order: [ "desc" ] } + - match: { .$backing_index.settings.index.sort.mode: [ "max" ] } + - match: { .$backing_index.settings.index.sort.missing: [ "_last" ] } +--- +create logsdb data stream with host as text and name as double: + - requires: + test_runner_features: [ "allowed_warnings" ] + + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host: + type: "text" + fields: + name: + type: "double" + "@timestamp": + type: "date" + format: "strict_date_optional_time" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: "true" } + - match: { .$backing_index.settings.index.sort.field: [ "host.name", "@timestamp" ] } + - match: { .$backing_index.settings.index.sort.order: [ "asc", "desc" ] } + - match: { .$backing_index.settings.index.sort.mode: [ "min", "max" ] } + - match: { .$backing_index.settings.index.sort.missing: [ "_last", "_last" ] }