diff --git a/docs/changelog/135886.yaml b/docs/changelog/135886.yaml new file mode 100644 index 0000000000000..c2fe3b1a6f10a --- /dev/null +++ b/docs/changelog/135886.yaml @@ -0,0 +1,6 @@ +pr: 135886 +summary: Provide defaults for index sort settings +area: Mapping +type: bug +issues: + - 129062 diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index a048fb07edeef..04e106d75ce4c 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -98,6 +98,9 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.skipTest("update/100_synthetic_source/stored text", "synthetic recovery source means _recovery_source field will not be present") task.skipTest("logsdb/10_settings/start time not allowed in logs mode", "we don't validate for index_mode=tsdb when setting start_date/end_date anymore") task.skipTest("logsdb/10_settings/end time not allowed in logs mode", "we don't validate for index_mode=tsdb when setting start_date/end_date anymore") + task.skipTest("logsdb/10_settings/override sort mode settings", "we changed the error message") + task.skipTest("logsdb/10_settings/override sort missing settings", "we changed the error message") + task.skipTest("logsdb/10_settings/override sort order settings", "we changed the error message") task.skipTest("tsdb/10_settings/set start_time and end_time without timeseries mode", "we don't validate for index_mode=tsdb when setting start_date/end_date anymore") task.skipTest("tsdb/10_settings/set start_time, end_time and routing_path via put settings api without time_series mode", "we don't validate for index_mode=tsdb when setting start_date/end_date anymore") // Expected deprecation warning to compat yaml tests: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml index f76b4eb7bc32f..30313fda108f9 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml @@ -276,7 +276,7 @@ override sort order settings: type: text - match: { error.type: "illegal_argument_exception" } - - match: { error.reason: "index.sort.fields:[] index.sort.order:[asc, asc], size mismatch" } + - match: { error.reason: "setting [index.sort.order] requires [index.sort.field] to be configured" } --- override sort missing settings: @@ -312,7 +312,7 @@ override sort missing settings: type: text - match: { error.type: "illegal_argument_exception" } - - match: { error.reason: "index.sort.fields:[] index.sort.missing:[_last, _first], size mismatch" } + - match: { error.reason: "setting [index.sort.missing] requires [index.sort.field] to be configured" } --- override sort mode settings: @@ -348,7 +348,7 @@ override sort mode settings: type: text - match: { error.type: "illegal_argument_exception" } - - match: { error.reason: "index.sort.fields:[] index.sort.mode:[MAX, MAX], size mismatch" } + - match: { error.reason: "setting [index.sort.mode] requires [index.sort.field] to be configured" } --- override sort field using nested field type in sorting: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml index c54556e99bee9..8aa2d04b5434d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml @@ -576,3 +576,29 @@ check time_series empty time bound value: "@timestamp": "2021-09-26T03:09:52.123456789Z", "metricset": "pod" } + +--- +default sort field: + - requires: + cluster_features: [ "mapper.provide_index_sort_setting_defaults" ] + reason: "testing index sort setting defaults" + + - do: + indices.create: + index: test_index + body: + settings: + index: + mode: time_series + routing_path: foo + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + + - do: + indices.get_settings: + index: test_index + include_defaults: true + - match: { .test_index.settings.index.mode: time_series } + - match: { .test_index.defaults.index.sort.field: [ "_tsid", "@timestamp" ] } + - match: { .test_index.defaults.index.sort.order: [ "asc", "desc" ] } diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index 1ccaeee626fc9..35e6de2a6f249 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -1821,11 +1821,19 @@ public static Setting> stringListSetting(String key, Property... pr } public static Setting> stringListSetting(String key, List defValue, Property... properties) { - return new ListSetting<>(key, null, s -> defValue, s -> parseableStringToList(s, Function.identity()), v -> {}, properties) { + return stringListSettingWithDefaultProvider(key, s -> defValue, properties); + } + + public static Setting> stringListSettingWithDefaultProvider( + String key, + Function> defValueProvider, + Property... properties + ) { + return new ListSetting<>(key, null, defValueProvider, s -> parseableStringToList(s, Function.identity()), v -> {}, properties) { @Override public List get(Settings settings) { checkDeprecation(settings); - return settings.getAsList(getKey(), defValue); + return settings.getAsList(getKey(), defValueProvider.apply(settings)); } }; } @@ -1852,6 +1860,15 @@ public static Setting> listSetting( return listSetting(key, null, singleValueParser, s -> defaultStringValue, properties); } + public static Setting> listSetting( + final String key, + final Function> defaultStringValueProvider, + final Function singleValueParser, + final Property... properties + ) { + return listSetting(key, null, singleValueParser, defaultStringValueProvider, properties); + } + public static Setting> listSetting( final String key, final List defaultStringValue, @@ -1981,7 +1998,7 @@ public void diff(Settings.Builder builder, Settings source, Settings defaultSett if (exists(source) == false) { List asList = defaultSettings.getAsList(getKey(), null); if (asList == null) { - builder.putList(getKey(), defaultStringValue.apply(defaultSettings)); + builder.putList(getKey(), defaultStringValue.apply(source)); } else { builder.putList(getKey(), asList); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 46c790a4b5b70..10e604126f934 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -140,8 +140,11 @@ void validateWithOtherSettings(Map, Object> settings) { if (settings.get(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING) != Integer.valueOf(1)) { throw new IllegalArgumentException(error(IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING)); } + + var settingsWithIndexMode = Settings.builder().put(IndexSettings.MODE.getKey(), getName()).build(); + for (Setting unsupported : TIME_SERIES_UNSUPPORTED) { - if (false == Objects.equals(unsupported.getDefault(Settings.EMPTY), settings.get(unsupported))) { + if (false == Objects.equals(unsupported.getDefault(settingsWithIndexMode), settings.get(unsupported))) { throw new IllegalArgumentException(error(unsupported)); } } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index a3ed94416b7e8..8235b94b4c3a9 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -13,6 +13,7 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.SortedSetSortField; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Setting; @@ -25,9 +26,10 @@ import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.SortOrder; -import java.util.Collections; +import java.util.Arrays; import java.util.EnumSet; import java.util.List; +import java.util.Locale; import java.util.Objects; import java.util.function.BiFunction; import java.util.function.Function; @@ -59,8 +61,9 @@ public final class IndexSortConfig { /** * The list of field names */ - public static final Setting> INDEX_SORT_FIELD_SETTING = Setting.stringListSetting( + public static final Setting> INDEX_SORT_FIELD_SETTING = Setting.stringListSettingWithDefaultProvider( "index.sort.field", + IndexSortConfigDefaults::getDefaultSortFields, Setting.Property.IndexScope, Setting.Property.Final, Setting.Property.ServerlessPublic @@ -71,7 +74,7 @@ public final class IndexSortConfig { */ public static final Setting> INDEX_SORT_ORDER_SETTING = Setting.listSetting( "index.sort.order", - Collections.emptyList(), + IndexSortConfigDefaults::getDefaultSortOrder, IndexSortConfig::parseOrderMode, Setting.Property.IndexScope, Setting.Property.Final, @@ -83,7 +86,7 @@ public final class IndexSortConfig { */ public static final Setting> INDEX_SORT_MODE_SETTING = Setting.listSetting( "index.sort.mode", - Collections.emptyList(), + IndexSortConfigDefaults::getDefaultSortMode, IndexSortConfig::parseMultiValueMode, Setting.Property.IndexScope, Setting.Property.Final, @@ -95,30 +98,113 @@ public final class IndexSortConfig { */ public static final Setting> INDEX_SORT_MISSING_SETTING = Setting.listSetting( "index.sort.missing", - Collections.emptyList(), + IndexSortConfigDefaults::getDefaultSortMissing, IndexSortConfig::validateMissingValue, Setting.Property.IndexScope, Setting.Property.Final, Setting.Property.ServerlessPublic ); - public static final FieldSortSpec[] TIME_SERIES_SORT, TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_BWC_SORT; - static { - FieldSortSpec timeStampSpec = new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH); - timeStampSpec.order = SortOrder.DESC; - TIME_SERIES_SORT = new FieldSortSpec[] { new FieldSortSpec(TimeSeriesIdFieldMapper.NAME), timeStampSpec }; - TIMESTAMP_SORT = new FieldSortSpec[] { timeStampSpec }; - - FieldSortSpec hostnameSpec = new FieldSortSpec(IndexMode.HOST_NAME); - hostnameSpec.order = SortOrder.ASC; - hostnameSpec.missingValue = "_last"; - hostnameSpec.mode = MultiValueMode.MIN; - HOSTNAME_TIMESTAMP_SORT = new FieldSortSpec[] { hostnameSpec, timeStampSpec }; - - // Older indexes use ascending ordering for host name and timestamp. - HOSTNAME_TIMESTAMP_BWC_SORT = new FieldSortSpec[] { - new FieldSortSpec(IndexMode.HOST_NAME), - new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH) }; + public static class IndexSortConfigDefaults { + public static final FieldSortSpec[] TIME_SERIES_SORT, TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_BWC_SORT; + + static { + FieldSortSpec timeStampSpec = new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH); + timeStampSpec.order = SortOrder.DESC; + TIME_SERIES_SORT = new FieldSortSpec[] { new FieldSortSpec(TimeSeriesIdFieldMapper.NAME), timeStampSpec }; + TIMESTAMP_SORT = new FieldSortSpec[] { timeStampSpec }; + + FieldSortSpec hostnameSpec = new FieldSortSpec(IndexMode.HOST_NAME); + hostnameSpec.order = SortOrder.ASC; + hostnameSpec.missingValue = "_last"; + hostnameSpec.mode = MultiValueMode.MIN; + HOSTNAME_TIMESTAMP_SORT = new FieldSortSpec[] { hostnameSpec, timeStampSpec }; + + // Older indexes use ascending ordering for host name and timestamp. + HOSTNAME_TIMESTAMP_BWC_SORT = new FieldSortSpec[] { + new FieldSortSpec(IndexMode.HOST_NAME), + new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH) }; + } + + public static FieldSortSpec[] getDefaultSortSpecs(Settings settings) { + if (settings.isEmpty()) { + return new FieldSortSpec[0]; + } + + // Can't use IndexSettings.MODE.get(settings) here because the validation logic for IndexSettings.MODE uses the default value + // of index.sort.*, which causes infinite recursion (since we're already in the default value provider for those settings). + // So we need to get the mode while bypassing the validation. + String indexMode = settings.get(IndexSettings.MODE.getKey()); + if (indexMode != null) { + indexMode = indexMode.toLowerCase(Locale.ROOT); + } + + if (IndexMode.TIME_SERIES.getName().equals(indexMode)) { + return TIME_SERIES_SORT; + } else if (IndexMode.LOGSDB.getName().equals(indexMode)) { + var version = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings); + if (version.onOrAfter(IndexVersions.LOGSB_OPTIONAL_SORTING_ON_HOST_NAME) + || version.between( + IndexVersions.LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT, + IndexVersions.UPGRADE_TO_LUCENE_10_0_0 + )) { + return (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(settings)) ? HOSTNAME_TIMESTAMP_SORT : TIMESTAMP_SORT; + } else { + return HOSTNAME_TIMESTAMP_BWC_SORT; + } + } + + return new FieldSortSpec[0]; + } + + public static FieldSortSpec[] getSortSpecs(Settings settings) { + if (INDEX_SORT_FIELD_SETTING.exists(settings) == false) { + return IndexSortConfigDefaults.getDefaultSortSpecs(settings); + } + + List fields = INDEX_SORT_FIELD_SETTING.get(settings); + FieldSortSpec[] sortSpecs = fields.stream().map(FieldSortSpec::new).toArray(FieldSortSpec[]::new); + + // Need to populate `order` because the default value of `mode` depends on it + if (INDEX_SORT_ORDER_SETTING.exists(settings)) { + List orders = INDEX_SORT_ORDER_SETTING.get(settings); + for (int i = 0; i < sortSpecs.length; i++) { + sortSpecs[i].order = orders.get(i); + } + } + + return sortSpecs; + } + + public static List getDefaultSortFields(Settings settings) { + return Arrays.stream(getDefaultSortSpecs(settings)).map(sortSpec -> sortSpec.field).toList(); + } + + public static List getDefaultSortOrder(Settings settings) { + return Arrays.stream(getSortSpecs(settings)) + .map(sortSpec -> sortSpec.order != null ? sortSpec.order : SortOrder.ASC) + .map(Enum::toString) + .toList(); + } + + public static List getDefaultSortMode(Settings settings) { + return Arrays.stream(getSortSpecs(settings)).map(sortSpec -> { + if (sortSpec.mode != null) { + return sortSpec.mode; + } else if (sortSpec.order == SortOrder.DESC) { + return MultiValueMode.MAX; + } else { + return MultiValueMode.MIN; + } + }).map(order -> order.toString().toLowerCase(Locale.ROOT)).toList(); + } + + public static List getDefaultSortMissing(Settings settings) { + // _last is the default per IndexFieldData.XFieldComparatorSource.Nested#sortMissingLast + return Arrays.stream(getSortSpecs(settings)) + .map(sortSpec -> sortSpec.missingValue != null ? sortSpec.missingValue : "_last") + .toList(); + } } private static String validateMissingValue(String missing) { @@ -146,6 +232,35 @@ private static MultiValueMode parseMultiValueMode(String value) { return mode; } + private static void checkSizeMismatch(String firstKey, List first, String secondKey, List second) { + if (first.size() != second.size()) { + throw new IllegalArgumentException(firstKey + ":" + first + " " + secondKey + ":" + second + ", size mismatch"); + } + } + + private static void validateSortSettings(Settings settings) { + if (INDEX_SORT_FIELD_SETTING.exists(settings) == false) { + for (Setting setting : new Setting[] { INDEX_SORT_ORDER_SETTING, INDEX_SORT_MODE_SETTING, INDEX_SORT_MISSING_SETTING }) { + if (setting.exists(settings)) { + throw new IllegalArgumentException( + "setting [" + setting.getKey() + "] requires [" + INDEX_SORT_FIELD_SETTING.getKey() + "] to be configured" + ); + } + } + } + + List fields = INDEX_SORT_FIELD_SETTING.get(settings); + + var order = INDEX_SORT_ORDER_SETTING.get(settings); + checkSizeMismatch(INDEX_SORT_FIELD_SETTING.getKey(), fields, INDEX_SORT_ORDER_SETTING.getKey(), order); + + var mode = INDEX_SORT_MODE_SETTING.get(settings); + checkSizeMismatch(INDEX_SORT_FIELD_SETTING.getKey(), fields, INDEX_SORT_MODE_SETTING.getKey(), mode); + + var missing = INDEX_SORT_MISSING_SETTING.get(settings); + checkSizeMismatch(INDEX_SORT_FIELD_SETTING.getKey(), fields, INDEX_SORT_MISSING_SETTING.getKey(), missing); + } + // visible for tests final FieldSortSpec[] sortSpecs; private final IndexVersion indexCreatedVersion; @@ -158,66 +273,24 @@ public IndexSortConfig(IndexSettings indexSettings) { this.indexName = indexSettings.getIndex().getName(); this.indexMode = indexSettings.getMode(); - if (indexMode == IndexMode.TIME_SERIES) { - sortSpecs = TIME_SERIES_SORT; - return; - } + validateSortSettings(settings); List fields = INDEX_SORT_FIELD_SETTING.get(settings); - if (indexMode == IndexMode.LOGSDB && INDEX_SORT_FIELD_SETTING.exists(settings) == false) { - if (INDEX_SORT_ORDER_SETTING.exists(settings)) { - var order = INDEX_SORT_ORDER_SETTING.get(settings); - throw new IllegalArgumentException("index.sort.fields:" + fields + " index.sort.order:" + order + ", size mismatch"); - } - if (INDEX_SORT_MODE_SETTING.exists(settings)) { - var mode = INDEX_SORT_MODE_SETTING.get(settings); - throw new IllegalArgumentException("index.sort.fields:" + fields + " index.sort.mode:" + mode + ", size mismatch"); - } - if (INDEX_SORT_MISSING_SETTING.exists(settings)) { - var missing = INDEX_SORT_MISSING_SETTING.get(settings); - throw new IllegalArgumentException("index.sort.fields:" + fields + " index.sort.missing:" + missing + ", size mismatch"); - } - var version = indexSettings.getIndexVersionCreated(); - if (version.onOrAfter(IndexVersions.LOGSB_OPTIONAL_SORTING_ON_HOST_NAME) - || version.between(IndexVersions.LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0)) { - sortSpecs = (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(settings)) ? HOSTNAME_TIMESTAMP_SORT : TIMESTAMP_SORT; - } else { - sortSpecs = HOSTNAME_TIMESTAMP_BWC_SORT; - } - return; - } sortSpecs = fields.stream().map(FieldSortSpec::new).toArray(FieldSortSpec[]::new); - if (INDEX_SORT_ORDER_SETTING.exists(settings)) { - List orders = INDEX_SORT_ORDER_SETTING.get(settings); - if (orders.size() != sortSpecs.length) { - throw new IllegalArgumentException("index.sort.field:" + fields + " index.sort.order:" + orders + ", size mismatch"); - } - for (int i = 0; i < sortSpecs.length; i++) { - sortSpecs[i].order = orders.get(i); - } + List orders = INDEX_SORT_ORDER_SETTING.get(settings); + for (int i = 0; i < sortSpecs.length; i++) { + sortSpecs[i].order = orders.get(i); } - if (INDEX_SORT_MODE_SETTING.exists(settings)) { - List modes = INDEX_SORT_MODE_SETTING.get(settings); - if (modes.size() != sortSpecs.length) { - throw new IllegalArgumentException("index.sort.field:" + fields + " index.sort.mode:" + modes + ", size mismatch"); - } - for (int i = 0; i < sortSpecs.length; i++) { - sortSpecs[i].mode = modes.get(i); - } + List modes = INDEX_SORT_MODE_SETTING.get(settings); + for (int i = 0; i < sortSpecs.length; i++) { + sortSpecs[i].mode = modes.get(i); } - if (INDEX_SORT_MISSING_SETTING.exists(settings)) { - List missingValues = INDEX_SORT_MISSING_SETTING.get(settings); - if (missingValues.size() != sortSpecs.length) { - throw new IllegalArgumentException( - "index.sort.field:" + fields + " index.sort.missing:" + missingValues + ", size mismatch" - ); - } - for (int i = 0; i < sortSpecs.length; i++) { - sortSpecs[i].missingValue = missingValues.get(i); - } + List missingValues = INDEX_SORT_MISSING_SETTING.get(settings); + for (int i = 0; i < sortSpecs.length; i++) { + sortSpecs[i].missingValue = missingValues.get(i); } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java index 86d962f55a558..4e7cf8ead2d7b 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java @@ -148,7 +148,7 @@ public SortField sortField( SortField sortField = sortField(missingValue, sortMode, nested, reverse); if (getNumericType() == NumericType.DATE_NANOSECONDS && indexCreatedVersion.before(IndexVersions.V_7_14_0) - && missingValue == null + && missingValue.equals("_last") && Long.valueOf(0L).equals(sortField.getMissingValue())) { // 7.14 changed the default missing value of sort on date_nanos, from Long.MIN_VALUE // to 0L - for compatibility we require to a missing value of MIN_VALUE to allow to diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 914acd791e0ee..1853364738822 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -53,6 +53,7 @@ public class MapperFeatures implements FeatureSpecification { public static final NodeFeature MULTI_FIELD_UNICODE_OPTIMISATION_FIX = new NodeFeature("mapper.multi_field.unicode_optimisation_fix"); static final NodeFeature PATTERN_TEXT_RENAME = new NodeFeature("mapper.pattern_text_rename"); public static final NodeFeature DISKBBQ_ON_DISK_RESCORING = new NodeFeature("mapper.vectors.diskbbq_on_disk_rescoring"); + static final NodeFeature PROVIDE_INDEX_SORT_SETTING_DEFAULTS = new NodeFeature("mapper.provide_index_sort_setting_defaults"); @Override public Set getTestFeatures() { @@ -91,7 +92,8 @@ public Set getTestFeatures() { MULTI_FIELD_UNICODE_OPTIMISATION_FIX, MATCH_ONLY_TEXT_BLOCK_LOADER_FIX, PATTERN_TEXT_RENAME, - DISKBBQ_ON_DISK_RESCORING + DISKBBQ_ON_DISK_RESCORING, + PROVIDE_INDEX_SORT_SETTING_DEFAULTS ); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java index 313f8e43014d0..02947a0ead663 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcher.java @@ -39,7 +39,7 @@ import java.util.List; import java.util.function.IntSupplier; -import static org.elasticsearch.index.IndexSortConfig.TIME_SERIES_SORT; +import static org.elasticsearch.index.IndexSortConfig.IndexSortConfigDefaults.TIME_SERIES_SORT; /** * An IndexSearcher wrapper that executes the searches in time-series indices by traversing them by tsid and timestamp diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index ea18734190d7f..7b5ea2137300e 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -268,6 +268,10 @@ public static IndexMetadata newIndexMeta(String name, Settings indexSettings) { return IndexMetadata.builder(name).settings(indexSettings(IndexVersion.current(), 1, 1).put(indexSettings)).build(); } + public static IndexMetadata newIndexMeta(String name, Settings indexSettings, IndexVersion indexVersion) { + return IndexMetadata.builder(name).settings(indexSettings(indexVersion, 1, 1).put(indexSettings)).build(); + } + public void testUpdateDurability() { IndexMetadata metadata = newIndexMeta( "index", diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index 478996be08e93..0c789db7a412f 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.index.IndexVersionUtils; import java.util.ArrayList; import java.util.Collections; @@ -49,6 +50,10 @@ private static IndexSettings indexSettings(Settings settings) { return new IndexSettings(newIndexMeta("test", settings), Settings.EMPTY); } + private static IndexSettings indexSettings(Settings settings, IndexVersion indexVersion) { + return new IndexSettings(newIndexMeta("test", settings, indexVersion), Settings.EMPTY); + } + public void testNoIndexSort() { IndexSettings indexSettings = indexSettings(Settings.EMPTY); assertFalse(indexSettings.getIndexSortConfig().hasIndexSort()); @@ -89,8 +94,8 @@ public void testIndexSortWithArrays() { assertThat(config.sortSpecs[1].order, equalTo(SortOrder.DESC)); assertThat(config.sortSpecs[0].missingValue, equalTo("_last")); assertThat(config.sortSpecs[1].missingValue, equalTo("_first")); - assertNull(config.sortSpecs[0].mode); - assertNull(config.sortSpecs[1].mode); + assertThat(config.sortSpecs[0].mode, equalTo(MultiValueMode.MIN)); + assertThat(config.sortSpecs[1].mode, equalTo(MultiValueMode.MAX)); } public void testInvalidIndexSort() { @@ -257,6 +262,111 @@ public void testTimeSeriesModeNoTimestamp() { assertThat(e.getMessage(), equalTo("unknown index sort field:[@timestamp] required by [index.mode=time_series]")); } + public void testLogsdbIndexSortWithArrays() { + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "logsdb") + .putList("index.sort.field", "field1", "field2") + .putList("index.sort.order", "asc", "desc") + .putList("index.sort.missing", "_last", "_first") + .build(); + IndexSettings indexSettings = indexSettings(settings); + IndexSortConfig config = indexSettings.getIndexSortConfig(); + assertTrue(config.hasIndexSort()); + assertThat(config.sortSpecs.length, equalTo(2)); + + assertThat(config.sortSpecs[0].field, equalTo("field1")); + assertThat(config.sortSpecs[1].field, equalTo("field2")); + assertThat(config.sortSpecs[0].order, equalTo(SortOrder.ASC)); + assertThat(config.sortSpecs[1].order, equalTo(SortOrder.DESC)); + assertThat(config.sortSpecs[0].missingValue, equalTo("_last")); + assertThat(config.sortSpecs[1].missingValue, equalTo("_first")); + assertThat(config.sortSpecs[0].mode, equalTo(MultiValueMode.MIN)); + assertThat(config.sortSpecs[1].mode, equalTo(MultiValueMode.MAX)); + } + + public void testLogsdbInvalidIndexSortOrder() { + final Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "logsdb") + .putList("index.sort.order", new String[] { "asc", "desc" }) + .build(); + IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + assertThat(exc.getMessage(), containsString("setting [index.sort.order] requires [index.sort.field] to be configured")); + } + + public void testLogsdbInvalidIndexSortMode() { + final Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "logsdb") + .putList("index.sort.mode", new String[] { "max" }) + .build(); + IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + assertThat(exc.getMessage(), containsString("setting [index.sort.mode] requires [index.sort.field] to be configured")); + } + + public void testLogsdbInvalidIndexSortMissing() { + final Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "logsdb") + .putList("index.sort.missing", new String[] { "_last", "_last" }) + .build(); + IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> indexSettings(settings)); + assertThat(exc.getMessage(), containsString("setting [index.sort.missing] requires [index.sort.field] to be configured")); + } + + public void testLogsdbIndexSortWithHostname() { + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "logsdb") + .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true) + .build(); + IndexSettings indexSettings = indexSettings(settings); + IndexSortConfig config = indexSettings.getIndexSortConfig(); + assertTrue(config.hasIndexSort()); + assertThat(config.sortSpecs.length, equalTo(2)); + + assertThat(config.sortSpecs[0].field, equalTo("host.name")); + assertThat(config.sortSpecs[1].field, equalTo("@timestamp")); + assertThat(config.sortSpecs[0].order, equalTo(SortOrder.ASC)); + assertThat(config.sortSpecs[1].order, equalTo(SortOrder.DESC)); + assertThat(config.sortSpecs[0].missingValue, equalTo("_last")); + assertThat(config.sortSpecs[1].missingValue, equalTo("_last")); + assertThat(config.sortSpecs[0].mode, equalTo(MultiValueMode.MIN)); + assertThat(config.sortSpecs[1].mode, equalTo(MultiValueMode.MAX)); + } + + public void testLogsdbIndexSortTimestampOnly() { + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "logsdb") + .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), false) + .build(); + IndexSettings indexSettings = indexSettings(settings); + IndexSortConfig config = indexSettings.getIndexSortConfig(); + assertTrue(config.hasIndexSort()); + assertThat(config.sortSpecs.length, equalTo(1)); + + assertThat(config.sortSpecs[0].field, equalTo("@timestamp")); + assertThat(config.sortSpecs[0].order, equalTo(SortOrder.DESC)); + assertThat(config.sortSpecs[0].missingValue, equalTo("_last")); + assertThat(config.sortSpecs[0].mode, equalTo(MultiValueMode.MAX)); + } + + public void testLogsdbIndexSortTimestampBWC() { + Settings settings = Settings.builder().put(IndexSettings.MODE.getKey(), "logsdb").build(); + IndexSettings indexSettings = indexSettings( + settings, + IndexVersionUtils.getPreviousVersion(IndexVersions.LOGSB_OPTIONAL_SORTING_ON_HOST_NAME) + ); + IndexSortConfig config = indexSettings.getIndexSortConfig(); + assertTrue(config.hasIndexSort()); + assertThat(config.sortSpecs.length, equalTo(2)); + + assertThat(config.sortSpecs[0].field, equalTo("host.name")); + assertThat(config.sortSpecs[1].field, equalTo("@timestamp")); + assertThat(config.sortSpecs[0].order, equalTo(SortOrder.ASC)); + assertThat(config.sortSpecs[1].order, equalTo(SortOrder.ASC)); + assertThat(config.sortSpecs[0].missingValue, equalTo("_last")); + assertThat(config.sortSpecs[1].missingValue, equalTo("_last")); + assertThat(config.sortSpecs[0].mode, equalTo(MultiValueMode.MIN)); + assertThat(config.sortSpecs[1].mode, equalTo(MultiValueMode.MIN)); + } + private Sort buildIndexSort(IndexSettings indexSettings, MappedFieldType... mfts) { Map lookup = Maps.newMapWithExpectedSize(mfts.length); for (MappedFieldType mft : mfts) { diff --git a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java index 6f3077defa4a3..ccdd9c45f6a69 100644 --- a/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java @@ -74,7 +74,7 @@ public void testDefaultHostNameSortWithOrder() { .build() ) ); - assertEquals("index.sort.fields:[] index.sort.order:[desc], size mismatch", exception.getMessage()); + assertEquals("setting [index.sort.order] requires [index.sort.field] to be configured", exception.getMessage()); } public void testDefaultHostNameSortWithMode() { @@ -90,7 +90,7 @@ public void testDefaultHostNameSortWithMode() { .build() ) ); - assertEquals("index.sort.fields:[] index.sort.mode:[MAX], size mismatch", exception.getMessage()); + assertEquals("setting [index.sort.mode] requires [index.sort.field] to be configured", exception.getMessage()); } public void testDefaultHostNameSortWithMissing() { @@ -106,7 +106,7 @@ public void testDefaultHostNameSortWithMissing() { .build() ) ); - assertEquals("index.sort.fields:[] index.sort.missing:[_first], size mismatch", exception.getMessage()); + assertEquals("setting [index.sort.missing] requires [index.sort.field] to be configured", exception.getMessage()); } public void testCustomSortField() { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java index 71fd3a4761cbe..d7cf0b76681a6 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java @@ -43,7 +43,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; -import static org.elasticsearch.index.IndexSortConfig.TIME_SERIES_SORT; +import static org.elasticsearch.index.IndexSortConfig.IndexSortConfigDefaults.TIME_SERIES_SORT; import static org.hamcrest.Matchers.greaterThan; public class TimeSeriesIndexSearcherTests extends ESTestCase { diff --git a/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java b/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java new file mode 100644 index 0000000000000..75f7a41db9d26 --- /dev/null +++ b/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java @@ -0,0 +1,276 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import org.apache.lucene.index.DirectoryReader; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; +import org.elasticsearch.action.datastreams.GetDataStreamAction; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.features.FeatureService; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexSortConfig; +import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.license.LicenseSettings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.XPackPlugin; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.function.Consumer; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; + +public class LogsdbSortConfigIT extends ESSingleNodeTestCase { + + @Override + protected Collection> getPlugins() { + return List.of(InternalSettingsPlugin.class, XPackPlugin.class, LogsDBPlugin.class, DataStreamsPlugin.class); + } + + @Override + protected Settings nodeSettings() { + return Settings.builder() + .put(super.nodeSettings()) + .put("cluster.logsdb.enabled", "true") + .put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial") + .build(); + } + + private record DocWithId(String id, String source) { + public String source() { + return source.replace("%id%", id); + } + } + + private int id = 0; + + private DocWithId doc(String source) { + return new DocWithId(Integer.toString(id++), source); + } + + public void testHostnameTimestampSortConfig() throws IOException { + final String dataStreamName = "test-logsdb-sort-hostname-timestamp"; + + final String MAPPING = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "keyword" + }, + "test_id": { + "type": "text", + "store": true + } + } + } + } + """; + + final DocWithId[] orderedDocs = { + doc("{\"@timestamp\": \"2025-01-01T15:00:00\", \"host.name\": \"aaa\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T14:00:00\", \"host.name\": [\"aaa\", \"bbb\"], \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T12:30:00\", \"host.name\": [\"aaa\", \"bbb\"], \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T12:00:00\", \"host.name\": \"aaa\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T12:00:00\", \"host.name\": \"bbb\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T11:00:00\", \"host.name\": \"bbb\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T16:00:00\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T11:00:00\", \"test_id\": \"%id%\"}") }; + + createDataStream(dataStreamName, MAPPING); + + List shuffledDocs = shuffledList(Arrays.asList(orderedDocs)); + indexDocuments(dataStreamName, shuffledDocs); + + Index backingIndex = getBackingIndex(dataStreamName); + + var featureService = getInstanceFromNode(FeatureService.class); + if (featureService.getNodeFeatures().containsKey("mapper.provide_index_sort_setting_defaults")) { + assertSettings(backingIndex, settings -> { + assertThat(IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(settings), equalTo(List.of("host.name", "@timestamp"))); + assertThat(IndexSortConfig.INDEX_SORT_ORDER_SETTING.get(settings), equalTo(List.of(SortOrder.ASC, SortOrder.DESC))); + assertThat(IndexSortConfig.INDEX_SORT_MODE_SETTING.get(settings), equalTo(List.of(MultiValueMode.MIN, MultiValueMode.MAX))); + assertThat(IndexSortConfig.INDEX_SORT_MISSING_SETTING.get(settings), equalTo(List.of("_last", "_last"))); + }); + } + + assertOrder(backingIndex, orderedDocs); + } + + public void testTimestampOnlySortConfig() throws IOException { + final String dataStreamName = "test-logsdb-sort-timestamp-only"; + + final String MAPPING = """ + { + "_doc": { + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "type": "keyword" + }, + "test_id": { + "type": "text", + "store": true + } + } + } + } + """; + + final DocWithId[] orderedDocs = { + doc("{\"@timestamp\": \"2025-01-01T15:00:00\", \"host\": \"aaa\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T14:00:00\", \"host\": [\"aaa\", \"bbb\"], \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T12:30:00\", \"host\": [\"aaa\", \"bbb\"], \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T12:15:00\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T12:00:00\", \"host\": \"aaa\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T11:45:00\", \"host\": \"bbb\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T11:15:00\", \"host\": \"bbb\", \"test_id\": \"%id%\"}"), + doc("{\"@timestamp\": \"2025-01-01T11:00:00\", \"test_id\": \"%id%\"}") }; + + createDataStream(dataStreamName, MAPPING); + + List shuffledDocs = shuffledList(Arrays.asList(orderedDocs)); + indexDocuments(dataStreamName, shuffledDocs); + + Index backingIndex = getBackingIndex(dataStreamName); + + var featureService = getInstanceFromNode(FeatureService.class); + if (featureService.getNodeFeatures().containsKey("mapper.provide_index_sort_setting_defaults")) { + assertSettings(backingIndex, settings -> { + assertThat(IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(settings), equalTo(List.of("@timestamp"))); + assertThat(IndexSortConfig.INDEX_SORT_ORDER_SETTING.get(settings), equalTo(List.of(SortOrder.DESC))); + assertThat(IndexSortConfig.INDEX_SORT_MODE_SETTING.get(settings), equalTo(List.of(MultiValueMode.MAX))); + assertThat(IndexSortConfig.INDEX_SORT_MISSING_SETTING.get(settings), equalTo(List.of("_last"))); + }); + } + + assertOrder(backingIndex, orderedDocs); + } + + private void createDataStream(String dataStreamName, String mapping) throws IOException { + var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); + putTemplateRequest.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStreamName + "*")) + .template(new Template(indexSettings(1, 0).put("index.mode", "logsdb").build(), new CompressedXContent(mapping), null)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, putTemplateRequest).actionGet(); + + client().execute( + CreateDataStreamAction.INSTANCE, + new CreateDataStreamAction.Request(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, dataStreamName) + ).actionGet(); + } + + private void indexDocuments(String dataStreamName, List docs) { + BulkRequest bulkRequest = new BulkRequest(dataStreamName); + for (DocWithId doc : docs) { + var indexRequest = new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.CREATE); + indexRequest.source(doc.source(), XContentType.JSON); + bulkRequest.add(indexRequest); + } + var bulkResponse = client().bulk(bulkRequest).actionGet(); + assertFalse(bulkResponse.hasFailures()); + } + + private Index getBackingIndex(String dataStreamName) { + var getDataStreamRequest = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName }); + getDataStreamRequest.indicesOptions( + IndicesOptions.builder() + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .includeHidden(true) + .matchOpen(true) + .matchClosed(true) + .allowEmptyExpressions(true) + .resolveAliases(false) + .build() + ) + .build() + ); + + var response = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet(); + var dataStreams = response.getDataStreams(); + assertThat(dataStreams, hasSize(1)); + + var indices = dataStreams.getFirst().getDataStream().getIndices(); + assertThat(indices, hasSize(1)); + + return indices.getFirst(); + } + + private void assertSettings(Index backingIndex, Consumer settingsTest) { + final GetSettingsResponse getSettingsResponse = indicesAdmin().getSettings( + new GetSettingsRequest(TEST_REQUEST_TIMEOUT).indices(backingIndex.getName()).includeDefaults(true) + ).actionGet(); + final Settings combinedSettings = Settings.builder() + .put(getSettingsResponse.getIndexToDefaultSettings().get(backingIndex.getName())) + .put(getSettingsResponse.getIndexToSettings().get(backingIndex.getName())) + .build(); + settingsTest.accept(combinedSettings); + } + + private void assertOrder(Index backingIndex, DocWithId[] orderedDocs) throws IOException { + indicesAdmin().forceMerge(new ForceMergeRequest(backingIndex.getName()).maxNumSegments(1).flush(true)).actionGet(); + + IndexService indexService = getInstanceFromNode(IndicesService.class).indexServiceSafe(backingIndex); + assertThat(indexService.numberOfShards(), equalTo(1)); + IndexShard shard = indexService.getShard(0); + try (Engine.Searcher searcher = shard.acquireSearcher("test")) { + DirectoryReader directoryReader = searcher.getDirectoryReader(); + + assertThat(directoryReader.numDocs(), equalTo(orderedDocs.length)); + + var segments = directoryReader.leaves(); + assertThat(segments, hasSize(1)); + + var segment = segments.getFirst(); + var reader = segment.reader(); + var storedFields = reader.storedFields(); + + int expectedDocIdx = 0; + + for (int docId = 0; docId < reader.maxDoc(); docId++) { + String expectedId = orderedDocs[expectedDocIdx++].id; + String actualId = storedFields.document(docId).get("test_id"); + assertEquals(expectedId, actualId); + } + } + } + +} diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeEnabledRestTestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeEnabledRestTestIT.java index 63094852c3626..e473442642cfd 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeEnabledRestTestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeEnabledRestTestIT.java @@ -258,8 +258,8 @@ public void testLogsAtSettingWithTimeSeriesOverride() throws IOException { "index": { "routing_path": [ "hostname" ], "mode": "time_series", - "sort.field": [], - "sort.order": [] + "sort.field": ["_tsid", "@timestamp"], + "sort.order": ["asc", "desc"] } }, "mappings": { diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index ed9bd43262f0c..60af6b0635f2e 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -240,7 +240,7 @@ MappingHints getMappingHints( // stored when _source.mode mapping attribute is stored is fine as it has no effect, but avoids creating MapperService. var sourceMode = IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(tmpIndexMetadata.getSettings()); hasSyntheticSourceUsage = sourceMode == SourceFieldMapper.Mode.SYNTHETIC; - if (IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(indexTemplateAndCreateRequestSettings).isEmpty() == false) { + if (IndexSortConfig.INDEX_SORT_FIELD_SETTING.exists(indexTemplateAndCreateRequestSettings)) { // Custom sort config, no point for further checks on [host.name] field. return new MappingHints(hasSyntheticSourceUsage, false, false, true); } diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml index e2b426775745e..c0097538cfa22 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml @@ -1099,8 +1099,7 @@ create logsdb data stream with custom empty sorting: index: $backing_index - match: { .$backing_index.mappings.properties.@timestamp.type: date } - - match: { .$backing_index.mappings.properties.host.properties.name.type: keyword } - - match: { .$backing_index.mappings.properties.host.properties.name.ignore_above: 1024 } + - match: { .$backing_index.mappings.properties.host: null } --- create logsdb data stream with custom sorting on timestamp: diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/80_index_sort_defaults.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/80_index_sort_defaults.yml new file mode 100644 index 0000000000000..86497444cd8ae --- /dev/null +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/80_index_sort_defaults.yml @@ -0,0 +1,559 @@ +setup: + - requires: + cluster_features: [ "mapper.provide_index_sort_setting_defaults" ] + reason: "testing index sort setting defaults" + - requires: + test_runner_features: [ "allowed_warnings" ] + +--- +"test standard mode": + - do: + indices.create: + index: test-standard + body: + mappings: + properties: + "@timestamp": + type: date + host.name: + type: keyword + + - do: + indices.get_settings: + index: test-standard + include_defaults: true + + - match: { test-standard.defaults.index.mode: standard } + - match: { test-standard.defaults.index.sort.field: [] } + - match: { test-standard.defaults.index.sort.order: [] } + - match: { test-standard.defaults.index.sort.mode: [] } + - match: { test-standard.defaults.index.sort.missing: [] } + +--- +create logsdb data stream with host.name as keyword: + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host.name: + type: "keyword" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: "true" } + - match: { .$backing_index.defaults.index.sort.field: [ "host.name", "@timestamp" ] } + - match: { .$backing_index.defaults.index.sort.order: [ "asc", "desc" ] } + - match: { .$backing_index.defaults.index.sort.mode: [ "min", "max" ] } + - match: { .$backing_index.defaults.index.sort.missing: [ "_last", "_last" ] } + +--- +create logsdb data stream with host.name as keyword and timestamp as date: + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host.name: + type: "keyword" + "@timestamp": + type: "date" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: "true" } + - match: { .$backing_index.defaults.index.sort.field: [ "host.name", "@timestamp" ] } + - match: { .$backing_index.defaults.index.sort.order: [ "asc", "desc" ] } + - match: { .$backing_index.defaults.index.sort.mode: [ "min", "max" ] } + - match: { .$backing_index.defaults.index.sort.missing: [ "_last", "_last" ] } + +--- +create logsdb data stream with host.name as integer and timestamp as date: + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host.name: + type: "integer" + "@timestamp": + type: "date" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: "true" } + - match: { .$backing_index.defaults.index.sort.field: [ "host.name", "@timestamp" ] } + - match: { .$backing_index.defaults.index.sort.order: [ "asc", "desc" ] } + - match: { .$backing_index.defaults.index.sort.mode: [ "min", "max" ] } + - match: { .$backing_index.defaults.index.sort.missing: [ "_last", "_last" ] } +--- + +create logsdb data stream with no host.name and timestamp as date: + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + "@timestamp": + type: "date" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: [ "logsdb" ] + data_stream: { } + composed_of: [ "logsdb-mappings" ] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: "true" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: "true" } + - match: { .$backing_index.defaults.index.sort.field: [ "host.name", "@timestamp" ] } + - match: { .$backing_index.defaults.index.sort.order: [ "asc", "desc" ] } + - match: { .$backing_index.defaults.index.sort.mode: [ "min", "max" ] } + - match: { .$backing_index.defaults.index.sort.missing: [ "_last", "_last" ] } + +--- +create logsdb data stream with host as keyword and timestamp as date: + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host: + type: "keyword" + "@timestamp": + type: "date" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: [ "logsdb" ] + data_stream: { } + composed_of: [ "logsdb-mappings" ] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: null } + - match: { .$backing_index.defaults.index.logsdb.sort_on_host_name: "false" } + - match: { .$backing_index.defaults.index.sort.field: [ "@timestamp" ] } + - match: { .$backing_index.defaults.index.sort.order: [ "desc" ] } + - match: { .$backing_index.defaults.index.sort.mode: [ "max" ] } + - match: { .$backing_index.defaults.index.sort.missing: [ "_last" ] } + +--- +create logsdb data stream with host as keyword: + - requires: + test_runner_features: [ "allowed_warnings" ] + + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host: + type: "keyword" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: null } + - match: { .$backing_index.defaults.index.logsdb.sort_on_host_name: "false" } + - match: { .$backing_index.defaults.index.sort.field: [ "@timestamp" ] } + - match: { .$backing_index.defaults.index.sort.order: [ "desc" ] } + - match: { .$backing_index.defaults.index.sort.mode: [ "max" ] } + - match: { .$backing_index.defaults.index.sort.missing: [ "_last" ] } +--- +create logsdb data stream with host as text and multi fields: + - requires: + test_runner_features: [ "allowed_warnings" ] + + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host: + type: "text" + fields: + keyword: + ignore_above: 256 + type: "keyword" + "@timestamp": + type: "date" + format: "strict_date_optional_time" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: null } + - match: { .$backing_index.defaults.index.logsdb.sort_on_host_name: "false" } + - match: { .$backing_index.defaults.index.sort.field: [ "@timestamp" ] } + - match: { .$backing_index.defaults.index.sort.order: [ "desc" ] } + - match: { .$backing_index.defaults.index.sort.mode: [ "max" ] } + - match: { .$backing_index.defaults.index.sort.missing: [ "_last" ] } + +--- +create logsdb data stream with host as text: + - requires: + test_runner_features: [ "allowed_warnings" ] + + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host: + type: "text" + "@timestamp": + type: "date" + format: "strict_date_optional_time" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: null } + - match: { .$backing_index.defaults.index.logsdb.sort_on_host_name: "false" } + - match: { .$backing_index.defaults.index.sort.field: [ "@timestamp" ] } + - match: { .$backing_index.defaults.index.sort.order: [ "desc" ] } + - match: { .$backing_index.defaults.index.sort.mode: [ "max" ] } + - match: { .$backing_index.defaults.index.sort.missing: [ "_last" ] } +--- +create logsdb data stream with host as text and name as double: + - requires: + test_runner_features: [ "allowed_warnings" ] + + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host: + type: "text" + fields: + name: + type: "double" + "@timestamp": + type: "date" + format: "strict_date_optional_time" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: "true" } + - match: { .$backing_index.defaults.index.sort.field: [ "host.name", "@timestamp" ] } + - match: { .$backing_index.defaults.index.sort.order: [ "asc", "desc" ] } + - match: { .$backing_index.defaults.index.sort.mode: [ "min", "max" ] } + - match: { .$backing_index.defaults.index.sort.missing: [ "_last", "_last" ] } + +--- +create logsdb data stream with non-default sort field: + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + index.sort.field: ["some_field"] + mappings: + properties: + some_field: + type: "keyword" + "@timestamp": + type: "date" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.sort.field: [ "some_field" ] } + - match: { .$backing_index.defaults.index.sort.order: [ "asc" ] } + - match: { .$backing_index.defaults.index.sort.mode: [ "min" ] } + - match: { .$backing_index.defaults.index.sort.missing: [ "_last" ] }