diff --git a/docs/reference/elasticsearch/mapping-reference/pattern-text.md b/docs/reference/elasticsearch/mapping-reference/pattern-text.md index a9944fe7480a9..52a7727b4f0d5 100644 --- a/docs/reference/elasticsearch/mapping-reference/pattern-text.md +++ b/docs/reference/elasticsearch/mapping-reference/pattern-text.md @@ -46,14 +46,16 @@ In both cases, all queries return a constant score of 1.0. ## Index sorting for improved compression The compression provided by `pattern_text` can be significantly improved if the index is sorted by the `template_id` field. -For example, a typical approach would be to sort first by `message.template_id`, then by `@timestamp`, as shown in the following example. +This sorting is not applied by default, but can be enabled for the `message` field of LogsDB indices (assuming it is of type `pattern_text`) by setting the index setting `index.logsdb.default_sort_on_message_template` to `true`. +This will cause the index to be sorted by `host.name` (if present), then `message.template_id`, and finally by `@timestamp`. +If the index is not LogsDB or the `pattern_text` field is named something other than `message`, index sorting can still be manually applied as shown in the following example. ```console PUT logs { "settings": { "index": { - "sort.field": [ "message.template_id", "@timestamp" ], + "sort.field": [ "notice.template_id", "@timestamp" ], "sort.order": [ "asc", "desc" ] } }, @@ -62,7 +64,7 @@ PUT logs "@timestamp": { "type": "date" }, - "message": { + "notice": { "type": "pattern_text" } } diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index d18fdc2702d56..9f2c6c4d17355 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -198,6 +198,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING, IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS, IndexSettings.LOGSDB_SORT_ON_HOST_NAME, + IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE, IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD, IndexSettings.PREFER_ILM_SETTING, DataStreamFailureStoreDefinition.FAILURE_STORE_DEFINITION_VERSION_SETTING, diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index a5fd8144c5492..26f747d2d2315 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -668,6 +668,14 @@ public boolean isES87TSDBCodecEnabled() { Property.Final ); + public static final Setting LOGSDB_SORT_ON_MESSAGE_TEMPLATE = Setting.boolSetting( + "index.logsdb.sort_on_message_template", + false, + Property.IndexScope, + Property.PrivateIndex, + Property.Final + ); + public static final boolean DOC_VALUES_SKIPPER = new FeatureFlag("doc_values_skipper").isEnabled(); public static final Setting USE_DOC_VALUES_SKIPPER = Setting.boolSetting( "index.mapping.use_doc_values_skipper", diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index 8235b94b4c3a9..fd445d470837c 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -26,6 +26,7 @@ import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.SortOrder; +import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; import java.util.List; @@ -106,19 +107,21 @@ public final class IndexSortConfig { ); public static class IndexSortConfigDefaults { - public static final FieldSortSpec[] TIME_SERIES_SORT, TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_BWC_SORT; + public static final FieldSortSpec[] TIME_SERIES_SORT, HOSTNAME_TIMESTAMP_BWC_SORT; + + private static final FieldSortSpec HOSTNAME_SPEC, MESSAGE_PATTERN_SPEC, TIMESTAMP_SPEC; static { - FieldSortSpec timeStampSpec = new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH); - timeStampSpec.order = SortOrder.DESC; - TIME_SERIES_SORT = new FieldSortSpec[] { new FieldSortSpec(TimeSeriesIdFieldMapper.NAME), timeStampSpec }; - TIMESTAMP_SORT = new FieldSortSpec[] { timeStampSpec }; + TIMESTAMP_SPEC = new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH); + TIMESTAMP_SPEC.order = SortOrder.DESC; + TIME_SERIES_SORT = new FieldSortSpec[] { new FieldSortSpec(TimeSeriesIdFieldMapper.NAME), TIMESTAMP_SPEC }; + + HOSTNAME_SPEC = new FieldSortSpec(IndexMode.HOST_NAME); + HOSTNAME_SPEC.order = SortOrder.ASC; + HOSTNAME_SPEC.missingValue = "_last"; + HOSTNAME_SPEC.mode = MultiValueMode.MIN; - FieldSortSpec hostnameSpec = new FieldSortSpec(IndexMode.HOST_NAME); - hostnameSpec.order = SortOrder.ASC; - hostnameSpec.missingValue = "_last"; - hostnameSpec.mode = MultiValueMode.MIN; - HOSTNAME_TIMESTAMP_SORT = new FieldSortSpec[] { hostnameSpec, timeStampSpec }; + MESSAGE_PATTERN_SPEC = new FieldSortSpec("message.template_id"); // Older indexes use ascending ordering for host name and timestamp. HOSTNAME_TIMESTAMP_BWC_SORT = new FieldSortSpec[] { @@ -148,7 +151,17 @@ public static FieldSortSpec[] getDefaultSortSpecs(Settings settings) { IndexVersions.LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0 )) { - return (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(settings)) ? HOSTNAME_TIMESTAMP_SORT : TIMESTAMP_SORT; + + List sortSpecs = new ArrayList<>(3); + if (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(settings)) { + sortSpecs.add(HOSTNAME_SPEC); + } + if (IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.get(settings)) { + sortSpecs.add(MESSAGE_PATTERN_SPEC); + } + sortSpecs.add(TIMESTAMP_SPEC); + + return sortSpecs.toArray(FieldSortSpec[]::new); } else { return HOSTNAME_TIMESTAMP_BWC_SORT; } diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index 6c3103e4a1f35..76e8412953006 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -332,6 +332,51 @@ public void testLogsdbIndexSortWithHostname() { assertThat(config.sortSpecs[1].mode, equalTo(MultiValueMode.MAX)); } + public void testLogsdbIndexSortWithMessage() { + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "logsdb") + .put(IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.getKey(), true) + .build(); + IndexSettings indexSettings = indexSettings(settings); + IndexSortConfig config = indexSettings.getIndexSortConfig(); + assertTrue(config.hasIndexSort()); + assertThat(config.sortSpecs.length, equalTo(2)); + + assertThat(config.sortSpecs[0].field, equalTo("message.template_id")); + assertThat(config.sortSpecs[1].field, equalTo("@timestamp")); + assertThat(config.sortSpecs[0].order, equalTo(SortOrder.ASC)); + assertThat(config.sortSpecs[1].order, equalTo(SortOrder.DESC)); + assertThat(config.sortSpecs[0].missingValue, equalTo("_last")); + assertThat(config.sortSpecs[1].missingValue, equalTo("_last")); + assertThat(config.sortSpecs[0].mode, equalTo(MultiValueMode.MIN)); + assertThat(config.sortSpecs[1].mode, equalTo(MultiValueMode.MAX)); + } + + public void testLogsdbIndexSortWithMessageAndHostname() { + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "logsdb") + .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true) + .put(IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.getKey(), true) + .build(); + IndexSettings indexSettings = indexSettings(settings); + IndexSortConfig config = indexSettings.getIndexSortConfig(); + assertTrue(config.hasIndexSort()); + assertThat(config.sortSpecs.length, equalTo(3)); + + assertThat(config.sortSpecs[0].field, equalTo("host.name")); + assertThat(config.sortSpecs[1].field, equalTo("message.template_id")); + assertThat(config.sortSpecs[2].field, equalTo("@timestamp")); + assertThat(config.sortSpecs[0].order, equalTo(SortOrder.ASC)); + assertThat(config.sortSpecs[1].order, equalTo(SortOrder.ASC)); + assertThat(config.sortSpecs[2].order, equalTo(SortOrder.DESC)); + assertThat(config.sortSpecs[0].missingValue, equalTo("_last")); + assertThat(config.sortSpecs[1].missingValue, equalTo("_last")); + assertThat(config.sortSpecs[2].missingValue, equalTo("_last")); + assertThat(config.sortSpecs[0].mode, equalTo(MultiValueMode.MIN)); + assertThat(config.sortSpecs[1].mode, equalTo(MultiValueMode.MIN)); + assertThat(config.sortSpecs[2].mode, equalTo(MultiValueMode.MAX)); + } + public void testLogsdbIndexSortTimestampOnly() { Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), "logsdb") diff --git a/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java b/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java index 75f7a41db9d26..4470e5928c0b6 100644 --- a/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java +++ b/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java @@ -44,6 +44,7 @@ import java.util.Collection; import java.util.List; import java.util.function.Consumer; +import java.util.function.UnaryOperator; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -76,6 +77,89 @@ private DocWithId doc(String source) { return new DocWithId(Integer.toString(id++), source); } + public void testHostnameMessageTimestampSortConfig() throws IOException { + final String dataStreamName = "test-logsdb-sort-hostname-message-timestamp"; + + final String mapping = """ + { + "_doc": { + "properties": { + "@timestmap": { + "type": "date" + }, + "host.name": { + "type": "keyword" + }, + "message": { + "type": "pattern_text" + }, + "test_id": { + "type": "text", + "store": true + } + } + } + } + """; + + final DocWithId[] orderedDocs = { + doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"host.name\":\"aaa\",\"message\":\"bar 5\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"host.name\":[\"aaa\",\"bbb\"],\"message\":\"bar 7\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"host.name\":\"aaa\",\"message\":\"bar 9\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"host.name\":\"aaa\",\"message\":\"foo 6\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"host.name\":\"aaa\",\"message\":\"foo 1\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"host.name\":[\"aaa\",\"bbb\"],\"message\":\"foo 9\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"host.name\":[\"aaa\",\"bbb\"],\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"host.name\":[\"aaa\",\"bbb\"],\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"host.name\":[\"aaa\",\"bbb\"],\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"host.name\":\"bbb\",\"message\":\"bar 4\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"host.name\":\"bbb\",\"message\":\"bar 5\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"host.name\":\"bbb\",\"message\":\"bar 2\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"host.name\":\"bbb\",\"message\":\"foo 7\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"host.name\":\"bbb\",\"message\":\"foo 3\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"host.name\":\"bbb\",\"message\":\"foo 6\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"host.name\":\"bbb\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"host.name\":\"bbb\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"host.name\":\"bbb\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"message\":\"bar 4\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"message\":\"bar 1\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"message\":\"bar 4\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"message\":\"foo 1\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"message\":\"foo 9\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"message\":\"foo 3\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"test_id\": \"%id%\"}") }; + + createDataStream(dataStreamName, mapping, b -> b.put("index.logsdb.default_sort_on_message_template", true)); + + List shuffledDocs = shuffledList(Arrays.asList(orderedDocs)); + indexDocuments(dataStreamName, shuffledDocs); + + Index backingIndex = getBackingIndex(dataStreamName); + + var featureService = getInstanceFromNode(FeatureService.class); + if (featureService.getNodeFeatures().containsKey("mapper.provide_index_sort_setting_defaults")) { + assertSettings(backingIndex, settings -> { + assertThat( + IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(settings), + equalTo(List.of("host.name", "message.template_id", "@timestamp")) + ); + assertThat( + IndexSortConfig.INDEX_SORT_ORDER_SETTING.get(settings), + equalTo(List.of(SortOrder.ASC, SortOrder.ASC, SortOrder.DESC)) + ); + assertThat( + IndexSortConfig.INDEX_SORT_MODE_SETTING.get(settings), + equalTo(List.of(MultiValueMode.MIN, MultiValueMode.MIN, MultiValueMode.MAX)) + ); + assertThat(IndexSortConfig.INDEX_SORT_MISSING_SETTING.get(settings), equalTo(List.of("_last", "_last", "_last"))); + }); + } + + assertOrder(backingIndex, orderedDocs); + } + public void testHostnameTimestampSortConfig() throws IOException { final String dataStreamName = "test-logsdb-sort-hostname-timestamp"; @@ -181,11 +265,21 @@ public void testTimestampOnlySortConfig() throws IOException { } private void createDataStream(String dataStreamName, String mapping) throws IOException { + createDataStream(dataStreamName, mapping, UnaryOperator.identity()); + } + + private void createDataStream(String dataStreamName, String mapping, UnaryOperator settings) throws IOException { var putTemplateRequest = new TransportPutComposableIndexTemplateAction.Request("id"); putTemplateRequest.indexTemplate( ComposableIndexTemplate.builder() .indexPatterns(List.of(dataStreamName + "*")) - .template(new Template(indexSettings(1, 0).put("index.mode", "logsdb").build(), new CompressedXContent(mapping), null)) + .template( + new Template( + settings.apply(indexSettings(1, 0)).put("index.mode", "logsdb").build(), + new CompressedXContent(mapping), + null + ) + ) .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) .build() ); diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextBasicLicenseIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextBasicLicenseIT.java new file mode 100644 index 0000000000000..e83f37864c94a --- /dev/null +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextBasicLicenseIT.java @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb.patterntext; + +import org.elasticsearch.xpack.logsdb.DataStreamLicenseChangeTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.hasEntry; + +public class PatternTextBasicLicenseIT extends DataStreamLicenseChangeTestCase { + @Before + public void checkClusterFeature() { + assumeTrue("[patterned_text] must be available", clusterHasFeature("mapper.patterned_text")); + } + + @SuppressWarnings("unchecked") + private Map getFieldMapping(Map mapping, String fieldName) { + return (Map) ((Map) mapping.get("properties")).get(fieldName); + } + + public void testStandardIndex() throws IOException { + final String dataStreamName = "test-foo"; + + final String mappingStr = """ + { + "index_patterns": ["%name%"], + "priority": 500, + "data_stream": {}, + "template": { + "mappings": { + "properties": { + "pattern_field": { + "type": "pattern_text" + } + } + } + } + }"""; + + final String doc = """ + {"index": {}} + {"@timestamp": "2025-01-01T00:00:00.000Z", "pattern_field": "foo"} + """; + + assertOK(putTemplate(client(), "logs@custom", mappingStr.replace("%name%", dataStreamName))); + assertOK(bulkIndex(client(), dataStreamName, () -> doc)); + + String backingIndex0 = getDataStreamBackingIndex(client(), dataStreamName, 0); + { + assertEquals("true", getSetting(client(), backingIndex0, "index.mapping.pattern_text.disable_templating")); + Map mapping = getMapping(client(), backingIndex0); + Map patternFieldMapping = getFieldMapping(mapping, "pattern_field"); + assertThat(patternFieldMapping, hasEntry("disable_templating", true)); + } + + assertOK(rolloverDataStream(client(), dataStreamName)); + + String backingIndex1 = getDataStreamBackingIndex(client(), dataStreamName, 1); + { + assertEquals("true", getSetting(client(), backingIndex1, "index.mapping.pattern_text.disable_templating")); + Map mapping = getMapping(client(), backingIndex1); + Map patternFieldMapping = getFieldMapping(mapping, "pattern_field"); + assertThat(patternFieldMapping, hasEntry("disable_templating", true)); + } + } + + public void testLogsdbIndex() throws IOException { + final String dataStreamName = "test-bar"; + + final String mappingStr = """ + { + "index_patterns": ["%name%"], + "priority": 500, + "data_stream": {}, + "template": { + "settings": { + "index.mode": "logsdb", + "index.logsdb.default_sort_on_message_template": true + }, + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "message": { + "type": "pattern_text" + }, + "host.name": { + "type": "keyword" + } + } + } + } + } + """; + + final String doc = """ + {"index": {}} + {"@timestamp": "2025-01-01T00:00:00.000Z", "message": "foo 123", "host.name": "bar"} + """; + + assertOK(putTemplate(client(), "logs@custom", mappingStr.replace("%name%", dataStreamName))); + assertOK(bulkIndex(client(), dataStreamName, () -> doc)); + + String backingIndex0 = getDataStreamBackingIndex(client(), dataStreamName, 0); + { + assertEquals("true", getSetting(client(), backingIndex0, "index.mapping.pattern_text.disable_templating")); + assertEquals( + List.of("host.name", "message.template_id", "@timestamp"), + getSetting(client(), backingIndex0, "index.sort.field") + ); + Map mapping = getMapping(client(), backingIndex0); + Map patternFieldMapping = getFieldMapping(mapping, "message"); + assertThat(patternFieldMapping, hasEntry("disable_templating", true)); + } + + assertOK(rolloverDataStream(client(), dataStreamName)); + + String backingIndex1 = getDataStreamBackingIndex(client(), dataStreamName, 1); + { + assertEquals("true", getSetting(client(), backingIndex1, "index.mapping.pattern_text.disable_templating")); + assertEquals( + List.of("host.name", "message.template_id", "@timestamp"), + getSetting(client(), backingIndex1, "index.sort.field") + ); + Map mapping = getMapping(client(), backingIndex1); + Map patternFieldMapping = getFieldMapping(mapping, "message"); + assertThat(patternFieldMapping, hasEntry("disable_templating", true)); + } + } +} diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java index 3c7ceb883fd73..49b6bf6072976 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java @@ -48,6 +48,12 @@ public class LogsDBPlugin extends Plugin implements ActionPlugin, MapperPlugin { Setting.Property.Dynamic, Setting.Property.NodeScope ); + static final Setting LOGSDB_DEFAULT_SORT_ON_MESSAGE_TEMPLATE = Setting.boolSetting( + "index.logsdb.default_sort_on_message_template", + false, + Setting.Property.IndexScope, + Setting.Property.Final + ); private final LogsdbIndexModeSettingsProvider logsdbIndexModeSettingsProvider; @@ -97,7 +103,8 @@ public List> getSettings() { FALLBACK_SETTING, CLUSTER_LOGSDB_ENABLED, LOGSDB_PRIOR_LOGS_USAGE, - PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING + PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING, + LOGSDB_DEFAULT_SORT_ON_MESSAGE_TEMPLATE ); } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index 7ca223e31cd4f..0588f1b84c97e 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -58,10 +58,13 @@ final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider { private static final Logger LOGGER = LogManager.getLogger(LogsdbIndexModeSettingsProvider.class); static final String LOGS_PATTERN = "logs-*-*"; - private static final Set MAPPING_INCLUDES = Set.of("_source.*", "properties.host**", "properties.resource**", "subobjects") - .stream() - .flatMap(v -> Stream.of(v, "_doc." + v)) - .collect(Collectors.toSet()); + private static final Set MAPPING_INCLUDES = Set.of( + "_source.*", + "properties.host**", + "properties.resource**", + "subobjects", + "properties.message**" + ).stream().flatMap(v -> Stream.of(v, "_doc." + v)).collect(Collectors.toSet()); private static final Function, Map> MAPPING_INCLUDES_FILTER = XContentMapValues.filter( MAPPING_INCLUDES.toArray(String[]::new), new String[0] @@ -159,6 +162,12 @@ && matchesLogsPattern(dataStreamName)) { additionalSettings.put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true); } + if (mappingHints.sortOnMessageTemplate + && (LogsDBPlugin.LOGSDB_DEFAULT_SORT_ON_MESSAGE_TEMPLATE.get(settings) + || IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.get(settings))) { + additionalSettings.put(IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.getKey(), true); + } + // Inject routing path matching sort fields. if (settings.getAsBoolean(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), false)) { if (supportFallbackLogsdbRouting.get() == false || licenseService.allowLogsdbRoutingOnSortField(isTemplateValidation)) { @@ -207,8 +216,14 @@ && matchesLogsPattern(dataStreamName)) { } } - record MappingHints(boolean hasSyntheticSourceUsage, boolean sortOnHostName, boolean addHostNameField, boolean maybeUsesPatternText) { - static MappingHints EMPTY = new MappingHints(false, false, false, false); + record MappingHints( + boolean hasSyntheticSourceUsage, + boolean sortOnHostName, + boolean addHostNameField, + boolean sortOnMessageTemplate, + boolean maybeUsesPatternText + ) { + static MappingHints EMPTY = new MappingHints(false, false, false, false, false); } private static boolean matchesLogsPattern(final String name) { @@ -247,12 +262,13 @@ MappingHints getMappingHints( hasSyntheticSourceUsage = sourceMode == SourceFieldMapper.Mode.SYNTHETIC; if (IndexSortConfig.INDEX_SORT_FIELD_SETTING.exists(indexTemplateAndCreateRequestSettings)) { // Custom sort config, no point for further checks on [host.name] field. - return new MappingHints(hasSyntheticSourceUsage, false, false, true); + return new MappingHints(hasSyntheticSourceUsage, false, false, false, true); } if (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(indexTemplateAndCreateRequestSettings) - && IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(indexTemplateAndCreateRequestSettings)) { + && IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(indexTemplateAndCreateRequestSettings) + && IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.get(indexTemplateAndCreateRequestSettings)) { // Settings for adding and sorting on [host.name] are already set, propagate them. - return new MappingHints(hasSyntheticSourceUsage, true, true, true); + return new MappingHints(hasSyntheticSourceUsage, true, true, true, true); } } @@ -285,6 +301,7 @@ MappingHints getMappingHints( } mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE); Mapper hostName = mapperService.mappingLookup().getMapper("host.name"); + Mapper messageField = mapperService.mappingLookup().getMapper("message"); hasSyntheticSourceUsage = hasSyntheticSourceUsage || mapperService.documentMapper().sourceMapper().isSynthetic(); boolean addHostNameField = IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(indexTemplateAndCreateRequestSettings) || (hostName == null @@ -295,7 +312,15 @@ MappingHints getMappingHints( || addHostNameField || (hostName instanceof NumberFieldMapper nfm && nfm.fieldType().hasDocValues()) || (hostName instanceof KeywordFieldMapper kfm && kfm.fieldType().hasDocValues()); - return new MappingHints(hasSyntheticSourceUsage, sortOnHostName, addHostNameField, maybeUsesPatternText); + boolean sortOnMessageTemplate = IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.get(indexTemplateAndCreateRequestSettings) + || (messageField instanceof PatternTextFieldMapper); + return new MappingHints( + hasSyntheticSourceUsage, + sortOnHostName, + addHostNameField, + sortOnMessageTemplate, + maybeUsesPatternText + ); } } catch (AssertionError | Exception e) { // In case invalid mappings or setting are provided, then mapper service creation can fail. diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index d8a229877b686..4f1331b80fa30 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -978,15 +978,9 @@ public void testPatternTextNotAllowedByLicense() throws IOException { "{\"properties\":{\"foo\":{\"type\":\"pattern_text\"}}}", "{\"properties\":{\"bar\":{\"properties\":{\"baz\":{\"type\":\"pattern_text\"}}}}}" }; - var expectedSettings = Settings.builder() - .put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true) - .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true) - .put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true) - .build(); - for (String mapping : patternTextLicenceCheckedFieldMappings) { var result = generateLogsdbSettings(Settings.EMPTY, getMapping(mapping), Version.CURRENT, basicLogsdbLicenseService); - assertEquals(expectedSettings, result); + assertTrue(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.get(result)); } } @@ -1006,10 +1000,12 @@ public void testSortAndHostNamePropagateValue() throws Exception { .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB) .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true) .put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true) + .put(IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.getKey(), true) .build(); Settings result = generateLogsdbSettings(settings); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertTrue(IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.get(result)); assertEquals(0, newMapperServiceCounter.get()); } @@ -1076,6 +1072,61 @@ public void testSortAndHostNameKeyword() throws Exception { assertEquals(1, newMapperServiceCounter.get()); } + public void testSortAndHostNameKeywordAndMessage() throws Exception { + var settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB) + .put(LogsDBPlugin.LOGSDB_DEFAULT_SORT_ON_MESSAGE_TEMPLATE.getKey(), true) + .build(); + + var mappings = """ + { + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "keyword" + }, + "message": { + "type": "pattern_text" + } + } + } + """; + + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); + assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); + assertTrue(IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.get(result)); + assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertEquals(1, newMapperServiceCounter.get()); + } + + public void testSortAndHostNameKeywordAndDisabledMessage() throws Exception { + var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); + + var mappings = """ + { + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "keyword" + }, + "message": { + "type": "pattern_text" + } + } + } + """; + + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); + assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); + assertFalse(IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.get(result)); + assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertEquals(1, newMapperServiceCounter.get()); + } + public void testSortAndHostNameKeywordNoDocvalues() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/80_index_sort_defaults.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/80_index_sort_defaults.yml index 86497444cd8ae..663ab108ca0ab 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/80_index_sort_defaults.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/80_index_sort_defaults.yml @@ -129,6 +129,175 @@ create logsdb data stream with host.name as keyword and timestamp as date: - match: { .$backing_index.defaults.index.sort.mode: [ "min", "max" ] } - match: { .$backing_index.defaults.index.sort.missing: [ "_last", "_last" ] } +--- +create logsdb data stream with message as pattern text, and timestamp as date: + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + logsdb.default_sort_on_message_template: true + mappings: + properties: + message: + type: pattern_text + host: + type: keyword + "@timestamp": + type: "date" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: null } + - match: { .$backing_index.defaults.index.logsdb.sort_on_host_name: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_message_template: "true" } + - match: { .$backing_index.defaults.index.sort.field: [ "message.template_id", "@timestamp" ] } + - match: { .$backing_index.defaults.index.sort.order: [ "asc", "desc" ] } + - match: { .$backing_index.defaults.index.sort.mode: [ "min", "max" ] } + - match: { .$backing_index.defaults.index.sort.missing: [ "_last", "_last" ] } + + +--- +create logsdb data stream with host.name as keyword, message as pattern text, and timestamp as date: + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + logsdb.default_sort_on_message_template: true + mappings: + properties: + host.name: + type: "keyword" + message: + type: pattern_text + "@timestamp": + type: "date" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: "true" } + - match: { .$backing_index.settings.index.logsdb.sort_on_message_template: "true" } + - match: { .$backing_index.defaults.index.sort.field: [ "host.name", "message.template_id", "@timestamp" ] } + - match: { .$backing_index.defaults.index.sort.order: [ "asc", "asc", "desc" ] } + - match: { .$backing_index.defaults.index.sort.mode: [ "min", "min", "max" ] } + - match: { .$backing_index.defaults.index.sort.missing: [ "_last", "_last", "_last" ] } + +--- +create logsdb data stream with host.name as keyword, message as pattern text, and timestamp as date, but message sort disabled: + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host.name: + type: "keyword" + message: + type: pattern_text + "@timestamp": + type: "date" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: "true" } + - match: { .$backing_index.settings.index.logsdb.sort_on_message_template: null} + - match: { .$backing_index.defaults.index.logsdb.sort_on_message_template: "false" } + - match: { .$backing_index.settings.index.logsdb.default_sort_on_message_template: null } + - match: { .$backing_index.defaults.index.logsdb.default_sort_on_message_template: "false" } + - match: { .$backing_index.defaults.index.sort.field: [ "host.name", "@timestamp" ] } + - match: { .$backing_index.defaults.index.sort.order: [ "asc", "desc" ] } + - match: { .$backing_index.defaults.index.sort.mode: [ "min", "max" ] } + - match: { .$backing_index.defaults.index.sort.missing: [ "_last", "_last" ] } + --- create logsdb data stream with host.name as integer and timestamp as date: - do: