diff --git a/docs/changelog/136119.yaml b/docs/changelog/136119.yaml new file mode 100644 index 0000000000000..d900562147824 --- /dev/null +++ b/docs/changelog/136119.yaml @@ -0,0 +1,6 @@ +pr: 136119 +summary: Fix logsdb settings provider mapping filters +area: Logs +type: bug +issues: + - 136107 diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index a98f0a1e33af9..d7b51722d35e7 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -42,6 +42,8 @@ import java.util.Locale; import java.util.Set; import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH; import static org.elasticsearch.xpack.logsdb.LogsDBPlugin.CLUSTER_LOGSDB_ENABLED; @@ -49,7 +51,10 @@ final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider { private static final Logger LOGGER = LogManager.getLogger(LogsdbIndexModeSettingsProvider.class); static final String LOGS_PATTERN = "logs-*-*"; - private static final Set MAPPING_INCLUDES = Set.of("_doc._source.*", "_doc.properties.host**", "_doc.subobjects"); + private static final Set MAPPING_INCLUDES = Set.of("_source.*", "properties.host**", "subobjects") + .stream() + .flatMap(v -> Stream.of(v, "_doc." + v)) + .collect(Collectors.toSet()); private final LogsdbLicenseService licenseService; private final SetOnce> mapperServiceFactory = new SetOnce<>(); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index 0efca7175a56e..84a816620aab2 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -53,17 +53,15 @@ public class LogsdbIndexModeSettingsProviderTests extends ESTestCase { private static final String DATA_STREAM_NAME = "logs-app1"; public static final String DEFAULT_MAPPING = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "message": { - "type": "keyword" - }, - "host.name": { - "type": "keyword" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "message": { + "type": "keyword" + }, + "host.name": { + "type": "keyword" } } } @@ -84,6 +82,14 @@ public void setup() throws Exception { logsdbLicenseService.setLicenseService(mockLicenseService); } + private static String getMapping(String contents) { + if (randomBoolean()) { + return "{\"_doc\":" + contents + "}"; + } else { + return contents; + } + } + private LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled) { return withSyntheticSourceDemotionSupport(enabled, Version.CURRENT); } @@ -153,7 +159,7 @@ public void testDisabled() throws IOException { Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertTrue(additionalIndexSettings.isEmpty()); @@ -172,7 +178,7 @@ public void testOnIndexCreation() throws IOException { Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertTrue(additionalIndexSettings.isEmpty()); @@ -191,7 +197,7 @@ public void testOnExplicitStandardIndex() throws IOException { Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.getName()).build(), - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertTrue(additionalIndexSettings.isEmpty()); @@ -210,7 +216,7 @@ public void testOnExplicitTimeSeriesIndex() throws IOException { Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()).build(), - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertTrue(additionalIndexSettings.isEmpty()); @@ -229,7 +235,7 @@ public void testNonLogsDataStream() throws IOException { Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertTrue(additionalIndexSettings.isEmpty()); @@ -244,7 +250,7 @@ public void testWithoutLogsComponentTemplate() throws IOException { buildMetadata(List.of("*"), List.of()), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); @@ -259,7 +265,7 @@ public void testWithLogsComponentTemplate() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); @@ -274,7 +280,7 @@ public void testWithMultipleComponentTemplates() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings", "logs@custom")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); @@ -289,7 +295,7 @@ public void testWithCustomComponentTemplatesOnly() throws IOException { buildMetadata(List.of("*"), List.of("logs@custom", "custom-component-template")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); @@ -304,7 +310,7 @@ public void testNonMatchingTemplateIndexPattern() throws IOException { buildMetadata(List.of("standard-apache-production"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); @@ -323,7 +329,7 @@ public void testCaseSensitivity() throws IOException { Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertTrue(additionalIndexSettings.isEmpty()); @@ -339,7 +345,7 @@ public void testMultipleHyphensInDataStreamName() throws IOException { Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); @@ -354,7 +360,7 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertTrue(beforeSettings.isEmpty()); @@ -368,7 +374,7 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertIndexMode(afterSettings, IndexMode.LOGSDB.getName()); @@ -382,14 +388,14 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertTrue(laterSettings.isEmpty()); } private static Metadata buildMetadata(final List indexPatterns, final List componentTemplates) throws IOException { - final Template template = new Template(Settings.EMPTY, new CompressedXContent(DEFAULT_MAPPING), null); + final Template template = new Template(Settings.EMPTY, new CompressedXContent(getMapping(DEFAULT_MAPPING)), null); final ComposableIndexTemplate composableTemplate = ComposableIndexTemplate.builder() .indexPatterns(indexPatterns) .template(template) @@ -414,19 +420,17 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { { String mapping = """ { - "_doc": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "my_field": { - "type": "keyword" - } + "_source": { + "mode": "synthetic" + }, + "properties": { + "my_field": { + "type": "keyword" } } } """; - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertTrue(result); assertThat(newMapperServiceCounter.get(), equalTo(1)); @@ -438,14 +442,12 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { if (withSourceMode) { mapping = """ { - "_doc": { - "_source": { - "mode": "stored" - }, - "properties": { - "my_field": { - "type": "keyword" - } + "_source": { + "mode": "stored" + }, + "properties": { + "my_field": { + "type": "keyword" } } } @@ -453,17 +455,15 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { } else { mapping = """ { - "_doc": { - "properties": { - "my_field": { - "type": "keyword" - } + "properties": { + "my_field": { + "type": "keyword" } } } """; } - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(2)); @@ -477,21 +477,19 @@ public void testValidateIndexName() throws IOException { String indexName = MetadataIndexTemplateService.VALIDATE_INDEX_NAME; String mapping = """ { - "_doc": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "my_field": { - "type": "keyword" - } + "_source": { + "mode": "synthetic" + }, + "properties": { + "my_field": { + "type": "keyword" } } } """; Settings settings = Settings.EMPTY; LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); } @@ -501,11 +499,9 @@ public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); String mapping = """ { - "_doc": { - "properties": { - "my_field": { - "type": "keyword" - } + "properties": { + "my_field": { + "type": "keyword" } } } @@ -513,7 +509,7 @@ public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); { Settings settings = Settings.builder().put("index.mode", "logsdb").build(); - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertTrue(result); assertThat(newMapperServiceCounter.get(), equalTo(1)); @@ -530,7 +526,7 @@ public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException assertThat(newMapperServiceCounter.get(), equalTo(3)); } { - boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(4)); @@ -542,12 +538,10 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); String mapping = """ { - "_doc": { - "properties": { - "my_field": { - "type": "keyword", - "time_series_dimension": true - } + "properties": { + "my_field": { + "type": "keyword", + "time_series_dimension": true } } } @@ -555,7 +549,7 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); { Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build(); - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertTrue(result); } @@ -569,7 +563,7 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { assertFalse(result); } { - boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); } @@ -583,19 +577,17 @@ public void testNewIndexHasSyntheticSourceUsageInvalidSettings() throws IOExcept { String mapping = """ { - "_doc": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "my_field": { - "type": "keyword" - } + "_source": { + "mode": "synthetic" + }, + "properties": { + "my_field": { + "type": "keyword" } } } """; - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(1)); @@ -603,16 +595,14 @@ public void testNewIndexHasSyntheticSourceUsageInvalidSettings() throws IOExcept { String mapping = """ { - "_doc": { - "properties": { - "my_field": { - "type": "keyword" - } + "properties": { + "my_field": { + "type": "keyword" } } } """; - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(2)); @@ -900,16 +890,14 @@ public void testSortAndHostNoHost() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - } + "properties": { + "@timestamp": { + "type": "date" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -919,16 +907,14 @@ public void testSortAndHostNoHostOldNode() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - } + "properties": { + "@timestamp": { + "type": "date" } } } """; - Settings result = generateLogsdbSettings(settings, mappings, Version.V_8_17_0); + Settings result = generateLogsdbSettings(settings, getMapping(mappings), Version.V_8_17_0); assertTrue(result.isEmpty()); } @@ -936,19 +922,17 @@ public void testSortAndHostNameKeyword() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "keyword" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "keyword" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -958,20 +942,18 @@ public void testSortAndHostNameKeywordNoDocvalues() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "keyword", - "doc_values": false - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "keyword", + "doc_values": false } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -981,19 +963,17 @@ public void testSortAndHostNameInteger() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "integer" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "integer" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1003,20 +983,18 @@ public void testSortAndHostNameIntegerNoDocvalues() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "integer", - "doc_values": false - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "integer", + "doc_values": false } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1026,19 +1004,17 @@ public void testSortAndHostNameBoolean() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "boolean" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "boolean" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1048,19 +1024,17 @@ public void testSortAndHostObject() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host": { - "type": "object" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "type": "object" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1070,19 +1044,17 @@ public void testSortAndHostField() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host": { - "type": "keyword" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "type": "keyword" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1092,20 +1064,18 @@ public void testSortAndHostFieldSubobjectsFalse() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "subobjects": false, - "properties": { - "@timestamp": { - "type": "date" - }, - "host": { - "type": "keyword" - } + "subobjects": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "type": "keyword" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1118,19 +1088,17 @@ public void testSortAndHostNameObject() throws Exception { .build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name.sub": { - "type": "keyword" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name.sub": { + "type": "keyword" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1143,11 +1111,9 @@ public void testSortFastRefresh() throws Exception { .build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - } + "properties": { + "@timestamp": { + "type": "date" } } } @@ -1173,7 +1139,7 @@ public void testSortFastRefresh() throws Exception { metadata, Instant.now(), settings, - List.of(new CompressedXContent(mappings)) + List.of(new CompressedXContent(getMapping(mappings))) ); Settings result = builder().put(additionalIndexSettings).build(); diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml index 92c80847cacca..7a79c517bc1ad 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml @@ -1215,3 +1215,63 @@ create logsdb data stream with custom sorting on timestamp: index: $backing_index - match: { .$backing_index.mappings.properties.@timestamp.type: date } + +--- +rollover with host as keyword: + - do: + indices.put_index_template: + name: "example" + body: + "index_patterns": [ "example*" ] + "template": + "settings": + "mode": "logsdb" + "mappings": + "properties": + "host": + "type": "text" + "@timestamp": + "type": "date" + "composed_of": [] + "priority": 500 + "data_stream": {} + + - do: + index: + index: "example" + body: + "@timestamp": "2025-01-01T00:00:00.000Z" + "host": "test-host" + + - do: + indices.rollover: + alias: "example" + - match: { rolled_over: true } + + - do: + indices.get_data_stream: + name: "example" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index_0 } + - set: { data_streams.0.indices.1.index_name: backing_index_1 } + + - do: + indices.get_settings: + index: $backing_index_0 + - is_false: .$backing_index_0.settings.index.logsdb.add_host_name_field + + - do: + indices.get_mapping: + index: $backing_index_0 + - match: { .$backing_index_0.mappings.properties.host.properties: null } + + - do: + indices.get_settings: + index: $backing_index_1 + - is_false: .$backing_index_1.settings.index.logsdb.add_host_name_field + + - do: + indices.get_mapping: + index: $backing_index_1 + - match: { .$backing_index_1.mappings.properties.host.properties: null }