diff --git a/docs/changelog/136119.yaml b/docs/changelog/136119.yaml new file mode 100644 index 0000000000000..d900562147824 --- /dev/null +++ b/docs/changelog/136119.yaml @@ -0,0 +1,6 @@ +pr: 136119 +summary: Fix logsdb settings provider mapping filters +area: Logs +type: bug +issues: + - 136107 diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index ed9bd43262f0c..441a3d7abec90 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -48,6 +48,8 @@ import java.util.Objects; import java.util.Set; import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH; import static org.elasticsearch.xpack.logsdb.LogsDBPlugin.CLUSTER_LOGSDB_ENABLED; @@ -55,12 +57,10 @@ final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider { private static final Logger LOGGER = LogManager.getLogger(LogsdbIndexModeSettingsProvider.class); static final String LOGS_PATTERN = "logs-*-*"; - private static final Set MAPPING_INCLUDES = Set.of( - "_doc._source.*", - "_doc.properties.host**", - "_doc.properties.resource**", - "_doc.subobjects" - ); + private static final Set MAPPING_INCLUDES = Set.of("_source.*", "properties.host**", "properties.resource**", "subobjects") + .stream() + .flatMap(v -> Stream.of(v, "_doc." + v)) + .collect(Collectors.toSet()); private final LogsdbLicenseService licenseService; private final SetOnce> mapperServiceFactory = new SetOnce<>(); @@ -305,11 +305,11 @@ MappingHints getMappingHints( @SuppressWarnings("unchecked") private boolean checkMappingForPatternText(Map mapping) { var docMapping = mapping.get("_doc"); - if ((docMapping instanceof Map) == false) { - return false; + if (docMapping instanceof Map) { + mapping = (Map) docMapping; } boolean[] usesPatternText = { false }; - MappingVisitor.visitMapping((Map) docMapping, (field, fieldMapping) -> { + MappingVisitor.visitMapping(mapping, (field, fieldMapping) -> { if (Objects.equals(fieldMapping.get("type"), PatternTextFieldType.CONTENT_TYPE)) { usesPatternText[0] = true; } diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index 92448722a01a1..d8a229877b686 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -59,17 +59,15 @@ public class LogsdbIndexModeSettingsProviderTests extends ESTestCase { private static final String DATA_STREAM_NAME = "logs-app1"; public static final String DEFAULT_MAPPING = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "message": { - "type": "keyword" - }, - "host.name": { - "type": "keyword" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "message": { + "type": "keyword" + }, + "host.name": { + "type": "keyword" } } } @@ -98,6 +96,14 @@ public void setup() throws Exception { basicLogsdbLicenseService.setLicenseService(basicLicenseService); } + private static String getMapping(String contents) { + if (randomBoolean()) { + return "{\"_doc\":" + contents + "}"; + } else { + return contents; + } + } + private LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled) { return withSyntheticSourceDemotionSupport(enabled, Version.CURRENT); } @@ -181,7 +187,7 @@ public void testDisabled() throws IOException { emptyProject(), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -204,7 +210,7 @@ public void testOnIndexCreation() throws IOException { emptyProject(), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -227,7 +233,7 @@ public void testOnExplicitStandardIndex() throws IOException { emptyProject(), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.getName()).build(), - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -250,7 +256,7 @@ public void testOnExplicitTimeSeriesIndex() throws IOException { emptyProject(), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()).build(), - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -273,7 +279,7 @@ public void testNonLogsDataStream() throws IOException { emptyProject(), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -292,7 +298,7 @@ public void testWithoutLogsComponentTemplate() throws IOException { buildMetadata(List.of("*"), List.of()), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -311,7 +317,7 @@ public void testWithLogsComponentTemplate() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -330,7 +336,7 @@ public void testWithMultipleComponentTemplates() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings", "logs@custom")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -349,7 +355,7 @@ public void testWithCustomComponentTemplatesOnly() throws IOException { buildMetadata(List.of("*"), List.of("logs@custom", "custom-component-template")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -368,7 +374,7 @@ public void testNonMatchingTemplateIndexPattern() throws IOException { buildMetadata(List.of("standard-apache-production"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -391,7 +397,7 @@ public void testCaseSensitivity() throws IOException { emptyProject(), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -411,7 +417,7 @@ public void testMultipleHyphensInDataStreamName() throws IOException { emptyProject(), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -430,7 +436,7 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -448,7 +454,7 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -466,7 +472,7 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -477,7 +483,7 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { private static ProjectMetadata buildMetadata(final List indexPatterns, final List componentTemplates) throws IOException { - final Template template = new Template(Settings.EMPTY, new CompressedXContent(DEFAULT_MAPPING), null); + final Template template = new Template(Settings.EMPTY, new CompressedXContent(getMapping(DEFAULT_MAPPING)), null); final ComposableIndexTemplate composableTemplate = ComposableIndexTemplate.builder() .indexPatterns(indexPatterns) .template(template) @@ -502,19 +508,17 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { { String mapping = """ { - "_doc": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "my_field": { - "type": "keyword" - } + "_source": { + "mode": "synthetic" + }, + "properties": { + "my_field": { + "type": "keyword" } } } """; - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse("_source.mode is a noop", result); assertThat(newMapperServiceCounter.get(), equalTo(1)); @@ -526,14 +530,12 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { if (withSourceMode) { mapping = """ { - "_doc": { - "_source": { - "mode": "stored" - }, - "properties": { - "my_field": { - "type": "keyword" - } + "_source": { + "mode": "stored" + }, + "properties": { + "my_field": { + "type": "keyword" } } } @@ -541,17 +543,15 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { } else { mapping = """ { - "_doc": { - "properties": { - "my_field": { - "type": "keyword" - } + "properties": { + "my_field": { + "type": "keyword" } } } """; } - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(2)); @@ -565,21 +565,19 @@ public void testValidateIndexName() throws IOException { String indexName = MetadataIndexTemplateService.VALIDATE_INDEX_NAME; String mapping = """ { - "_doc": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "my_field": { - "type": "keyword" - } + "_source": { + "mode": "synthetic" + }, + "properties": { + "my_field": { + "type": "keyword" } } } """; Settings settings = Settings.EMPTY; LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); } @@ -589,11 +587,9 @@ public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); String mapping = """ { - "_doc": { - "properties": { - "my_field": { - "type": "keyword" - } + "properties": { + "my_field": { + "type": "keyword" } } } @@ -601,7 +597,7 @@ public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); { Settings settings = Settings.builder().put("index.mode", "logsdb").build(); - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertTrue(result); assertThat(newMapperServiceCounter.get(), equalTo(1)); @@ -618,7 +614,7 @@ public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException assertThat(newMapperServiceCounter.get(), equalTo(3)); } { - boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(4)); @@ -630,12 +626,10 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); String mapping = """ { - "_doc": { - "properties": { - "my_field": { - "type": "keyword", - "time_series_dimension": true - } + "properties": { + "my_field": { + "type": "keyword", + "time_series_dimension": true } } } @@ -643,7 +637,7 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); { Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build(); - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertTrue(result); } @@ -657,7 +651,7 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { assertFalse(result); } { - boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); } @@ -671,19 +665,17 @@ public void testNewIndexHasSyntheticSourceUsageInvalidSettings() throws IOExcept { String mapping = """ { - "_doc": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "my_field": { - "type": "keyword" - } + "_source": { + "mode": "synthetic" + }, + "properties": { + "my_field": { + "type": "keyword" } } } """; - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(1)); @@ -691,16 +683,14 @@ public void testNewIndexHasSyntheticSourceUsageInvalidSettings() throws IOExcept { String mapping = """ { - "_doc": { - "properties": { - "my_field": { - "type": "keyword" - } + "properties": { + "my_field": { + "type": "keyword" } } } """; - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(2)); @@ -983,10 +973,10 @@ public void testExplicitRoutingPathNotAllowedByLicense() throws Exception { public void testPatternTextNotAllowedByLicense() throws IOException { String[] patternTextLicenceCheckedFieldMappings = { - "{\"_doc\":{\"properties\":{\"message\":{\"type\":\"pattern_text\"}}}}", - "{\"_doc\":{\"properties\":{\"error\":{\"properties\":{\"message\":{\"type\":\"pattern_text\"}}}}}}", - "{\"_doc\":{\"properties\":{\"foo\":{\"type\":\"pattern_text\"}}}}", - "{\"_doc\":{\"properties\":{\"bar\":{\"properties\":{\"baz\":{\"type\":\"pattern_text\"}}}}}}" }; + "{\"properties\":{\"message\":{\"type\":\"pattern_text\"}}}", + "{\"properties\":{\"error\":{\"properties\":{\"message\":{\"type\":\"pattern_text\"}}}}}", + "{\"properties\":{\"foo\":{\"type\":\"pattern_text\"}}}", + "{\"properties\":{\"bar\":{\"properties\":{\"baz\":{\"type\":\"pattern_text\"}}}}}" }; var expectedSettings = Settings.builder() .put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true) @@ -995,7 +985,7 @@ public void testPatternTextNotAllowedByLicense() throws IOException { .build(); for (String mapping : patternTextLicenceCheckedFieldMappings) { - var result = generateLogsdbSettings(Settings.EMPTY, mapping, Version.CURRENT, basicLogsdbLicenseService); + var result = generateLogsdbSettings(Settings.EMPTY, getMapping(mapping), Version.CURRENT, basicLogsdbLicenseService); assertEquals(expectedSettings, result); } } @@ -1038,16 +1028,14 @@ public void testSortAndHostNoHost() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - } + "properties": { + "@timestamp": { + "type": "date" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1057,16 +1045,14 @@ public void testSortAndHostNoHostOldNode() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - } + "properties": { + "@timestamp": { + "type": "date" } } } """; - Settings result = generateLogsdbSettings(settings, mappings, Version.V_8_17_0); + Settings result = generateLogsdbSettings(settings, getMapping(mappings), Version.V_8_17_0); assertTrue(result.isEmpty()); } @@ -1074,19 +1060,17 @@ public void testSortAndHostNameKeyword() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "keyword" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "keyword" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1096,20 +1080,18 @@ public void testSortAndHostNameKeywordNoDocvalues() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "keyword", - "doc_values": false - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "keyword", + "doc_values": false } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1119,19 +1101,17 @@ public void testSortAndHostNameInteger() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "integer" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "integer" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1141,20 +1121,18 @@ public void testSortAndHostNameIntegerNoDocvalues() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "integer", - "doc_values": false - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "integer", + "doc_values": false } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1164,19 +1142,17 @@ public void testSortAndHostNameBoolean() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "boolean" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "boolean" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1186,19 +1162,17 @@ public void testSortAndHostObject() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host": { - "type": "object" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "type": "object" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1208,19 +1182,17 @@ public void testSortAndHostField() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host": { - "type": "keyword" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "type": "keyword" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1230,20 +1202,18 @@ public void testSortAndHostFieldSubobjectsFalse() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "subobjects": false, - "properties": { - "@timestamp": { - "type": "date" - }, - "host": { - "type": "keyword" - } + "subobjects": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "type": "keyword" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1256,19 +1226,17 @@ public void testSortAndHostNameObject() throws Exception { .build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name.sub": { - "type": "keyword" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name.sub": { + "type": "keyword" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1281,31 +1249,29 @@ public void testSortAndHostNameAlias() throws Exception { .build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "resource": { - "properties": { - "attributes": { - "properties": { - "host.arch": { - "type": "keyword" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "resource": { + "properties": { + "attributes": { + "properties": { + "host.arch": { + "type": "keyword" } } } - }, - "host.architecture": { - "type": "alias", - "path": "resource.attributes.host.arch" } + }, + "host.architecture": { + "type": "alias", + "path": "resource.attributes.host.arch" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1318,11 +1284,9 @@ public void testSortFastRefresh() throws Exception { .build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - } + "properties": { + "@timestamp": { + "type": "date" } } } @@ -1348,7 +1312,7 @@ public void testSortFastRefresh() throws Exception { emptyProject(), Instant.now(), settings, - List.of(new CompressedXContent(mappings)), + List.of(new CompressedXContent(getMapping(mappings))), IndexVersion.current(), settingsBuilder ); diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml index e2b426775745e..757b30806a827 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml @@ -1241,3 +1241,63 @@ create logsdb data stream with timestamp mixed date_nanos and date fields: - match: { columns.1.type: "date_nanos" } - length: { values: 1 } - match: { values.0: [ "2025-09-09T14:00:00.000Z", "2025-09-09T00:00:00.000Z" ] } + +--- +rollover with host as keyword: + - do: + indices.put_index_template: + name: "example" + body: + "index_patterns": [ "example*" ] + "template": + "settings": + "mode": "logsdb" + "mappings": + "properties": + "host": + "type": "text" + "@timestamp": + "type": "date" + "composed_of": [] + "priority": 500 + "data_stream": {} + + - do: + index: + index: "example" + body: + "@timestamp": "2025-01-01T00:00:00.000Z" + "host": "test-host" + + - do: + indices.rollover: + alias: "example" + - match: { rolled_over: true } + + - do: + indices.get_data_stream: + name: "example" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index_0 } + - set: { data_streams.0.indices.1.index_name: backing_index_1 } + + - do: + indices.get_settings: + index: $backing_index_0 + - is_false: .$backing_index_0.settings.index.logsdb.add_host_name_field + + - do: + indices.get_mapping: + index: $backing_index_0 + - match: { .$backing_index_0.mappings.properties.host.properties: null } + + - do: + indices.get_settings: + index: $backing_index_1 + - is_false: .$backing_index_1.settings.index.logsdb.add_host_name_field + + - do: + indices.get_mapping: + index: $backing_index_1 + - match: { .$backing_index_1.mappings.properties.host.properties: null }