From 239c7de000c9a6e3bce0f1caa0e80359055382f7 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Wed, 8 Oct 2025 07:24:14 -0700 Subject: [PATCH] Fix logsdb settings provider mapping filters (#136119) Fixes #136107 (cherry picked from commit d51372358baa760a2d5ae4f6ebfc73c1a1cdbc52) # Conflicts: # x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java # x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java # x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml --- docs/changelog/136119.yaml | 6 + .../LogsdbIndexModeSettingsProvider.java | 7 +- .../LogsdbIndexModeSettingsProviderTests.java | 348 ++++++++---------- .../test/30_logsdb_default_mapping.yml | 60 +++ 4 files changed, 229 insertions(+), 192 deletions(-) create mode 100644 docs/changelog/136119.yaml diff --git a/docs/changelog/136119.yaml b/docs/changelog/136119.yaml new file mode 100644 index 0000000000000..d900562147824 --- /dev/null +++ b/docs/changelog/136119.yaml @@ -0,0 +1,6 @@ +pr: 136119 +summary: Fix logsdb settings provider mapping filters +area: Logs +type: bug +issues: + - 136107 diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index d990a43148427..9c13ec4a72e0c 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -41,6 +41,8 @@ import java.util.Locale; import java.util.Set; import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH; import static org.elasticsearch.xpack.logsdb.LogsDBPlugin.CLUSTER_LOGSDB_ENABLED; @@ -48,7 +50,10 @@ final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider { private static final Logger LOGGER = LogManager.getLogger(LogsdbIndexModeSettingsProvider.class); static final String LOGS_PATTERN = "logs-*-*"; - private static final Set MAPPING_INCLUDES = Set.of("_doc._source.*", "_doc.properties.host**", "_doc.subobjects"); + private static final Set MAPPING_INCLUDES = Set.of("_source.*", "properties.host**", "subobjects") + .stream() + .flatMap(v -> Stream.of(v, "_doc." + v)) + .collect(Collectors.toSet()); private final LogsdbLicenseService licenseService; private final SetOnce> mapperServiceFactory = new SetOnce<>(); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index 1d6f43f370a3c..2c71ffafd651d 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -52,17 +52,15 @@ public class LogsdbIndexModeSettingsProviderTests extends ESTestCase { private static final String DATA_STREAM_NAME = "logs-app1"; public static final String DEFAULT_MAPPING = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "message": { - "type": "keyword" - }, - "host.name": { - "type": "keyword" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "message": { + "type": "keyword" + }, + "host.name": { + "type": "keyword" } } } @@ -83,6 +81,14 @@ public void setup() throws Exception { logsdbLicenseService.setLicenseService(mockLicenseService); } + private static String getMapping(String contents) { + if (randomBoolean()) { + return "{\"_doc\":" + contents + "}"; + } else { + return contents; + } + } + private LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled) { return withSyntheticSourceDemotionSupport(enabled, Version.CURRENT); } @@ -152,7 +158,7 @@ public void testDisabled() throws IOException { Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertTrue(additionalIndexSettings.isEmpty()); @@ -171,7 +177,7 @@ public void testOnIndexCreation() throws IOException { Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertTrue(additionalIndexSettings.isEmpty()); @@ -190,7 +196,7 @@ public void testOnExplicitStandardIndex() throws IOException { Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.getName()).build(), - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertTrue(additionalIndexSettings.isEmpty()); @@ -209,7 +215,7 @@ public void testOnExplicitTimeSeriesIndex() throws IOException { Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()).build(), - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertTrue(additionalIndexSettings.isEmpty()); @@ -228,7 +234,7 @@ public void testNonLogsDataStream() throws IOException { Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertTrue(additionalIndexSettings.isEmpty()); @@ -243,7 +249,7 @@ public void testWithoutLogsComponentTemplate() throws IOException { buildMetadata(List.of("*"), List.of()), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); @@ -258,7 +264,7 @@ public void testWithLogsComponentTemplate() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); @@ -273,7 +279,7 @@ public void testWithMultipleComponentTemplates() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings", "logs@custom")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); @@ -288,7 +294,7 @@ public void testWithCustomComponentTemplatesOnly() throws IOException { buildMetadata(List.of("*"), List.of("logs@custom", "custom-component-template")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); @@ -303,7 +309,7 @@ public void testNonMatchingTemplateIndexPattern() throws IOException { buildMetadata(List.of("standard-apache-production"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); @@ -322,7 +328,7 @@ public void testCaseSensitivity() throws IOException { Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertTrue(additionalIndexSettings.isEmpty()); @@ -338,7 +344,7 @@ public void testMultipleHyphensInDataStreamName() throws IOException { Metadata.EMPTY_METADATA, Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertIndexMode(additionalIndexSettings, IndexMode.LOGSDB.getName()); @@ -353,7 +359,7 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertTrue(beforeSettings.isEmpty()); @@ -367,7 +373,7 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertIndexMode(afterSettings, IndexMode.LOGSDB.getName()); @@ -381,14 +387,14 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)) + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))) ); assertTrue(laterSettings.isEmpty()); } private static Metadata buildMetadata(final List indexPatterns, final List componentTemplates) throws IOException { - final Template template = new Template(Settings.EMPTY, new CompressedXContent(DEFAULT_MAPPING), null); + final Template template = new Template(Settings.EMPTY, new CompressedXContent(getMapping(DEFAULT_MAPPING)), null); final ComposableIndexTemplate composableTemplate = ComposableIndexTemplate.builder() .indexPatterns(indexPatterns) .template(template) @@ -413,19 +419,17 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { { String mapping = """ { - "_doc": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "my_field": { - "type": "keyword" - } + "_source": { + "mode": "synthetic" + }, + "properties": { + "my_field": { + "type": "keyword" } } } """; - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse("_source.mode is a noop", result); assertThat(newMapperServiceCounter.get(), equalTo(1)); @@ -437,14 +441,12 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { if (withSourceMode) { mapping = """ { - "_doc": { - "_source": { - "mode": "stored" - }, - "properties": { - "my_field": { - "type": "keyword" - } + "_source": { + "mode": "stored" + }, + "properties": { + "my_field": { + "type": "keyword" } } } @@ -452,17 +454,15 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { } else { mapping = """ { - "_doc": { - "properties": { - "my_field": { - "type": "keyword" - } + "properties": { + "my_field": { + "type": "keyword" } } } """; } - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(2)); @@ -476,21 +476,19 @@ public void testValidateIndexName() throws IOException { String indexName = "validate-index-name"; String mapping = """ { - "_doc": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "my_field": { - "type": "keyword" - } + "_source": { + "mode": "synthetic" + }, + "properties": { + "my_field": { + "type": "keyword" } } } """; Settings settings = Settings.EMPTY; LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); } @@ -500,11 +498,9 @@ public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); String mapping = """ { - "_doc": { - "properties": { - "my_field": { - "type": "keyword" - } + "properties": { + "my_field": { + "type": "keyword" } } } @@ -512,7 +508,7 @@ public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); { Settings settings = Settings.builder().put("index.mode", "logsdb").build(); - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertTrue(result); assertThat(newMapperServiceCounter.get(), equalTo(1)); @@ -529,7 +525,7 @@ public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException assertThat(newMapperServiceCounter.get(), equalTo(3)); } { - boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(4)); @@ -541,12 +537,10 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); String mapping = """ { - "_doc": { - "properties": { - "my_field": { - "type": "keyword", - "time_series_dimension": true - } + "properties": { + "my_field": { + "type": "keyword", + "time_series_dimension": true } } } @@ -554,7 +548,7 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); { Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build(); - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertTrue(result); } @@ -568,7 +562,7 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { assertFalse(result); } { - boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); } @@ -582,19 +576,17 @@ public void testNewIndexHasSyntheticSourceUsageInvalidSettings() throws IOExcept { String mapping = """ { - "_doc": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "my_field": { - "type": "keyword" - } + "_source": { + "mode": "synthetic" + }, + "properties": { + "my_field": { + "type": "keyword" } } } """; - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(1)); @@ -602,16 +594,14 @@ public void testNewIndexHasSyntheticSourceUsageInvalidSettings() throws IOExcept { String mapping = """ { - "_doc": { - "properties": { - "my_field": { - "type": "keyword" - } + "properties": { + "my_field": { + "type": "keyword" } } } """; - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(2)); @@ -899,16 +889,14 @@ public void testSortAndHostNoHost() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - } + "properties": { + "@timestamp": { + "type": "date" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -918,16 +906,14 @@ public void testSortAndHostNoHostOldNode() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - } + "properties": { + "@timestamp": { + "type": "date" } } } """; - Settings result = generateLogsdbSettings(settings, mappings, Version.V_8_17_0); + Settings result = generateLogsdbSettings(settings, getMapping(mappings), Version.V_8_17_0); assertTrue(result.isEmpty()); } @@ -935,19 +921,17 @@ public void testSortAndHostNameKeyword() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "keyword" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "keyword" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -957,20 +941,18 @@ public void testSortAndHostNameKeywordNoDocvalues() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "keyword", - "doc_values": false - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "keyword", + "doc_values": false } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -980,19 +962,17 @@ public void testSortAndHostNameInteger() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "integer" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "integer" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1002,20 +982,18 @@ public void testSortAndHostNameIntegerNoDocvalues() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "integer", - "doc_values": false - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "integer", + "doc_values": false } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1025,19 +1003,17 @@ public void testSortAndHostNameBoolean() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "boolean" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "boolean" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1047,19 +1023,17 @@ public void testSortAndHostObject() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host": { - "type": "object" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "type": "object" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1069,19 +1043,17 @@ public void testSortAndHostField() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host": { - "type": "keyword" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "type": "keyword" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1091,20 +1063,18 @@ public void testSortAndHostFieldSubobjectsFalse() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "subobjects": false, - "properties": { - "@timestamp": { - "type": "date" - }, - "host": { - "type": "keyword" - } + "subobjects": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "type": "keyword" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1117,19 +1087,17 @@ public void testSortAndHostNameObject() throws Exception { .build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name.sub": { - "type": "keyword" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name.sub": { + "type": "keyword" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1142,11 +1110,9 @@ public void testSortFastRefresh() throws Exception { .build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - } + "properties": { + "@timestamp": { + "type": "date" } } } @@ -1172,7 +1138,7 @@ public void testSortFastRefresh() throws Exception { metadata, Instant.now(), settings, - List.of(new CompressedXContent(mappings)) + List.of(new CompressedXContent(getMapping(mappings))) ); Settings result = builder().put(additionalIndexSettings).build(); diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml index b49ada6e1809c..86fc634eb1361 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml @@ -1143,3 +1143,63 @@ create logsdb data stream with custom sorting on timestamp: index: $backing_index - match: { .$backing_index.mappings.properties.@timestamp.type: date } + +--- +rollover with host as keyword: + - do: + indices.put_index_template: + name: "example" + body: + "index_patterns": [ "example*" ] + "template": + "settings": + "mode": "logsdb" + "mappings": + "properties": + "host": + "type": "text" + "@timestamp": + "type": "date" + "composed_of": [] + "priority": 500 + "data_stream": {} + + - do: + index: + index: "example" + body: + "@timestamp": "2025-01-01T00:00:00.000Z" + "host": "test-host" + + - do: + indices.rollover: + alias: "example" + - match: { rolled_over: true } + + - do: + indices.get_data_stream: + name: "example" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index_0 } + - set: { data_streams.0.indices.1.index_name: backing_index_1 } + + - do: + indices.get_settings: + index: $backing_index_0 + - is_false: .$backing_index_0.settings.index.logsdb.add_host_name_field + + - do: + indices.get_mapping: + index: $backing_index_0 + - match: { .$backing_index_0.mappings.properties.host.properties: null } + + - do: + indices.get_settings: + index: $backing_index_1 + - is_false: .$backing_index_1.settings.index.logsdb.add_host_name_field + + - do: + indices.get_mapping: + index: $backing_index_1 + - match: { .$backing_index_1.mappings.properties.host.properties: null }