From d24c40fd659d5c78b8ba198a18a91f81560f3419 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Mon, 6 Oct 2025 12:39:21 -0700 Subject: [PATCH 1/4] Add tests reproducing issue --- .../PatternTextBasicLicenseTests.java | 75 ++++ .../LogsdbIndexModeSettingsProviderTests.java | 390 ++++++++---------- .../test/30_logsdb_default_mapping.yml | 60 +++ 3 files changed, 312 insertions(+), 213 deletions(-) create mode 100644 x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextBasicLicenseTests.java diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextBasicLicenseTests.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextBasicLicenseTests.java new file mode 100644 index 0000000000000..f6e72eb23a978 --- /dev/null +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextBasicLicenseTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb.patterntext; + +import org.elasticsearch.xpack.logsdb.DataStreamLicenseChangeTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.Map; + +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasKey; + +public class PatternTextBasicLicenseTests extends DataStreamLicenseChangeTestCase { + @Before + public void checkClusterFeature() { + assumeTrue("[patterned_text] must be available", clusterHasFeature("mapper.patterned_text")); + } + + private static final String patternTextMapping = """ + { + "index_patterns": ["%name%"], + "priority": 500, + "data_stream": {}, + "template": { + "mappings": { + "properties": { + "pattern_field": { + "type": "pattern_text" + } + } + } + } + }"""; + + @SuppressWarnings("unchecked") + public void testLicenseUpgrade() throws IOException { + final String dataStreamName = "test-foo"; + + final String doc = """ + {"index": {}} + {"@timestamp": "2025-01-01T00:00:00.000Z", "pattern_field": "foo"} + """; + + assertOK(putTemplate(client(), "logs@custom", patternTextMapping.replace("%name%", dataStreamName))); + assertOK(bulkIndex(client(), dataStreamName, () -> doc)); + + String backingIndex0 = getDataStreamBackingIndex(client(), dataStreamName, 0); + { + assertEquals("true", getSetting(client(), backingIndex0, "index.mapping.pattern_text.disable_templating")); + Map mapping = getMapping(client(), backingIndex0); + Map patternFieldMapping = (Map) ((Map) mapping.get("properties")).get( + "pattern_field" + ); + assertThat(patternFieldMapping, hasEntry("disable_templating", true)); + } + + assertOK(rolloverDataStream(client(), dataStreamName)); + + String backingIndex1 = getDataStreamBackingIndex(client(), dataStreamName, 1); + { + assertEquals("true", getSetting(client(), backingIndex1, "index.mapping.pattern_text.disable_templating")); + Map mapping = getMapping(client(), backingIndex1); + Map patternFieldMapping = (Map) ((Map) mapping.get("properties")).get( + "pattern_field" + ); + assertThat(patternFieldMapping, hasKey("disable_templating")); + } + } +} diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index 92448722a01a1..d8a229877b686 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -59,17 +59,15 @@ public class LogsdbIndexModeSettingsProviderTests extends ESTestCase { private static final String DATA_STREAM_NAME = "logs-app1"; public static final String DEFAULT_MAPPING = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "message": { - "type": "keyword" - }, - "host.name": { - "type": "keyword" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "message": { + "type": "keyword" + }, + "host.name": { + "type": "keyword" } } } @@ -98,6 +96,14 @@ public void setup() throws Exception { basicLogsdbLicenseService.setLicenseService(basicLicenseService); } + private static String getMapping(String contents) { + if (randomBoolean()) { + return "{\"_doc\":" + contents + "}"; + } else { + return contents; + } + } + private LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled) { return withSyntheticSourceDemotionSupport(enabled, Version.CURRENT); } @@ -181,7 +187,7 @@ public void testDisabled() throws IOException { emptyProject(), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -204,7 +210,7 @@ public void testOnIndexCreation() throws IOException { emptyProject(), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -227,7 +233,7 @@ public void testOnExplicitStandardIndex() throws IOException { emptyProject(), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.getName()).build(), - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -250,7 +256,7 @@ public void testOnExplicitTimeSeriesIndex() throws IOException { emptyProject(), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName()).build(), - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -273,7 +279,7 @@ public void testNonLogsDataStream() throws IOException { emptyProject(), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -292,7 +298,7 @@ public void testWithoutLogsComponentTemplate() throws IOException { buildMetadata(List.of("*"), List.of()), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -311,7 +317,7 @@ public void testWithLogsComponentTemplate() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -330,7 +336,7 @@ public void testWithMultipleComponentTemplates() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings", "logs@custom")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -349,7 +355,7 @@ public void testWithCustomComponentTemplatesOnly() throws IOException { buildMetadata(List.of("*"), List.of("logs@custom", "custom-component-template")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -368,7 +374,7 @@ public void testNonMatchingTemplateIndexPattern() throws IOException { buildMetadata(List.of("standard-apache-production"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -391,7 +397,7 @@ public void testCaseSensitivity() throws IOException { emptyProject(), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -411,7 +417,7 @@ public void testMultipleHyphensInDataStreamName() throws IOException { emptyProject(), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -430,7 +436,7 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -448,7 +454,7 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -466,7 +472,7 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { buildMetadata(List.of("*"), List.of("logs@settings")), Instant.now().truncatedTo(ChronoUnit.SECONDS), Settings.EMPTY, - List.of(new CompressedXContent(DEFAULT_MAPPING)), + List.of(new CompressedXContent(getMapping(DEFAULT_MAPPING))), IndexVersion.current(), settingsBuilder ); @@ -477,7 +483,7 @@ public void testBeforeAndAfterSettingUpdate() throws IOException { private static ProjectMetadata buildMetadata(final List indexPatterns, final List componentTemplates) throws IOException { - final Template template = new Template(Settings.EMPTY, new CompressedXContent(DEFAULT_MAPPING), null); + final Template template = new Template(Settings.EMPTY, new CompressedXContent(getMapping(DEFAULT_MAPPING)), null); final ComposableIndexTemplate composableTemplate = ComposableIndexTemplate.builder() .indexPatterns(indexPatterns) .template(template) @@ -502,19 +508,17 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { { String mapping = """ { - "_doc": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "my_field": { - "type": "keyword" - } + "_source": { + "mode": "synthetic" + }, + "properties": { + "my_field": { + "type": "keyword" } } } """; - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse("_source.mode is a noop", result); assertThat(newMapperServiceCounter.get(), equalTo(1)); @@ -526,14 +530,12 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { if (withSourceMode) { mapping = """ { - "_doc": { - "_source": { - "mode": "stored" - }, - "properties": { - "my_field": { - "type": "keyword" - } + "_source": { + "mode": "stored" + }, + "properties": { + "my_field": { + "type": "keyword" } } } @@ -541,17 +543,15 @@ public void testNewIndexHasSyntheticSourceUsage() throws IOException { } else { mapping = """ { - "_doc": { - "properties": { - "my_field": { - "type": "keyword" - } + "properties": { + "my_field": { + "type": "keyword" } } } """; } - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(2)); @@ -565,21 +565,19 @@ public void testValidateIndexName() throws IOException { String indexName = MetadataIndexTemplateService.VALIDATE_INDEX_NAME; String mapping = """ { - "_doc": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "my_field": { - "type": "keyword" - } + "_source": { + "mode": "synthetic" + }, + "properties": { + "my_field": { + "type": "keyword" } } } """; Settings settings = Settings.EMPTY; LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); } @@ -589,11 +587,9 @@ public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); String mapping = """ { - "_doc": { - "properties": { - "my_field": { - "type": "keyword" - } + "properties": { + "my_field": { + "type": "keyword" } } } @@ -601,7 +597,7 @@ public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); { Settings settings = Settings.builder().put("index.mode", "logsdb").build(); - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertTrue(result); assertThat(newMapperServiceCounter.get(), equalTo(1)); @@ -618,7 +614,7 @@ public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException assertThat(newMapperServiceCounter.get(), equalTo(3)); } { - boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(4)); @@ -630,12 +626,10 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); String mapping = """ { - "_doc": { - "properties": { - "my_field": { - "type": "keyword", - "time_series_dimension": true - } + "properties": { + "my_field": { + "type": "keyword", + "time_series_dimension": true } } } @@ -643,7 +637,7 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); { Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build(); - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertTrue(result); } @@ -657,7 +651,7 @@ public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { assertFalse(result); } { - boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, Settings.EMPTY, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); } @@ -671,19 +665,17 @@ public void testNewIndexHasSyntheticSourceUsageInvalidSettings() throws IOExcept { String mapping = """ { - "_doc": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "my_field": { - "type": "keyword" - } + "_source": { + "mode": "synthetic" + }, + "properties": { + "my_field": { + "type": "keyword" } } } """; - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(1)); @@ -691,16 +683,14 @@ public void testNewIndexHasSyntheticSourceUsageInvalidSettings() throws IOExcept { String mapping = """ { - "_doc": { - "properties": { - "my_field": { - "type": "keyword" - } + "properties": { + "my_field": { + "type": "keyword" } } } """; - boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(mapping))) + boolean result = provider.getMappingHints(indexName, null, settings, List.of(new CompressedXContent(getMapping(mapping)))) .hasSyntheticSourceUsage(); assertFalse(result); assertThat(newMapperServiceCounter.get(), equalTo(2)); @@ -983,10 +973,10 @@ public void testExplicitRoutingPathNotAllowedByLicense() throws Exception { public void testPatternTextNotAllowedByLicense() throws IOException { String[] patternTextLicenceCheckedFieldMappings = { - "{\"_doc\":{\"properties\":{\"message\":{\"type\":\"pattern_text\"}}}}", - "{\"_doc\":{\"properties\":{\"error\":{\"properties\":{\"message\":{\"type\":\"pattern_text\"}}}}}}", - "{\"_doc\":{\"properties\":{\"foo\":{\"type\":\"pattern_text\"}}}}", - "{\"_doc\":{\"properties\":{\"bar\":{\"properties\":{\"baz\":{\"type\":\"pattern_text\"}}}}}}" }; + "{\"properties\":{\"message\":{\"type\":\"pattern_text\"}}}", + "{\"properties\":{\"error\":{\"properties\":{\"message\":{\"type\":\"pattern_text\"}}}}}", + "{\"properties\":{\"foo\":{\"type\":\"pattern_text\"}}}", + "{\"properties\":{\"bar\":{\"properties\":{\"baz\":{\"type\":\"pattern_text\"}}}}}" }; var expectedSettings = Settings.builder() .put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true) @@ -995,7 +985,7 @@ public void testPatternTextNotAllowedByLicense() throws IOException { .build(); for (String mapping : patternTextLicenceCheckedFieldMappings) { - var result = generateLogsdbSettings(Settings.EMPTY, mapping, Version.CURRENT, basicLogsdbLicenseService); + var result = generateLogsdbSettings(Settings.EMPTY, getMapping(mapping), Version.CURRENT, basicLogsdbLicenseService); assertEquals(expectedSettings, result); } } @@ -1038,16 +1028,14 @@ public void testSortAndHostNoHost() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - } + "properties": { + "@timestamp": { + "type": "date" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1057,16 +1045,14 @@ public void testSortAndHostNoHostOldNode() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - } + "properties": { + "@timestamp": { + "type": "date" } } } """; - Settings result = generateLogsdbSettings(settings, mappings, Version.V_8_17_0); + Settings result = generateLogsdbSettings(settings, getMapping(mappings), Version.V_8_17_0); assertTrue(result.isEmpty()); } @@ -1074,19 +1060,17 @@ public void testSortAndHostNameKeyword() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "keyword" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "keyword" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1096,20 +1080,18 @@ public void testSortAndHostNameKeywordNoDocvalues() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "keyword", - "doc_values": false - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "keyword", + "doc_values": false } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1119,19 +1101,17 @@ public void testSortAndHostNameInteger() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "integer" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "integer" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1141,20 +1121,18 @@ public void testSortAndHostNameIntegerNoDocvalues() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "integer", - "doc_values": false - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "integer", + "doc_values": false } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1164,19 +1142,17 @@ public void testSortAndHostNameBoolean() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name": { - "type": "boolean" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "boolean" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1186,19 +1162,17 @@ public void testSortAndHostObject() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host": { - "type": "object" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "type": "object" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1208,19 +1182,17 @@ public void testSortAndHostField() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host": { - "type": "keyword" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "type": "keyword" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1230,20 +1202,18 @@ public void testSortAndHostFieldSubobjectsFalse() throws Exception { var settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB).build(); var mappings = """ { - "_doc": { - "subobjects": false, - "properties": { - "@timestamp": { - "type": "date" - }, - "host": { - "type": "keyword" - } + "subobjects": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "type": "keyword" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1256,19 +1226,17 @@ public void testSortAndHostNameObject() throws Exception { .build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "host.name.sub": { - "type": "keyword" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name.sub": { + "type": "keyword" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertFalse(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertFalse(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1281,31 +1249,29 @@ public void testSortAndHostNameAlias() throws Exception { .build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - }, - "resource": { - "properties": { - "attributes": { - "properties": { - "host.arch": { - "type": "keyword" - } + "properties": { + "@timestamp": { + "type": "date" + }, + "resource": { + "properties": { + "attributes": { + "properties": { + "host.arch": { + "type": "keyword" } } } - }, - "host.architecture": { - "type": "alias", - "path": "resource.attributes.host.arch" } + }, + "host.architecture": { + "type": "alias", + "path": "resource.attributes.host.arch" } } } """; - Settings result = generateLogsdbSettings(settings, mappings); + Settings result = generateLogsdbSettings(settings, getMapping(mappings)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertEquals(1, newMapperServiceCounter.get()); @@ -1318,11 +1284,9 @@ public void testSortFastRefresh() throws Exception { .build(); var mappings = """ { - "_doc": { - "properties": { - "@timestamp": { - "type": "date" - } + "properties": { + "@timestamp": { + "type": "date" } } } @@ -1348,7 +1312,7 @@ public void testSortFastRefresh() throws Exception { emptyProject(), Instant.now(), settings, - List.of(new CompressedXContent(mappings)), + List.of(new CompressedXContent(getMapping(mappings))), IndexVersion.current(), settingsBuilder ); diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml index e2b426775745e..757b30806a827 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml @@ -1241,3 +1241,63 @@ create logsdb data stream with timestamp mixed date_nanos and date fields: - match: { columns.1.type: "date_nanos" } - length: { values: 1 } - match: { values.0: [ "2025-09-09T14:00:00.000Z", "2025-09-09T00:00:00.000Z" ] } + +--- +rollover with host as keyword: + - do: + indices.put_index_template: + name: "example" + body: + "index_patterns": [ "example*" ] + "template": + "settings": + "mode": "logsdb" + "mappings": + "properties": + "host": + "type": "text" + "@timestamp": + "type": "date" + "composed_of": [] + "priority": 500 + "data_stream": {} + + - do: + index: + index: "example" + body: + "@timestamp": "2025-01-01T00:00:00.000Z" + "host": "test-host" + + - do: + indices.rollover: + alias: "example" + - match: { rolled_over: true } + + - do: + indices.get_data_stream: + name: "example" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index_0 } + - set: { data_streams.0.indices.1.index_name: backing_index_1 } + + - do: + indices.get_settings: + index: $backing_index_0 + - is_false: .$backing_index_0.settings.index.logsdb.add_host_name_field + + - do: + indices.get_mapping: + index: $backing_index_0 + - match: { .$backing_index_0.mappings.properties.host.properties: null } + + - do: + indices.get_settings: + index: $backing_index_1 + - is_false: .$backing_index_1.settings.index.logsdb.add_host_name_field + + - do: + indices.get_mapping: + index: $backing_index_1 + - match: { .$backing_index_1.mappings.properties.host.properties: null } From fae8ec678a8c70d3855ab9464bd4f6aa277f54a4 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Tue, 7 Oct 2025 10:34:09 -0700 Subject: [PATCH 2/4] Update mapping filters to optionally include _doc --- .../LogsdbIndexModeSettingsProvider.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index ed9bd43262f0c..441a3d7abec90 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -48,6 +48,8 @@ import java.util.Objects; import java.util.Set; import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH; import static org.elasticsearch.xpack.logsdb.LogsDBPlugin.CLUSTER_LOGSDB_ENABLED; @@ -55,12 +57,10 @@ final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider { private static final Logger LOGGER = LogManager.getLogger(LogsdbIndexModeSettingsProvider.class); static final String LOGS_PATTERN = "logs-*-*"; - private static final Set MAPPING_INCLUDES = Set.of( - "_doc._source.*", - "_doc.properties.host**", - "_doc.properties.resource**", - "_doc.subobjects" - ); + private static final Set MAPPING_INCLUDES = Set.of("_source.*", "properties.host**", "properties.resource**", "subobjects") + .stream() + .flatMap(v -> Stream.of(v, "_doc." + v)) + .collect(Collectors.toSet()); private final LogsdbLicenseService licenseService; private final SetOnce> mapperServiceFactory = new SetOnce<>(); @@ -305,11 +305,11 @@ MappingHints getMappingHints( @SuppressWarnings("unchecked") private boolean checkMappingForPatternText(Map mapping) { var docMapping = mapping.get("_doc"); - if ((docMapping instanceof Map) == false) { - return false; + if (docMapping instanceof Map) { + mapping = (Map) docMapping; } boolean[] usesPatternText = { false }; - MappingVisitor.visitMapping((Map) docMapping, (field, fieldMapping) -> { + MappingVisitor.visitMapping(mapping, (field, fieldMapping) -> { if (Objects.equals(fieldMapping.get("type"), PatternTextFieldType.CONTENT_TYPE)) { usesPatternText[0] = true; } From f06b46ecc781e38ef97b9508036a8420f26cbc1a Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Tue, 7 Oct 2025 10:43:12 -0700 Subject: [PATCH 3/4] Update docs/changelog/136119.yaml --- docs/changelog/136119.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/136119.yaml diff --git a/docs/changelog/136119.yaml b/docs/changelog/136119.yaml new file mode 100644 index 0000000000000..d900562147824 --- /dev/null +++ b/docs/changelog/136119.yaml @@ -0,0 +1,6 @@ +pr: 136119 +summary: Fix logsdb settings provider mapping filters +area: Logs +type: bug +issues: + - 136107 From 7d6504bc7b6029d24f0a5b9c79d2477df72e29f2 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Tue, 7 Oct 2025 11:04:52 -0700 Subject: [PATCH 4/4] Remove PatternTextBasicLicenseTests --- .../PatternTextBasicLicenseTests.java | 75 ------------------- 1 file changed, 75 deletions(-) delete mode 100644 x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextBasicLicenseTests.java diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextBasicLicenseTests.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextBasicLicenseTests.java deleted file mode 100644 index f6e72eb23a978..0000000000000 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextBasicLicenseTests.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.logsdb.patterntext; - -import org.elasticsearch.xpack.logsdb.DataStreamLicenseChangeTestCase; -import org.junit.Before; - -import java.io.IOException; -import java.util.Map; - -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.hasKey; - -public class PatternTextBasicLicenseTests extends DataStreamLicenseChangeTestCase { - @Before - public void checkClusterFeature() { - assumeTrue("[patterned_text] must be available", clusterHasFeature("mapper.patterned_text")); - } - - private static final String patternTextMapping = """ - { - "index_patterns": ["%name%"], - "priority": 500, - "data_stream": {}, - "template": { - "mappings": { - "properties": { - "pattern_field": { - "type": "pattern_text" - } - } - } - } - }"""; - - @SuppressWarnings("unchecked") - public void testLicenseUpgrade() throws IOException { - final String dataStreamName = "test-foo"; - - final String doc = """ - {"index": {}} - {"@timestamp": "2025-01-01T00:00:00.000Z", "pattern_field": "foo"} - """; - - assertOK(putTemplate(client(), "logs@custom", patternTextMapping.replace("%name%", dataStreamName))); - assertOK(bulkIndex(client(), dataStreamName, () -> doc)); - - String backingIndex0 = getDataStreamBackingIndex(client(), dataStreamName, 0); - { - assertEquals("true", getSetting(client(), backingIndex0, "index.mapping.pattern_text.disable_templating")); - Map mapping = getMapping(client(), backingIndex0); - Map patternFieldMapping = (Map) ((Map) mapping.get("properties")).get( - "pattern_field" - ); - assertThat(patternFieldMapping, hasEntry("disable_templating", true)); - } - - assertOK(rolloverDataStream(client(), dataStreamName)); - - String backingIndex1 = getDataStreamBackingIndex(client(), dataStreamName, 1); - { - assertEquals("true", getSetting(client(), backingIndex1, "index.mapping.pattern_text.disable_templating")); - Map mapping = getMapping(client(), backingIndex1); - Map patternFieldMapping = (Map) ((Map) mapping.get("properties")).get( - "pattern_field" - ); - assertThat(patternFieldMapping, hasKey("disable_templating")); - } - } -}