diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java index 37352410811a8..ba89c0c9c570b 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java @@ -11,6 +11,7 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.env.Environment; @@ -61,6 +62,17 @@ public static MapperService newMapperService( Settings settings, IndicesModule indicesModule, String indexName + ) throws IOException { + return newMapperService(xContentRegistry, tempDir, settings, indicesModule, indexName, new Setting[0]); + } + + public static MapperService newMapperService( + NamedXContentRegistry xContentRegistry, + Path tempDir, + Settings settings, + IndicesModule indicesModule, + String indexName, + Setting[] additionalSettings ) throws IOException { Settings.Builder settingsBuilder = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), tempDir).put(settings); if (settings.get(IndexMetadata.SETTING_VERSION_CREATED) == null) { @@ -68,7 +80,7 @@ public static MapperService newMapperService( } Settings finalSettings = settingsBuilder.build(); MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); - IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexName, finalSettings); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexName, finalSettings, additionalSettings); IndexAnalyzers indexAnalyzers = createTestAnalysis(indexSettings, finalSettings).indexAnalyzers; SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap()); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP); diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextLicenseDowngradeIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextLicenseDowngradeIT.java index e6f8ec12075d7..95cf4094e241b 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextLicenseDowngradeIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextLicenseDowngradeIT.java @@ -28,8 +28,12 @@ public void checkClusterFeature() { "template": { "mappings": { "properties": { - "pattern_field": { - "type": "pattern_text" + "error": { + "properties": { + "message": { + "type": "pattern_text" + } + } } } } @@ -37,6 +41,14 @@ public void checkClusterFeature() { }"""; @SuppressWarnings("unchecked") + private Map resolveSubmap(Map map, String... keys) { + Map curr = map; + for (String key : keys) { + curr = (Map) curr.get(key); + } + return curr; + } + public void testLicenseDowngrade() throws IOException { final String dataStreamName = "logs-test-pattern-text"; @@ -48,9 +60,7 @@ public void testLicenseDowngrade() throws IOException { { assertEquals("false", getSetting(client(), backingIndex0, "index.mapping.pattern_text.disable_templating")); Map mapping = getMapping(client(), backingIndex0); - Map patternFieldMapping = (Map) ((Map) mapping.get("properties")).get( - "pattern_field" - ); + Map patternFieldMapping = resolveSubmap(mapping, "properties", "error", "properties", "message"); assertThat(patternFieldMapping, not(hasKey("disable_templating"))); } @@ -60,9 +70,7 @@ public void testLicenseDowngrade() throws IOException { { assertEquals("false", getSetting(client(), backingIndex0, "index.mapping.pattern_text.disable_templating")); Map mapping = getMapping(client(), backingIndex0); - Map patternFieldMapping = (Map) ((Map) mapping.get("properties")).get( - "pattern_field" - ); + Map patternFieldMapping = resolveSubmap(mapping, "properties", "error", "properties", "message"); assertThat(patternFieldMapping, not(hasKey("disable_templating"))); } @@ -70,9 +78,7 @@ public void testLicenseDowngrade() throws IOException { { assertEquals("true", getSetting(client(), backingIndex1, "index.mapping.pattern_text.disable_templating")); Map mapping = getMapping(client(), backingIndex1); - Map patternFieldMapping = (Map) ((Map) mapping.get("properties")).get( - "pattern_field" - ); + Map patternFieldMapping = resolveSubmap(mapping, "properties", "error", "properties", "message"); assertThat(patternFieldMapping, hasEntry("disable_templating", true)); } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextLicenseUpgradeIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextLicenseUpgradeIT.java index 680e170efb48f..cabd27e4773e8 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextLicenseUpgradeIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextLicenseUpgradeIT.java @@ -28,7 +28,7 @@ public void checkClusterFeature() { "template": { "mappings": { "properties": { - "pattern_field": { + "message": { "type": "pattern_text" } } @@ -48,7 +48,7 @@ public void testLicenseUpgrade() throws IOException { assertEquals("true", getSetting(client(), backingIndex0, "index.mapping.pattern_text.disable_templating")); Map mapping = getMapping(client(), backingIndex0); Map patternFieldMapping = (Map) ((Map) mapping.get("properties")).get( - "pattern_field" + "message" ); assertThat(patternFieldMapping, hasEntry("disable_templating", true)); } @@ -60,7 +60,7 @@ public void testLicenseUpgrade() throws IOException { assertEquals("true", getSetting(client(), backingIndex0, "index.mapping.pattern_text.disable_templating")); Map mapping = getMapping(client(), backingIndex0); Map patternFieldMapping = (Map) ((Map) mapping.get("properties")).get( - "pattern_field" + "message" ); assertThat(patternFieldMapping, hasEntry("disable_templating", true)); } @@ -70,7 +70,7 @@ public void testLicenseUpgrade() throws IOException { assertEquals("false", getSetting(client(), backingIndex1, "index.mapping.pattern_text.disable_templating")); Map mapping = getMapping(client(), backingIndex1); Map patternFieldMapping = (Map) ((Map) mapping.get("properties")).get( - "pattern_field" + "message" ); assertThat(patternFieldMapping, not(hasKey("disable_templating"))); } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index 64fe8f1376621..8749dd0dafc6e 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -54,7 +54,9 @@ final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider { "_doc._source.*", "_doc.properties.host**", "_doc.properties.resource**", - "_doc.subobjects" + "_doc.subobjects", + "_doc.properties.message", + "_doc.properties.error.properties.message" ); private final LogsdbLicenseService licenseService; @@ -192,13 +194,13 @@ && matchesLogsPattern(dataStreamName)) { } } - if (licenseService.allowPatternTextTemplating(isTemplateValidation) == false) { + if (licenseService.allowPatternTextTemplating(isTemplateValidation) == false && mappingHints.maybeUsesPatternText) { additionalSettings.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true); } } - record MappingHints(boolean hasSyntheticSourceUsage, boolean sortOnHostName, boolean addHostNameField) { - static MappingHints EMPTY = new MappingHints(false, false, false); + record MappingHints(boolean hasSyntheticSourceUsage, boolean sortOnHostName, boolean addHostNameField, boolean maybeUsesPatternText) { + static MappingHints EMPTY = new MappingHints(false, false, false, false); } private static boolean matchesLogsPattern(final String name) { @@ -237,12 +239,12 @@ MappingHints getMappingHints( hasSyntheticSourceUsage = sourceMode == SourceFieldMapper.Mode.SYNTHETIC; if (IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(indexTemplateAndCreateRequestSettings).isEmpty() == false) { // Custom sort config, no point for further checks on [host.name] field. - return new MappingHints(hasSyntheticSourceUsage, false, false); + return new MappingHints(hasSyntheticSourceUsage, false, false, true); } if (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(indexTemplateAndCreateRequestSettings) && IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(indexTemplateAndCreateRequestSettings)) { // Settings for adding and sorting on [host.name] are already set, propagate them. - return new MappingHints(hasSyntheticSourceUsage, true, true); + return new MappingHints(hasSyntheticSourceUsage, true, true, true); } } @@ -277,7 +279,10 @@ MappingHints getMappingHints( || addHostNameField || (hostName instanceof NumberFieldMapper nfm && nfm.fieldType().hasDocValues()) || (hostName instanceof KeywordFieldMapper kfm && kfm.fieldType().hasDocValues()); - return new MappingHints(hasSyntheticSourceUsage, sortOnHostName, addHostNameField); + boolean usesPatternText = PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.get(indexTemplateAndCreateRequestSettings) + || mapperService.mappingLookup().getMapper("message") instanceof PatternTextFieldMapper + || mapperService.mappingLookup().getMapper("error.message") instanceof PatternTextFieldMapper; + return new MappingHints(hasSyntheticSourceUsage, sortOnHostName, addHostNameField, usesPatternText); } } catch (AssertionError | Exception e) { // In case invalid mappings or setting are provided, then mapper service creation can fail. diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index f5b7916364bad..4bd4dab2df4ba 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; @@ -26,9 +27,12 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.MapperTestUtils; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.license.License; import org.elasticsearch.license.LicenseService; import org.elasticsearch.license.MockLicenseState; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.license.internal.XPackLicenseStatus; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.logsdb.patterntext.PatternTextFieldMapper; import org.junit.Before; @@ -72,6 +76,7 @@ public class LogsdbIndexModeSettingsProviderTests extends ESTestCase { """; private LogsdbLicenseService logsdbLicenseService; + private LogsdbLicenseService basicLogsdbLicenseService; private final AtomicInteger newMapperServiceCounter = new AtomicInteger(); @Before @@ -84,6 +89,13 @@ public void setup() throws Exception { logsdbLicenseService = new LogsdbLicenseService(Settings.EMPTY); logsdbLicenseService.setLicenseState(licenseState); logsdbLicenseService.setLicenseService(mockLicenseService); + + var basicLicenseState = new XPackLicenseState(() -> 0L, new XPackLicenseStatus(License.OperationMode.BASIC, true, null)); + var basicLicenseService = mock(LicenseService.class); + when(basicLicenseService.getLicense()).thenReturn(null); + basicLogsdbLicenseService = new LogsdbLicenseService(Settings.EMPTY); + basicLogsdbLicenseService.setLicenseState(basicLicenseState); + basicLogsdbLicenseService.setLicenseService(basicLicenseService); } private LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled) { @@ -121,13 +133,23 @@ private Settings generateLogsdbSettings(Settings settings, String mapping) throw } private Settings generateLogsdbSettings(Settings settings, String mapping, Version version) throws IOException { - var provider = new LogsdbIndexModeSettingsProvider( - logsdbLicenseService, - Settings.builder().put("cluster.logsdb.enabled", true).build() - ); + return generateLogsdbSettings(settings, mapping, version, logsdbLicenseService); + } + + private Settings generateLogsdbSettings(Settings settings, String mapping, Version version, LogsdbLicenseService licenseService) + throws IOException { + var provider = new LogsdbIndexModeSettingsProvider(licenseService, Settings.builder().put("cluster.logsdb.enabled", true).build()); + var logsdbPlugin = new LogsDBPlugin(settings); provider.init(im -> { newMapperServiceCounter.incrementAndGet(); - return MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()); + return MapperTestUtils.newMapperService( + xContentRegistry(), + createTempDir(), + im.getSettings(), + new IndicesModule(List.of(logsdbPlugin)), + im.getIndex().getName(), + logsdbPlugin.getSettings().stream().filter(Setting::hasIndexScope).toArray(Setting[]::new) + ); }, IndexVersion::current, () -> version, true, true); Settings.Builder settingsBuilder = builder(); provider.provideAdditionalSettings( @@ -959,19 +981,32 @@ public void testExplicitRoutingPathNotAllowedByLicense() throws Exception { assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), empty()); } - public void testPatternTextNotAllowedByLicense() throws Exception { - MockLicenseState licenseState = MockLicenseState.createMock(); - when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); - when(licenseState.isAllowed(same(LogsdbLicenseService.PATTERN_TEXT_TEMPLATING_FEATURE))).thenReturn(false); - logsdbLicenseService = new LogsdbLicenseService(Settings.EMPTY); - logsdbLicenseService.setLicenseState(licenseState); + public void testPatternTextNotAllowedByLicense() throws IOException { + String[] patternTextLicenceCheckedFieldMappings = { + "{\"_doc\":{\"properties\":{\"message\":{\"type\":\"pattern_text\"}}}}", + "{\"_doc\":{\"properties\":{\"error\":{\"properties\":{\"message\":{\"type\":\"pattern_text\"}}}}}}}" }; - var settings = Settings.builder() - .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message") - .put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), true) + var expectedSettings = Settings.builder() + .put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true) + .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true) + .put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true) .build(); - Settings result = generateLogsdbSettings(settings); - assertTrue(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.get(result)); + + for (String mapping : patternTextLicenceCheckedFieldMappings) { + var result = generateLogsdbSettings(Settings.EMPTY, mapping, Version.CURRENT, basicLogsdbLicenseService); + assertEquals(expectedSettings, result); + } + } + + public void testPatternTextNotAllowedByLicenseAlreadyDisallowed() throws IOException { + Settings settings = Settings.builder().put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), "true").build(); + var result = generateLogsdbSettings(settings, null, Version.CURRENT, basicLogsdbLicenseService); + var expected = Settings.builder() + .put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true) + .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true) + .put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true) + .build(); + assertEquals(expected, result); } public void testSortAndHostNamePropagateValue() throws Exception { diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java index f0411deaf81de..19b30822e4df6 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.license.internal.XPackLicenseStatus; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.logsdb.patterntext.PatternTextFieldMapper; import org.junit.Before; import java.io.IOException; @@ -77,10 +76,7 @@ public void testGetAdditionalIndexSettingsDefault() { builder ); var result = builder.build(); - var expected = Settings.builder() - .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "STORED") - .put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true) - .build(); + var expected = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "STORED").build(); assertEquals(expected, result); } @@ -101,17 +97,11 @@ public void testGetAdditionalIndexSettingsApm() throws IOException { builder ); var result = builder.build(); - Settings expectedAdditionalSettings = Settings.builder() - .put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true) - .build(); - assertEquals(expectedAdditionalSettings, result); + assertEquals(Settings.EMPTY, result); } public void testGetAdditionalIndexSettingsProfiling() throws IOException { Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); - Settings expectedAdditionalSettings = Settings.builder() - .put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true) - .build(); for (String dataStreamName : new String[] { "profiling-metrics", "profiling-events" }) { String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); Settings.Builder builder = Settings.builder(); @@ -127,14 +117,14 @@ public void testGetAdditionalIndexSettingsProfiling() throws IOException { builder ); var result = builder.build(); - assertEquals(expectedAdditionalSettings, result); + assertEquals(Settings.EMPTY, result); } for (String indexName : new String[] { ".profiling-sq-executables", ".profiling-sq-leafframes", ".profiling-stacktraces" }) { Settings.Builder builder = Settings.builder(); provider.provideAdditionalSettings(indexName, null, null, null, null, settings, List.of(), IndexVersion.current(), builder); var result = builder.build(); - assertEquals(expectedAdditionalSettings, result); + assertEquals(Settings.EMPTY, result); } } @@ -155,10 +145,7 @@ public void testGetAdditionalIndexSettingsTsdb() throws IOException { builder ); var result = builder.build(); - Settings expectedAdditionalSettings = Settings.builder() - .put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true) - .build(); - assertEquals(expectedAdditionalSettings, result); + assertEquals(Settings.EMPTY, result); } public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception { @@ -202,10 +189,7 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception ); var result = builder.build(); - var expected = Settings.builder() - .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "STORED") - .put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true) - .build(); + var expected = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "STORED").build(); assertEquals(expected, result); } }