diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 45370ea6358fa..9d045edf52eb0 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -29,6 +29,8 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.MockBigArrays; @@ -88,6 +90,7 @@ import java.io.IOException; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -282,10 +285,15 @@ public TestMapperServiceBuilder applyDefaultMapping(boolean applyDefaultMapping) } public MapperService build() { - IndexSettings indexSettings = createIndexSettings(indexVersion, settings); + Collection plugins = getPlugins(); + Collection> pluginIndexSettings = plugins.stream() + .flatMap(plugin -> plugin.getSettings().stream()) + .filter(Setting::hasIndexScope) + .toList(); + IndexSettings indexSettings = createIndexSettings(indexVersion, settings, pluginIndexSettings); SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of()); MapperRegistry mapperRegistry = new IndicesModule( - getPlugins().stream().filter(p -> p instanceof MapperPlugin).map(p -> (MapperPlugin) p).collect(toList()) + plugins.stream().filter(p -> p instanceof MapperPlugin).map(p -> (MapperPlugin) p).collect(toList()) ).getMapperRegistry(); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP); @@ -327,9 +335,19 @@ protected T compileScript(Script script, ScriptContext context) { } protected static IndexSettings createIndexSettings(IndexVersion version, Settings settings) { + return createIndexSettings(version, settings, List.of()); + } + + protected static IndexSettings createIndexSettings( + IndexVersion version, + Settings settings, + Collection> pluginIndexSettings + ) { settings = indexSettings(1, 0).put(settings).put(IndexMetadata.SETTING_VERSION_CREATED, version).build(); IndexMetadata meta = IndexMetadata.builder("index").settings(settings).build(); - return new IndexSettings(meta, settings); + Set> indexSettings = new HashSet<>(IndexScopedSettings.BUILT_IN_INDEX_SETTINGS); + indexSettings.addAll(pluginIndexSettings); + return new IndexSettings(meta, settings, new IndexScopedSettings(Settings.EMPTY, indexSettings)); } protected MapperMetrics createTestMapperMetrics() { diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/DataStreamLicenseChangeTestCase.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/DataStreamLicenseChangeTestCase.java new file mode 100644 index 0000000000000..b171770ddbef1 --- /dev/null +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/DataStreamLicenseChangeTestCase.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb; + +import org.elasticsearch.client.Request; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.junit.ClassRule; + +import java.io.IOException; + +public abstract class DataStreamLicenseChangeTestCase extends LogsIndexModeRestTestIT { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .module("data-streams") + .module("x-pack-stack") + .setting("cluster.logsdb.enabled", "true") + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "basic") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + protected static void startBasic() throws IOException { + Request startTrial = new Request("POST", "/_license/start_basic"); + startTrial.addParameter("acknowledge", "true"); + assertOK(client().performRequest(startTrial)); + } + + protected static void startTrial() throws IOException { + Request startTrial = new Request("POST", "/_license/start_trial"); + startTrial.addParameter("acknowledge", "true"); + assertOK(client().performRequest(startTrial)); + } + +} diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeRestTestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeRestTestIT.java index cc7f5bdb33871..702c66730d1d0 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeRestTestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeRestTestIT.java @@ -31,10 +31,10 @@ protected static void waitForLogs(RestClient client) throws Exception { }); } - protected static Response putComponentTemplate(final RestClient client, final String componentTemplate, final String contends) + protected static Response putComponentTemplate(final RestClient client, final String componentTemplate, final String contents) throws IOException { final Request request = new Request("PUT", "/_component_template/" + componentTemplate); - request.setJsonEntity(contends); + request.setJsonEntity(contents); return client.performRequest(request); } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java index 3a729546df9a3..c26bb32f41f71 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java @@ -81,7 +81,10 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException { boolean found = false; for (var feature : features) { if (feature.get("family") != null) { - assertThat(feature.get("name"), anyOf(equalTo("synthetic-source"), equalTo("logsdb-routing-on-sort-fields"))); + assertThat( + feature.get("name"), + anyOf(equalTo("synthetic-source"), equalTo("logsdb-routing-on-sort-fields"), equalTo("patterned-text-templating")) + ); assertThat(feature.get("license_level"), equalTo("enterprise")); found = true; } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/DataStreamLicenseChangeIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/SourceModeLicenseChangeTestCase.java similarity index 67% rename from x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/DataStreamLicenseChangeIT.java rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/SourceModeLicenseChangeTestCase.java index b84c982766e4b..9f7fcd5e504fb 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/DataStreamLicenseChangeIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/SourceModeLicenseChangeTestCase.java @@ -11,28 +11,11 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.index.mapper.SourceFieldMapper; -import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.test.cluster.local.distribution.DistributionType; -import org.junit.ClassRule; import java.io.IOException; import java.util.List; -public abstract class DataStreamLicenseChangeIT extends LogsIndexModeRestTestIT { - @ClassRule - public static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.DEFAULT) - .module("data-streams") - .module("x-pack-stack") - .setting("cluster.logsdb.enabled", "true") - .setting("xpack.security.enabled", "false") - .setting("xpack.license.self_generated.type", "basic") - .build(); - - @Override - protected String getTestRestCluster() { - return cluster.getHttpAddresses(); - } +public abstract class SourceModeLicenseChangeTestCase extends DataStreamLicenseChangeTestCase { protected interface TestCase { String dataStreamName(); @@ -88,18 +71,6 @@ public void testLicenseChange() throws IOException { } } - protected static void startBasic() throws IOException { - Request startTrial = new Request("POST", "/_license/start_basic"); - startTrial.addParameter("acknowledge", "true"); - assertOK(client().performRequest(startTrial)); - } - - protected static void startTrial() throws IOException { - Request startTrial = new Request("POST", "/_license/start_trial"); - startTrial.addParameter("acknowledge", "true"); - assertOK(client().performRequest(startTrial)); - } - protected static Response removeComponentTemplate(final RestClient client, final String componentTemplate) throws IOException { final Request request = new Request("DELETE", "/_component_template/" + componentTemplate); return client.performRequest(request); diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/DataStreamLicenceDowngradeIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/SourceModeLicenseDowngradeIT.java similarity index 99% rename from x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/DataStreamLicenceDowngradeIT.java rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/SourceModeLicenseDowngradeIT.java index f004189098c43..655bc7d15c1be 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/DataStreamLicenceDowngradeIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/SourceModeLicenseDowngradeIT.java @@ -12,7 +12,7 @@ import java.io.IOException; import java.util.List; -public class DataStreamLicenceDowngradeIT extends DataStreamLicenseChangeIT { +public class SourceModeLicenseDowngradeIT extends SourceModeLicenseChangeTestCase { @Override protected void applyInitialLicense() throws IOException { startTrial(); diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/DataStreamLicenseUpgradeIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/SourceModeLicenseUpgradeIT.java similarity index 99% rename from x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/DataStreamLicenseUpgradeIT.java rename to x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/SourceModeLicenseUpgradeIT.java index bce43ca046523..b404afefcaab6 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/DataStreamLicenseUpgradeIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/SourceModeLicenseUpgradeIT.java @@ -12,7 +12,7 @@ import java.io.IOException; import java.util.List; -public class DataStreamLicenseUpgradeIT extends DataStreamLicenseChangeIT { +public class SourceModeLicenseUpgradeIT extends SourceModeLicenseChangeTestCase { @Override protected void applyInitialLicense() {} diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextLicenseDowngradeIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextLicenseDowngradeIT.java new file mode 100644 index 0000000000000..cde8efc017939 --- /dev/null +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextLicenseDowngradeIT.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb.patternedtext; + +import org.elasticsearch.Build; +import org.elasticsearch.xpack.logsdb.DataStreamLicenseChangeTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.Map; + +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.not; + +public class PatternedTextLicenseDowngradeIT extends DataStreamLicenseChangeTestCase { + @Before + public void checkClusterFeature() { + assumeTrue("[patterned_text] must be available", clusterHasFeature("mapper.patterned_text")); + assumeTrue("[patterned_text] is only available in snapshot builds", Build.current().isSnapshot()); + } + + private static final String patternedTextMapping = """ + { + "template": { + "mappings": { + "properties": { + "patterned_field": { + "type": "patterned_text" + } + } + } + } + }"""; + + @SuppressWarnings("unchecked") + public void testLicenseDowngrade() throws IOException { + final String dataStreamName = "logs-test-patterned-text"; + + startTrial(); + assertOK(putComponentTemplate(client(), "logs@custom", patternedTextMapping)); + assertOK(createDataStream(client(), dataStreamName)); + + String backingIndex0 = getDataStreamBackingIndex(client(), dataStreamName, 0); + { + assertEquals("false", getSetting(client(), backingIndex0, "index.mapping.patterned_text.disable_templating")); + Map mapping = getMapping(client(), backingIndex0); + Map patternedFieldMapping = (Map) ((Map) mapping.get("properties")).get( + "patterned_field" + ); + assertThat(patternedFieldMapping, not(hasKey("disable_templating"))); + } + + startBasic(); + rolloverDataStream(client(), dataStreamName); + + { + assertEquals("false", getSetting(client(), backingIndex0, "index.mapping.patterned_text.disable_templating")); + Map mapping = getMapping(client(), backingIndex0); + Map patternedFieldMapping = (Map) ((Map) mapping.get("properties")).get( + "patterned_field" + ); + assertThat(patternedFieldMapping, not(hasKey("disable_templating"))); + } + + String backingIndex1 = getDataStreamBackingIndex(client(), dataStreamName, 1); + { + assertEquals("true", getSetting(client(), backingIndex1, "index.mapping.patterned_text.disable_templating")); + Map mapping = getMapping(client(), backingIndex1); + Map patternedFieldMapping = (Map) ((Map) mapping.get("properties")).get( + "patterned_field" + ); + assertThat(patternedFieldMapping, hasEntry("disable_templating", true)); + } + + } +} diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextLicenseUpgradeIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextLicenseUpgradeIT.java new file mode 100644 index 0000000000000..70808379f9496 --- /dev/null +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextLicenseUpgradeIT.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb.patternedtext; + +import org.elasticsearch.Build; +import org.elasticsearch.xpack.logsdb.DataStreamLicenseChangeTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.Map; + +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.not; + +public class PatternedTextLicenseUpgradeIT extends DataStreamLicenseChangeTestCase { + @Before + public void checkClusterFeature() { + assumeTrue("[patterned_text] must be available", clusterHasFeature("mapper.patterned_text")); + assumeTrue("[patterned_text] is only available in snapshot builds", Build.current().isSnapshot()); + } + + private static final String patternedTextMapping = """ + { + "template": { + "mappings": { + "properties": { + "patterned_field": { + "type": "patterned_text" + } + } + } + } + }"""; + + @SuppressWarnings("unchecked") + public void testLicenseUpgrade() throws IOException { + final String dataStreamName = "logs-test-patterned-text"; + + assertOK(putComponentTemplate(client(), "logs@custom", patternedTextMapping)); + assertOK(createDataStream(client(), dataStreamName)); + + String backingIndex0 = getDataStreamBackingIndex(client(), dataStreamName, 0); + { + assertEquals("true", getSetting(client(), backingIndex0, "index.mapping.patterned_text.disable_templating")); + Map mapping = getMapping(client(), backingIndex0); + Map patternedFieldMapping = (Map) ((Map) mapping.get("properties")).get( + "patterned_field" + ); + assertThat(patternedFieldMapping, hasEntry("disable_templating", true)); + } + + startTrial(); + rolloverDataStream(client(), dataStreamName); + + { + assertEquals("true", getSetting(client(), backingIndex0, "index.mapping.patterned_text.disable_templating")); + Map mapping = getMapping(client(), backingIndex0); + Map patternedFieldMapping = (Map) ((Map) mapping.get("properties")).get( + "patterned_field" + ); + assertThat(patternedFieldMapping, hasEntry("disable_templating", true)); + } + + String backingIndex1 = getDataStreamBackingIndex(client(), dataStreamName, 1); + { + assertEquals("false", getSetting(client(), backingIndex1, "index.mapping.patterned_text.disable_templating")); + Map mapping = getMapping(client(), backingIndex1); + Map patternedFieldMapping = (Map) ((Map) mapping.get("properties")).get( + "patterned_field" + ); + assertThat(patternedFieldMapping, not(hasKey("disable_templating"))); + } + + } +} diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java index 70236c8e085c0..55c81e3849aae 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -93,7 +94,11 @@ public Collection getAdditionalIndexSettingProviders(Index @Override public List> getSettings() { - return List.of(FALLBACK_SETTING, CLUSTER_LOGSDB_ENABLED, LOGSDB_PRIOR_LOGS_USAGE); + List> settings = new ArrayList<>(List.of(FALLBACK_SETTING, CLUSTER_LOGSDB_ENABLED, LOGSDB_PRIOR_LOGS_USAGE)); + if (PatternedTextFieldMapper.PATTERNED_TEXT_MAPPER.isEnabled()) { + settings.add(PatternedTextFieldMapper.DISABLE_TEMPLATING_SETTING); + } + return Collections.unmodifiableList(settings); } @Override diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index d70f3b4b58c15..bef6285b82af0 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -34,6 +34,7 @@ import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.logsdb.patternedtext.PatternedTextFieldMapper; import java.io.IOException; import java.time.Instant; @@ -192,6 +193,11 @@ && matchesLogsPattern(dataStreamName)) { } } } + + if (PatternedTextFieldMapper.PATTERNED_TEXT_MAPPER.isEnabled() + && licenseService.allowPatternedTextTemplating(isTemplateValidation) == false) { + additionalSettings.put(PatternedTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true); + } } record MappingHints(boolean hasSyntheticSourceUsage, boolean sortOnHostName, boolean addHostNameField) { diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbLicenseService.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbLicenseService.java index d3487e205b33e..ab3f88fd9783d 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbLicenseService.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbLicenseService.java @@ -59,6 +59,12 @@ final class LogsdbLicenseService { License.OperationMode.ENTERPRISE ); + static final LicensedFeature.Momentary PATTERNED_TEXT_TEMPLATING_FEATURE = LicensedFeature.momentary( + MAPPINGS_FEATURE_FAMILY, + "patterned-text-templating", + License.OperationMode.ENTERPRISE + ); + private final long cutoffDate; private LicenseService licenseService; private XPackLicenseState licenseState; @@ -103,6 +109,13 @@ && checkFeature(SYNTHETIC_SOURCE_FEATURE_LEGACY, licenseStateSnapshot, isTemplat return true; } + /** + * @return whether patterned_text fields should allow templating. + */ + public boolean allowPatternedTextTemplating(boolean isTemplateValidation) { + return checkFeature(PATTERNED_TEXT_TEMPLATING_FEATURE, licenseState.copyCurrentLicenseState(), isTemplateValidation); + } + /** * @return whether indexes in logsdb mode can use routing on sort fields. */ diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java index 50b2516e48a7f..a6cf55a29622f 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java @@ -16,6 +16,7 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.LeafReader; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; @@ -49,6 +50,17 @@ public class PatternedTextFieldMapper extends FieldMapper { public static final FeatureFlag PATTERNED_TEXT_MAPPER = new FeatureFlag("patterned_text"); private static final NamedAnalyzer STANDARD_ANALYZER = new NamedAnalyzer("standard", AnalyzerScope.GLOBAL, new StandardAnalyzer()); + /** + * A setting that indicates that patterned text fields should disable templating, usually because there is + * no valid enterprise license. + */ + public static final Setting DISABLE_TEMPLATING_SETTING = Setting.boolSetting( + "index.mapping.patterned_text.disable_templating", + false, + Setting.Property.IndexScope, + Setting.Property.PrivateIndex + ); + public static class Defaults { public static final FieldType FIELD_TYPE_DOCS; public static final FieldType FIELD_TYPE_POSITIONS; @@ -81,6 +93,7 @@ public static class Builder extends FieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); private final Parameter indexOptions = patternedTextIndexOptions(m -> ((PatternedTextFieldMapper) m).indexOptions); private final Parameter analyzer; + private final Parameter disableTemplating; public Builder(String name, MappingParserContext context) { this(name, context.indexVersionCreated(), context.getIndexSettings()); @@ -91,11 +104,12 @@ public Builder(String name, IndexVersion indexCreatedVersion, IndexSettings inde this.indexCreatedVersion = indexCreatedVersion; this.indexSettings = indexSettings; this.analyzer = analyzerParam(name, m -> ((PatternedTextFieldMapper) m).analyzer); + this.disableTemplating = disableTemplatingParameter(indexSettings); } @Override protected Parameter[] getParameters() { - return new Parameter[] { meta, indexOptions, analyzer }; + return new Parameter[] { meta, indexOptions, analyzer, disableTemplating }; } private PatternedTextFieldType buildFieldType(FieldType fieldType, MapperBuilderContext context) { @@ -106,6 +120,7 @@ private PatternedTextFieldType buildFieldType(FieldType fieldType, MapperBuilder tsi, analyzer, context.isSourceSynthetic(), + disableTemplating.getValue(), meta.getValue() ); } @@ -145,6 +160,31 @@ private static Parameter analyzerParam(String name, Function b.field(n, v.name()), NamedAnalyzer::name); } + /** + * A parameter that indicates the patterned_text mapper should disable templating, usually + * because there is no valid enterprise license. + *

+ * The parameter should only be explicitly enabled or left unset. When left unset, it defaults to the value determined from the + * associated index setting, which is set from the current license status. + */ + private static Parameter disableTemplatingParameter(IndexSettings indexSettings) { + boolean forceDisable = indexSettings.getValue(DISABLE_TEMPLATING_SETTING); + return Parameter.boolParam( + "disable_templating", + false, + m -> ((PatternedTextFieldMapper) m).fieldType().disableTemplating(), + forceDisable + ).addValidator(value -> { + if (value == false && forceDisable) { + throw new MapperParsingException( + "value [false] for mapping parameter [disable_templating] contradicts value [true] for index setting [" + + DISABLE_TEMPLATING_SETTING.getKey() + + "]" + ); + } + }).setSerializerCheck((includeDefaults, isConfigured, value) -> includeDefaults || isConfigured || value); + } + @Override public PatternedTextFieldMapper build(MapperBuilderContext context) { FieldType fieldType = buildLuceneFieldType(indexOptions); diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java index 575e69cd54ebd..35f1047559e89 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java @@ -64,13 +64,23 @@ public class PatternedTextFieldType extends StringFieldType { private final TextFieldMapper.TextFieldType textFieldType; private final boolean hasPositions; - PatternedTextFieldType(String name, TextSearchInfo tsi, Analyzer indexAnalyzer, boolean isSyntheticSource, Map meta) { + private final boolean disableTemplating; + + PatternedTextFieldType( + String name, + TextSearchInfo tsi, + Analyzer indexAnalyzer, + boolean isSyntheticSource, + boolean disableTemplating, + Map meta + ) { // Though this type is based on doc_values, hasDocValues is set to false as the patterned_text type is not aggregatable. // This does not stop its child .template type from being aggregatable. super(name, true, false, false, tsi, meta); this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer); this.textFieldType = new TextFieldMapper.TextFieldType(name, isSyntheticSource); this.hasPositions = tsi.hasPositions(); + this.disableTemplating = disableTemplating; } // For testing only @@ -85,6 +95,7 @@ public class PatternedTextFieldType extends StringFieldType { ), DelimiterAnalyzer.INSTANCE, syntheticSource, + false, Collections.emptyMap() ); } @@ -296,4 +307,8 @@ String storedNamed() { return name() + STORED_SUFFIX; } + boolean disableTemplating() { + return disableTemplating; + } + } diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index 4388a78a27e3b..25073f8f4b970 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.license.LicenseService; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.logsdb.patternedtext.PatternedTextFieldMapper; import org.junit.Before; import java.io.IOException; @@ -958,6 +959,23 @@ public void testExplicitRoutingPathNotAllowedByLicense() throws Exception { assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), empty()); } + public void testPatternedTextNotAllowedByLicense() throws Exception { + assumeTrue("patterned_text feature must be enabled", PatternedTextFieldMapper.PATTERNED_TEXT_MAPPER.isEnabled()); + + MockLicenseState licenseState = MockLicenseState.createMock(); + when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); + when(licenseState.isAllowed(same(LogsdbLicenseService.PATTERNED_TEXT_TEMPLATING_FEATURE))).thenReturn(false); + logsdbLicenseService = new LogsdbLicenseService(Settings.EMPTY); + logsdbLicenseService.setLicenseState(licenseState); + + var settings = Settings.builder() + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message") + .put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), true) + .build(); + Settings result = generateLogsdbSettings(settings); + assertTrue(PatternedTextFieldMapper.DISABLE_TEMPLATING_SETTING.get(result)); + } + public void testSortAndHostNamePropagateValue() throws Exception { var settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB) diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java index b01442020cefe..5d7bb143cc829 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.license.internal.XPackLicenseStatus; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.logsdb.patternedtext.PatternedTextFieldMapper; import org.junit.Before; import java.io.IOException; @@ -27,7 +28,6 @@ import java.util.List; import static org.elasticsearch.xpack.logsdb.LogsdbLicenseServiceTests.createGoldOrPlatinumLicense; -import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -67,8 +67,11 @@ public void testGetAdditionalIndexSettingsDefault() { Settings.Builder builder = Settings.builder(); provider.provideAdditionalMetadata(indexName, dataStreamName, null, null, null, settings, List.of(), builder, (k, v) -> {}); var result = builder.build(); - assertThat(result.size(), equalTo(1)); - assertThat(result.get(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED")); + var expectedBuilder = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "STORED"); + if (PatternedTextFieldMapper.PATTERNED_TEXT_MAPPER.isEnabled()) { + expectedBuilder.put(PatternedTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true); + } + assertEquals(expectedBuilder.build(), result); } public void testGetAdditionalIndexSettingsApm() throws IOException { @@ -78,24 +81,30 @@ public void testGetAdditionalIndexSettingsApm() throws IOException { Settings.Builder builder = Settings.builder(); provider.provideAdditionalMetadata(indexName, dataStreamName, null, null, null, settings, List.of(), builder, (k, v) -> {}); var result = builder.build(); - assertThat(result.size(), equalTo(0)); + Settings expectedAdditionalSettings = PatternedTextFieldMapper.PATTERNED_TEXT_MAPPER.isEnabled() + ? Settings.builder().put(PatternedTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true).build() + : Settings.EMPTY; + assertEquals(expectedAdditionalSettings, result); } public void testGetAdditionalIndexSettingsProfiling() throws IOException { Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings expectedAdditionalSettings = PatternedTextFieldMapper.PATTERNED_TEXT_MAPPER.isEnabled() + ? Settings.builder().put(PatternedTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true).build() + : Settings.EMPTY; for (String dataStreamName : new String[] { "profiling-metrics", "profiling-events" }) { String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); Settings.Builder builder = Settings.builder(); provider.provideAdditionalMetadata(indexName, dataStreamName, null, null, null, settings, List.of(), builder, (k, v) -> {}); var result = builder.build(); - assertThat(result.size(), equalTo(0)); + assertEquals(expectedAdditionalSettings, result); } for (String indexName : new String[] { ".profiling-sq-executables", ".profiling-sq-leafframes", ".profiling-stacktraces" }) { Settings.Builder builder = Settings.builder(); provider.provideAdditionalMetadata(indexName, null, null, null, null, settings, List.of(), builder, (k, v) -> {}); var result = builder.build(); - assertThat(result.size(), equalTo(0)); + assertEquals(expectedAdditionalSettings, result); } } @@ -116,7 +125,10 @@ public void testGetAdditionalIndexSettingsTsdb() throws IOException { (k, v) -> {} ); var result = builder.build(); - assertThat(result.size(), equalTo(0)); + Settings expectedAdditionalSettings = PatternedTextFieldMapper.PATTERNED_TEXT_MAPPER.isEnabled() + ? Settings.builder().put(PatternedTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true).build() + : Settings.EMPTY; + assertEquals(expectedAdditionalSettings, result); } public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception { @@ -159,7 +171,12 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception (k, v) -> {} ); var result = builder.build(); - assertThat(result.size(), equalTo(1)); - assertThat(result.get(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED")); + + var expectedBuilder = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "STORED"); + if (PatternedTextFieldMapper.PATTERNED_TEXT_MAPPER.isEnabled()) { + expectedBuilder.put(PatternedTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true); + } + + assertEquals(expectedBuilder.build(), result); } } diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbLicenseServiceTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbLicenseServiceTests.java index 60701e9402e6c..58d5db199b6c4 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbLicenseServiceTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbLicenseServiceTests.java @@ -24,6 +24,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.when; public class LogsdbLicenseServiceTests extends ESTestCase { @@ -59,6 +60,26 @@ public void testAllowRoutingOnSortFieldsTemplateValidation() { Mockito.verify(licenseState, Mockito.never()).featureUsed(any()); } + public void testAllowPatternedTextTemplating() { + MockLicenseState licenseState = MockLicenseState.createMock(); + when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); + when(licenseState.isAllowed(same(LogsdbLicenseService.PATTERNED_TEXT_TEMPLATING_FEATURE))).thenReturn(true); + licenseService.setLicenseState(licenseState); + licenseService.setLicenseService(mockLicenseService); + assertTrue(licenseService.allowPatternedTextTemplating(false)); + Mockito.verify(licenseState, Mockito.times(1)).featureUsed(any()); + } + + public void testAllowPatternedTextTemplatingTemplateValidation() { + MockLicenseState licenseState = MockLicenseState.createMock(); + when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); + when(licenseState.isAllowed(same(LogsdbLicenseService.PATTERNED_TEXT_TEMPLATING_FEATURE))).thenReturn(true); + licenseService.setLicenseState(licenseState); + licenseService.setLicenseService(mockLicenseService); + assertTrue(licenseService.allowPatternedTextTemplating(true)); + Mockito.verify(licenseState, never()).featureUsed(any()); + } + public void testLicenseAllowsSyntheticSource() { MockLicenseState licenseState = MockLicenseState.createMock(); when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapperTests.java index 2ee8149025e13..382afbbfaebbe 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapperTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapperTests.java @@ -197,6 +197,64 @@ public void testSimpleMerge() throws IOException { assertThat(mapperService.documentMapper().mappers().getMapper("other_field"), instanceOf(KeywordFieldMapper.class)); } + public void testDisableTemplatingParameter() throws IOException { + { + XContentBuilder mapping = fieldMapping(b -> b.field("type", "patterned_text")); + MapperService mapperService = createMapperService(mapping); + var mapper = (PatternedTextFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + assertFalse(mapper.fieldType().disableTemplating()); + } + + { + XContentBuilder mapping = fieldMapping(b -> b.field("type", "patterned_text").field("disable_templating", true)); + MapperService mapperService = createMapperService(mapping); + var mapper = (PatternedTextFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + assertTrue(mapper.fieldType().disableTemplating()); + } + + { + XContentBuilder mapping = fieldMapping(b -> b.field("type", "patterned_text").field("disable_templating", false)); + MapperService mapperService = createMapperService(mapping); + var mapper = (PatternedTextFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + assertFalse(mapper.fieldType().disableTemplating()); + } + } + + public void testDisableTemplatingParameterWhenDisallowedByLicense() throws IOException { + Settings indexSettings = Settings.builder() + .put(getIndexSettings()) + .put(PatternedTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true) + .build(); + { + XContentBuilder mapping = fieldMapping(b -> b.field("type", "patterned_text")); + MapperService mapperService = createMapperService(getVersion(), indexSettings, () -> true, mapping); + var mapper = (PatternedTextFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + assertTrue(mapper.fieldType().disableTemplating()); + } + + { + XContentBuilder mapping = fieldMapping(b -> b.field("type", "patterned_text").field("disable_templating", true)); + MapperService mapperService = createMapperService(getVersion(), indexSettings, () -> true, mapping); + var mapper = (PatternedTextFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + assertTrue(mapper.fieldType().disableTemplating()); + } + + { + XContentBuilder mapping = fieldMapping(b -> b.field("type", "patterned_text").field("disable_templating", false)); + Exception e = expectThrows( + MapperParsingException.class, + () -> createMapperService(getVersion(), indexSettings, () -> true, mapping) + ); + assertThat( + e.getMessage(), + containsString( + "value [false] for mapping parameter [disable_templating] contradicts value [true] for index " + + "setting [index.mapping.patterned_text.disable_templating]" + ) + ); + } + } + public void testDisabledSource() throws IOException { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_doc"); {