Skip to content

Only disable unlicensed pattern_text templating when mapper is used #135553

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

Only disable unlicensed pattern_text templating when mapper is used #135553

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ff9f1d1
Only disable pattern_text templating for message and error.message fi…
jordan-powers Sep 26, 2025
124241b
Update tests
jordan-powers Sep 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

import org.elasticsearch.TransportVersion;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.env.Environment;
Expand Down Expand Up @@ -61,14 +62,25 @@ public static MapperService newMapperService(
Settings settings,
IndicesModule indicesModule,
String indexName
) throws IOException {
return newMapperService(xContentRegistry, tempDir, settings, indicesModule, indexName, new Setting<?>[0]);
}

public static MapperService newMapperService(
NamedXContentRegistry xContentRegistry,
Path tempDir,
Settings settings,
IndicesModule indicesModule,
String indexName,
Setting<?>[] additionalSettings
) throws IOException {
Settings.Builder settingsBuilder = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), tempDir).put(settings);
if (settings.get(IndexMetadata.SETTING_VERSION_CREATED) == null) {
settingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current());
}
Settings finalSettings = settingsBuilder.build();
MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexName, finalSettings);
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexName, finalSettings, additionalSettings);
IndexAnalyzers indexAnalyzers = createTestAnalysis(indexSettings, finalSettings).indexAnalyzers;
SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap());
BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP);
Expand Down
28 changes: 17 additions & 11 deletions ...stTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextLicenseDowngradeIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,27 @@ public void checkClusterFeature() {
"template": {
"mappings": {
"properties": {
"pattern_field": {
"type": "pattern_text"
"error": {
"properties": {
"message": {
"type": "pattern_text"
}
}
}
}
}
}
}""";

@SuppressWarnings("unchecked")
private Map<String, Object> resolveSubmap(Map<String, Object> map, String... keys) {
Map<String, Object> curr = map;
for (String key : keys) {
curr = (Map<String, Object>) curr.get(key);
}
return curr;
}

public void testLicenseDowngrade() throws IOException {
final String dataStreamName = "logs-test-pattern-text";

Expand All @@ -48,9 +60,7 @@ public void testLicenseDowngrade() throws IOException {
{
assertEquals("false", getSetting(client(), backingIndex0, "index.mapping.pattern_text.disable_templating"));
Map<String, Object> mapping = getMapping(client(), backingIndex0);
Map<String, Object> patternFieldMapping = (Map<String, Object>) ((Map<String, Object>) mapping.get("properties")).get(
"pattern_field"
);
Map<String, Object> patternFieldMapping = resolveSubmap(mapping, "properties", "error", "properties", "message");
assertThat(patternFieldMapping, not(hasKey("disable_templating")));
}

Expand All @@ -60,19 +70,15 @@ public void testLicenseDowngrade() throws IOException {
{
assertEquals("false", getSetting(client(), backingIndex0, "index.mapping.pattern_text.disable_templating"));
Map<String, Object> mapping = getMapping(client(), backingIndex0);
Map<String, Object> patternFieldMapping = (Map<String, Object>) ((Map<String, Object>) mapping.get("properties")).get(
"pattern_field"
);
Map<String, Object> patternFieldMapping = resolveSubmap(mapping, "properties", "error", "properties", "message");
assertThat(patternFieldMapping, not(hasKey("disable_templating")));
}

String backingIndex1 = getDataStreamBackingIndex(client(), dataStreamName, 1);
{
assertEquals("true", getSetting(client(), backingIndex1, "index.mapping.pattern_text.disable_templating"));
Map<String, Object> mapping = getMapping(client(), backingIndex1);
Map<String, Object> patternFieldMapping = (Map<String, Object>) ((Map<String, Object>) mapping.get("properties")).get(
"pattern_field"
);
Map<String, Object> patternFieldMapping = resolveSubmap(mapping, "properties", "error", "properties", "message");
assertThat(patternFieldMapping, hasEntry("disable_templating", true));
}

Expand Down
8 changes: 4 additions & 4 deletions ...RestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextLicenseUpgradeIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public void checkClusterFeature() {
"template": {
"mappings": {
"properties": {
"pattern_field": {
"message": {
"type": "pattern_text"
}
}
Expand All @@ -48,7 +48,7 @@ public void testLicenseUpgrade() throws IOException {
assertEquals("true", getSetting(client(), backingIndex0, "index.mapping.pattern_text.disable_templating"));
Map<String, Object> mapping = getMapping(client(), backingIndex0);
Map<String, Object> patternFieldMapping = (Map<String, Object>) ((Map<String, Object>) mapping.get("properties")).get(
"pattern_field"
"message"
);
assertThat(patternFieldMapping, hasEntry("disable_templating", true));
}
Expand All @@ -60,7 +60,7 @@ public void testLicenseUpgrade() throws IOException {
assertEquals("true", getSetting(client(), backingIndex0, "index.mapping.pattern_text.disable_templating"));
Map<String, Object> mapping = getMapping(client(), backingIndex0);
Map<String, Object> patternFieldMapping = (Map<String, Object>) ((Map<String, Object>) mapping.get("properties")).get(
"pattern_field"
"message"
);
assertThat(patternFieldMapping, hasEntry("disable_templating", true));
}
Expand All @@ -70,7 +70,7 @@ public void testLicenseUpgrade() throws IOException {
assertEquals("false", getSetting(client(), backingIndex1, "index.mapping.pattern_text.disable_templating"));
Map<String, Object> mapping = getMapping(client(), backingIndex1);
Map<String, Object> patternFieldMapping = (Map<String, Object>) ((Map<String, Object>) mapping.get("properties")).get(
"pattern_field"
"message"
);
assertThat(patternFieldMapping, not(hasKey("disable_templating")));
}
Expand Down
19 changes: 12 additions & 7 deletions .../logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,9 @@ final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider {
"_doc._source.*",
"_doc.properties.host**",
"_doc.properties.resource**",
"_doc.subobjects"
"_doc.subobjects",
"_doc.properties.message",
"_doc.properties.error.properties.message"
);

private final LogsdbLicenseService licenseService;
Expand Down Expand Up @@ -192,13 +194,13 @@ && matchesLogsPattern(dataStreamName)) {
}
}

if (licenseService.allowPatternTextTemplating(isTemplateValidation) == false) {
if (licenseService.allowPatternTextTemplating(isTemplateValidation) == false && mappingHints.maybeUsesPatternText) {
additionalSettings.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true);
}
}

record MappingHints(boolean hasSyntheticSourceUsage, boolean sortOnHostName, boolean addHostNameField) {
static MappingHints EMPTY = new MappingHints(false, false, false);
record MappingHints(boolean hasSyntheticSourceUsage, boolean sortOnHostName, boolean addHostNameField, boolean maybeUsesPatternText) {
static MappingHints EMPTY = new MappingHints(false, false, false, false);
}

private static boolean matchesLogsPattern(final String name) {
Expand Down Expand Up @@ -237,12 +239,12 @@ MappingHints getMappingHints(
hasSyntheticSourceUsage = sourceMode == SourceFieldMapper.Mode.SYNTHETIC;
if (IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(indexTemplateAndCreateRequestSettings).isEmpty() == false) {
// Custom sort config, no point for further checks on [host.name] field.
return new MappingHints(hasSyntheticSourceUsage, false, false);
return new MappingHints(hasSyntheticSourceUsage, false, false, true);
}
if (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(indexTemplateAndCreateRequestSettings)
&& IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(indexTemplateAndCreateRequestSettings)) {
// Settings for adding and sorting on [host.name] are already set, propagate them.
return new MappingHints(hasSyntheticSourceUsage, true, true);
return new MappingHints(hasSyntheticSourceUsage, true, true, true);
}
}

Expand Down Expand Up @@ -277,7 +279,10 @@ MappingHints getMappingHints(
|| addHostNameField
|| (hostName instanceof NumberFieldMapper nfm && nfm.fieldType().hasDocValues())
|| (hostName instanceof KeywordFieldMapper kfm && kfm.fieldType().hasDocValues());
return new MappingHints(hasSyntheticSourceUsage, sortOnHostName, addHostNameField);
boolean usesPatternText = PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.get(indexTemplateAndCreateRequestSettings)
|| mapperService.mappingLookup().getMapper("message") instanceof PatternTextFieldMapper
|| mapperService.mappingLookup().getMapper("error.message") instanceof PatternTextFieldMapper;
return new MappingHints(hasSyntheticSourceUsage, sortOnHostName, addHostNameField, usesPatternText);
}
} catch (AssertionError | Exception e) {
// In case invalid mappings or setting are provided, then mapper service creation can fail.
Expand Down
67 changes: 51 additions & 16 deletions ...db/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.elasticsearch.cluster.metadata.ProjectMetadata;
import org.elasticsearch.cluster.metadata.Template;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.index.IndexMode;
Expand All @@ -26,9 +27,12 @@
import org.elasticsearch.index.IndexVersion;
import org.elasticsearch.index.MapperTestUtils;
import org.elasticsearch.index.mapper.SourceFieldMapper;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.license.License;
import org.elasticsearch.license.LicenseService;
import org.elasticsearch.license.MockLicenseState;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.license.internal.XPackLicenseStatus;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.logsdb.patterntext.PatternTextFieldMapper;
import org.junit.Before;
Expand Down Expand Up @@ -72,6 +76,7 @@ public class LogsdbIndexModeSettingsProviderTests extends ESTestCase {
""";

private LogsdbLicenseService logsdbLicenseService;
private LogsdbLicenseService basicLogsdbLicenseService;
private final AtomicInteger newMapperServiceCounter = new AtomicInteger();

@Before
Expand All @@ -84,6 +89,13 @@ public void setup() throws Exception {
logsdbLicenseService = new LogsdbLicenseService(Settings.EMPTY);
logsdbLicenseService.setLicenseState(licenseState);
logsdbLicenseService.setLicenseService(mockLicenseService);

var basicLicenseState = new XPackLicenseState(() -> 0L, new XPackLicenseStatus(License.OperationMode.BASIC, true, null));
var basicLicenseService = mock(LicenseService.class);
when(basicLicenseService.getLicense()).thenReturn(null);
basicLogsdbLicenseService = new LogsdbLicenseService(Settings.EMPTY);
basicLogsdbLicenseService.setLicenseState(basicLicenseState);
basicLogsdbLicenseService.setLicenseService(basicLicenseService);
}

private LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled) {
Expand Down Expand Up @@ -121,13 +133,23 @@ private Settings generateLogsdbSettings(Settings settings, String mapping) throw
}

private Settings generateLogsdbSettings(Settings settings, String mapping, Version version) throws IOException {
var provider = new LogsdbIndexModeSettingsProvider(
logsdbLicenseService,
Settings.builder().put("cluster.logsdb.enabled", true).build()
);
return generateLogsdbSettings(settings, mapping, version, logsdbLicenseService);
}

private Settings generateLogsdbSettings(Settings settings, String mapping, Version version, LogsdbLicenseService licenseService)
throws IOException {
var provider = new LogsdbIndexModeSettingsProvider(licenseService, Settings.builder().put("cluster.logsdb.enabled", true).build());
var logsdbPlugin = new LogsDBPlugin(settings);
provider.init(im -> {
newMapperServiceCounter.incrementAndGet();
return MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName());
return MapperTestUtils.newMapperService(
xContentRegistry(),
createTempDir(),
im.getSettings(),
new IndicesModule(List.of(logsdbPlugin)),
im.getIndex().getName(),
logsdbPlugin.getSettings().stream().filter(Setting::hasIndexScope).toArray(Setting<?>[]::new)
);
}, IndexVersion::current, () -> version, true, true);
Settings.Builder settingsBuilder = builder();
provider.provideAdditionalSettings(
Expand Down Expand Up @@ -959,19 +981,32 @@ public void testExplicitRoutingPathNotAllowedByLicense() throws Exception {
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), empty());
}

public void testPatternTextNotAllowedByLicense() throws Exception {
MockLicenseState licenseState = MockLicenseState.createMock();
when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState);
when(licenseState.isAllowed(same(LogsdbLicenseService.PATTERN_TEXT_TEMPLATING_FEATURE))).thenReturn(false);
logsdbLicenseService = new LogsdbLicenseService(Settings.EMPTY);
logsdbLicenseService.setLicenseState(licenseState);
public void testPatternTextNotAllowedByLicense() throws IOException {
String[] patternTextLicenceCheckedFieldMappings = {
"{\"_doc\":{\"properties\":{\"message\":{\"type\":\"pattern_text\"}}}}",
"{\"_doc\":{\"properties\":{\"error\":{\"properties\":{\"message\":{\"type\":\"pattern_text\"}}}}}}}" };

var settings = Settings.builder()
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message")
.put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), true)
var expectedSettings = Settings.builder()
.put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true)
.put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true)
.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true)
.build();
Settings result = generateLogsdbSettings(settings);
assertTrue(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.get(result));

for (String mapping : patternTextLicenceCheckedFieldMappings) {
var result = generateLogsdbSettings(Settings.EMPTY, mapping, Version.CURRENT, basicLogsdbLicenseService);
assertEquals(expectedSettings, result);
}
}

public void testPatternTextNotAllowedByLicenseAlreadyDisallowed() throws IOException {
Settings settings = Settings.builder().put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), "true").build();
var result = generateLogsdbSettings(settings, null, Version.CURRENT, basicLogsdbLicenseService);
var expected = Settings.builder()
.put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true)
.put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true)
.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true)
.build();
assertEquals(expected, result);
}

public void testSortAndHostNamePropagateValue() throws Exception {
Expand Down
Loading