Skip to content

Only disable unlicensed pattern_text templating when mapper is used #135561

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Sep 29, 2025
Merged

Only disable unlicensed pattern_text templating when mapper is used #135561

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ff9f1d1
Only disable pattern_text templating for message and error.message fi…
jordan-powers Sep 26, 2025
124241b
Update tests
jordan-powers Sep 26, 2025
c840902
Check mappings for pattern_text to determine usage
jordan-powers Sep 26, 2025
f4f4d0c
Update tests
jordan-powers Sep 26, 2025
1942ce4
Use MappingVisitor to check for pattern_text usage
jordan-powers Sep 29, 2025
1aff2d4
Merge remote-tracking branch 'upstream/main' into disable-unlicensed-…
jordan-powers Sep 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

import org.elasticsearch.TransportVersion;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.env.Environment;
Expand Down Expand Up @@ -61,14 +62,25 @@ public static MapperService newMapperService(
Settings settings,
IndicesModule indicesModule,
String indexName
) throws IOException {
return newMapperService(xContentRegistry, tempDir, settings, indicesModule, indexName, new Setting<?>[0]);
}

public static MapperService newMapperService(
NamedXContentRegistry xContentRegistry,
Path tempDir,
Settings settings,
IndicesModule indicesModule,
String indexName,
Setting<?>[] additionalSettings
) throws IOException {
Settings.Builder settingsBuilder = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), tempDir).put(settings);
if (settings.get(IndexMetadata.SETTING_VERSION_CREATED) == null) {
settingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current());
}
Settings finalSettings = settingsBuilder.build();
MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexName, finalSettings);
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexName, finalSettings, additionalSettings);
IndexAnalyzers indexAnalyzers = createTestAnalysis(indexSettings, finalSettings).indexAnalyzers;
SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap());
BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP);
Expand Down
17 changes: 11 additions & 6 deletions .../logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.elasticsearch.index.mapper.SourceFieldMapper;
import org.elasticsearch.xcontent.XContentType;
import org.elasticsearch.xpack.logsdb.patterntext.PatternTextFieldMapper;
import org.elasticsearch.xpack.logsdb.patterntext.PatternTextFieldType;

import java.io.IOException;
import java.time.Instant;
Expand Down Expand Up @@ -192,13 +193,13 @@ && matchesLogsPattern(dataStreamName)) {
}
}

if (licenseService.allowPatternTextTemplating(isTemplateValidation) == false) {
if (licenseService.allowPatternTextTemplating(isTemplateValidation) == false && mappingHints.maybeUsesPatternText) {
additionalSettings.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true);
}
}

record MappingHints(boolean hasSyntheticSourceUsage, boolean sortOnHostName, boolean addHostNameField) {
static MappingHints EMPTY = new MappingHints(false, false, false);
record MappingHints(boolean hasSyntheticSourceUsage, boolean sortOnHostName, boolean addHostNameField, boolean maybeUsesPatternText) {
static MappingHints EMPTY = new MappingHints(false, false, false, false);
}

private static boolean matchesLogsPattern(final String name) {
Expand Down Expand Up @@ -237,16 +238,17 @@ MappingHints getMappingHints(
hasSyntheticSourceUsage = sourceMode == SourceFieldMapper.Mode.SYNTHETIC;
if (IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(indexTemplateAndCreateRequestSettings).isEmpty() == false) {
// Custom sort config, no point for further checks on [host.name] field.
return new MappingHints(hasSyntheticSourceUsage, false, false);
return new MappingHints(hasSyntheticSourceUsage, false, false, true);
}
if (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(indexTemplateAndCreateRequestSettings)
&& IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(indexTemplateAndCreateRequestSettings)) {
// Settings for adding and sorting on [host.name] are already set, propagate them.
return new MappingHints(hasSyntheticSourceUsage, true, true);
return new MappingHints(hasSyntheticSourceUsage, true, true, true);
}
}

try (var mapperService = mapperServiceFactory.get().apply(tmpIndexMetadata)) {
boolean maybeUsesPatternText = PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.get(indexTemplateAndCreateRequestSettings);
// combinedTemplateMappings can be null when creating system indices
// combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping.
if (combinedTemplateMappings == null || combinedTemplateMappings.isEmpty()) {
Expand All @@ -260,6 +262,9 @@ MappingHints getMappingHints(
List<CompressedXContent> filteredMappings = new ArrayList<>(combinedTemplateMappings.size());
for (CompressedXContent mappingSource : combinedTemplateMappings) {
var ref = mappingSource.compressedReference();
if (maybeUsesPatternText == false) {
maybeUsesPatternText = mappingSource.string().contains(PatternTextFieldType.CONTENT_TYPE);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that just checking whether mapping contains pattern_text string can also give false positives. For example if field name is named pattern_text. We would need to search for "type":"pattern_text", but we would need to be lenient with whitespaces.

Maybe instead we should MappingVisitor here after line 268. Then we already parsed the mapping as a map. Then checking for type key with pattern_text value is less error prone.

}
var map = XContentHelper.convertToMap(ref, true, XContentType.JSON, MAPPING_INCLUDES, Set.of()).v2();
filteredMappings.add(new CompressedXContent(map));
}
Expand All @@ -277,7 +282,7 @@ MappingHints getMappingHints(
|| addHostNameField
|| (hostName instanceof NumberFieldMapper nfm && nfm.fieldType().hasDocValues())
|| (hostName instanceof KeywordFieldMapper kfm && kfm.fieldType().hasDocValues());
return new MappingHints(hasSyntheticSourceUsage, sortOnHostName, addHostNameField);
return new MappingHints(hasSyntheticSourceUsage, sortOnHostName, addHostNameField, maybeUsesPatternText);
}
} catch (AssertionError | Exception e) {
// In case invalid mappings or setting are provided, then mapper service creation can fail.
Expand Down
69 changes: 53 additions & 16 deletions ...db/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.elasticsearch.cluster.metadata.ProjectMetadata;
import org.elasticsearch.cluster.metadata.Template;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.index.IndexMode;
Expand All @@ -26,9 +27,12 @@
import org.elasticsearch.index.IndexVersion;
import org.elasticsearch.index.MapperTestUtils;
import org.elasticsearch.index.mapper.SourceFieldMapper;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.license.License;
import org.elasticsearch.license.LicenseService;
import org.elasticsearch.license.MockLicenseState;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.license.internal.XPackLicenseStatus;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.logsdb.patterntext.PatternTextFieldMapper;
import org.junit.Before;
Expand Down Expand Up @@ -72,6 +76,7 @@ public class LogsdbIndexModeSettingsProviderTests extends ESTestCase {
""";

private LogsdbLicenseService logsdbLicenseService;
private LogsdbLicenseService basicLogsdbLicenseService;
private final AtomicInteger newMapperServiceCounter = new AtomicInteger();

@Before
Expand All @@ -84,6 +89,13 @@ public void setup() throws Exception {
logsdbLicenseService = new LogsdbLicenseService(Settings.EMPTY);
logsdbLicenseService.setLicenseState(licenseState);
logsdbLicenseService.setLicenseService(mockLicenseService);

var basicLicenseState = new XPackLicenseState(() -> 0L, new XPackLicenseStatus(License.OperationMode.BASIC, true, null));
var basicLicenseService = mock(LicenseService.class);
when(basicLicenseService.getLicense()).thenReturn(null);
basicLogsdbLicenseService = new LogsdbLicenseService(Settings.EMPTY);
basicLogsdbLicenseService.setLicenseState(basicLicenseState);
basicLogsdbLicenseService.setLicenseService(basicLicenseService);
}

private LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled) {
Expand Down Expand Up @@ -121,13 +133,23 @@ private Settings generateLogsdbSettings(Settings settings, String mapping) throw
}

private Settings generateLogsdbSettings(Settings settings, String mapping, Version version) throws IOException {
var provider = new LogsdbIndexModeSettingsProvider(
logsdbLicenseService,
Settings.builder().put("cluster.logsdb.enabled", true).build()
);
return generateLogsdbSettings(settings, mapping, version, logsdbLicenseService);
}

private Settings generateLogsdbSettings(Settings settings, String mapping, Version version, LogsdbLicenseService licenseService)
throws IOException {
var provider = new LogsdbIndexModeSettingsProvider(licenseService, Settings.builder().put("cluster.logsdb.enabled", true).build());
var logsdbPlugin = new LogsDBPlugin(settings);
provider.init(im -> {
newMapperServiceCounter.incrementAndGet();
return MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName());
return MapperTestUtils.newMapperService(
xContentRegistry(),
createTempDir(),
im.getSettings(),
new IndicesModule(List.of(logsdbPlugin)),
im.getIndex().getName(),
logsdbPlugin.getSettings().stream().filter(Setting::hasIndexScope).toArray(Setting<?>[]::new)
);
}, IndexVersion::current, () -> version, true, true);
Settings.Builder settingsBuilder = builder();
provider.provideAdditionalSettings(
Expand Down Expand Up @@ -959,19 +981,34 @@ public void testExplicitRoutingPathNotAllowedByLicense() throws Exception {
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), empty());
}

public void testPatternTextNotAllowedByLicense() throws Exception {
MockLicenseState licenseState = MockLicenseState.createMock();
when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState);
when(licenseState.isAllowed(same(LogsdbLicenseService.PATTERN_TEXT_TEMPLATING_FEATURE))).thenReturn(false);
logsdbLicenseService = new LogsdbLicenseService(Settings.EMPTY);
logsdbLicenseService.setLicenseState(licenseState);
public void testPatternTextNotAllowedByLicense() throws IOException {
String[] patternTextLicenceCheckedFieldMappings = {
"{\"_doc\":{\"properties\":{\"message\":{\"type\":\"pattern_text\"}}}}",
"{\"_doc\":{\"properties\":{\"error\":{\"properties\":{\"message\":{\"type\":\"pattern_text\"}}}}}}",
"{\"_doc\":{\"properties\":{\"foo\":{\"type\":\"pattern_text\"}}}}",
"{\"_doc\":{\"properties\":{\"bar\":{\"properties\":{\"baz\":{\"type\":\"pattern_text\"}}}}}}" };

var settings = Settings.builder()
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message")
.put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), true)
var expectedSettings = Settings.builder()
.put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true)
.put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true)
.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true)
.build();
Settings result = generateLogsdbSettings(settings);
assertTrue(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.get(result));

for (String mapping : patternTextLicenceCheckedFieldMappings) {
var result = generateLogsdbSettings(Settings.EMPTY, mapping, Version.CURRENT, basicLogsdbLicenseService);
assertEquals(expectedSettings, result);
}
}

public void testPatternTextNotAllowedByLicenseAlreadyDisallowed() throws IOException {
Settings settings = Settings.builder().put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), "true").build();
var result = generateLogsdbSettings(settings, null, Version.CURRENT, basicLogsdbLicenseService);
var expected = Settings.builder()
.put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true)
.put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true)
.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true)
.build();
assertEquals(expected, result);
}

public void testSortAndHostNamePropagateValue() throws Exception {
Expand Down
28 changes: 6 additions & 22 deletions ...st/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.license.internal.XPackLicenseStatus;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.logsdb.patterntext.PatternTextFieldMapper;
import org.junit.Before;

import java.io.IOException;
Expand Down Expand Up @@ -77,10 +76,7 @@ public void testGetAdditionalIndexSettingsDefault() {
builder
);
var result = builder.build();
var expected = Settings.builder()
.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "STORED")
.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true)
.build();
var expected = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "STORED").build();
assertEquals(expected, result);
}

Expand All @@ -101,17 +97,11 @@ public void testGetAdditionalIndexSettingsApm() throws IOException {
builder
);
var result = builder.build();
Settings expectedAdditionalSettings = Settings.builder()
.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true)
.build();
assertEquals(expectedAdditionalSettings, result);
assertEquals(Settings.EMPTY, result);
}

public void testGetAdditionalIndexSettingsProfiling() throws IOException {
Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build();
Settings expectedAdditionalSettings = Settings.builder()
.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true)
.build();
for (String dataStreamName : new String[] { "profiling-metrics", "profiling-events" }) {
String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0);
Settings.Builder builder = Settings.builder();
Expand All @@ -127,14 +117,14 @@ public void testGetAdditionalIndexSettingsProfiling() throws IOException {
builder
);
var result = builder.build();
assertEquals(expectedAdditionalSettings, result);
assertEquals(Settings.EMPTY, result);
}

for (String indexName : new String[] { ".profiling-sq-executables", ".profiling-sq-leafframes", ".profiling-stacktraces" }) {
Settings.Builder builder = Settings.builder();
provider.provideAdditionalSettings(indexName, null, null, null, null, settings, List.of(), IndexVersion.current(), builder);
var result = builder.build();
assertEquals(expectedAdditionalSettings, result);
assertEquals(Settings.EMPTY, result);
}
}

Expand All @@ -155,10 +145,7 @@ public void testGetAdditionalIndexSettingsTsdb() throws IOException {
builder
);
var result = builder.build();
Settings expectedAdditionalSettings = Settings.builder()
.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true)
.build();
assertEquals(expectedAdditionalSettings, result);
assertEquals(Settings.EMPTY, result);
}

public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception {
Expand Down Expand Up @@ -202,10 +189,7 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception
);

var result = builder.build();
var expected = Settings.builder()
.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "STORED")
.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true)
.build();
var expected = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "STORED").build();
assertEquals(expected, result);
}
}