Skip to content

Commit 0a3247d

Browse files
jordan-powersjordan-powers
authored
Add disable_templating index setting/mapping parameter to patterned_text (#134466)
This patch is the first step to gate the storage savings of the patterned_text field mapper behind an enterprise license. A new index setting ``index.mapping.patterned_text.disable_templating` is set on index creation if the current license does not support the patterned text enterprise-level storage savings. This index setting is then used to set the parameter disable_templating on the patterned text mapper.
1 parent 2c6c042 commit 0a3247d

18 files changed

+443
-50
lines changed

‎test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929
import org.elasticsearch.common.compress.CompressedXContent;
3030
import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
3131
import org.elasticsearch.common.settings.ClusterSettings;
32+
import org.elasticsearch.common.settings.IndexScopedSettings;
33+
import org.elasticsearch.common.settings.Setting;
3234
import org.elasticsearch.common.settings.Settings;
3335
import org.elasticsearch.common.util.BigArrays;
3436
import org.elasticsearch.common.util.MockBigArrays;
@@ -88,6 +90,7 @@
8890
import java.io.IOException;
8991
import java.util.Collection;
9092
import java.util.Collections;
93+
import java.util.HashSet;
9194
import java.util.List;
9295
import java.util.Map;
9396
import java.util.Optional;
@@ -282,10 +285,15 @@ public TestMapperServiceBuilder applyDefaultMapping(boolean applyDefaultMapping)
282285
}
283286

284287
public MapperService build() {
285-
IndexSettings indexSettings = createIndexSettings(indexVersion, settings);
288+
Collection<? extends Plugin> plugins = getPlugins();
289+
Collection<Setting<?>> pluginIndexSettings = plugins.stream()
290+
.flatMap(plugin -> plugin.getSettings().stream())
291+
.filter(Setting::hasIndexScope)
292+
.toList();
293+
IndexSettings indexSettings = createIndexSettings(indexVersion, settings, pluginIndexSettings);
286294
SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
287295
MapperRegistry mapperRegistry = new IndicesModule(
288-
getPlugins().stream().filter(p -> p instanceof MapperPlugin).map(p -> (MapperPlugin) p).collect(toList())
296+
plugins.stream().filter(p -> p instanceof MapperPlugin).map(p -> (MapperPlugin) p).collect(toList())
289297
).getMapperRegistry();
290298

291299
BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP);
@@ -327,9 +335,19 @@ protected <T> T compileScript(Script script, ScriptContext<T> context) {
327335
}
328336

329337
protected static IndexSettings createIndexSettings(IndexVersion version, Settings settings) {
338+
return createIndexSettings(version, settings, List.of());
339+
}
340+
341+
protected static IndexSettings createIndexSettings(
342+
IndexVersion version,
343+
Settings settings,
344+
Collection<Setting<?>> pluginIndexSettings
345+
) {
330346
settings = indexSettings(1, 0).put(settings).put(IndexMetadata.SETTING_VERSION_CREATED, version).build();
331347
IndexMetadata meta = IndexMetadata.builder("index").settings(settings).build();
332-
return new IndexSettings(meta, settings);
348+
Set<Setting<?>> indexSettings = new HashSet<>(IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
349+
indexSettings.addAll(pluginIndexSettings);
350+
return new IndexSettings(meta, settings, new IndexScopedSettings(Settings.EMPTY, indexSettings));
333351
}
334352

335353
protected MapperMetrics createTestMapperMetrics() {

‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/DataStreamLicenseChangeTestCase.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.logsdb;
9+
10+
import org.elasticsearch.client.Request;
11+
import org.elasticsearch.test.cluster.ElasticsearchCluster;
12+
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
13+
import org.junit.ClassRule;
14+
15+
import java.io.IOException;
16+
17+
public abstract class DataStreamLicenseChangeTestCase extends LogsIndexModeRestTestIT {
18+
@ClassRule
19+
public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
20+
.distribution(DistributionType.DEFAULT)
21+
.module("data-streams")
22+
.module("x-pack-stack")
23+
.setting("cluster.logsdb.enabled", "true")
24+
.setting("xpack.security.enabled", "false")
25+
.setting("xpack.license.self_generated.type", "basic")
26+
.build();
27+
28+
@Override
29+
protected String getTestRestCluster() {
30+
return cluster.getHttpAddresses();
31+
}
32+
33+
protected static void startBasic() throws IOException {
34+
Request startTrial = new Request("POST", "/_license/start_basic");
35+
startTrial.addParameter("acknowledge", "true");
36+
assertOK(client().performRequest(startTrial));
37+
}
38+
39+
protected static void startTrial() throws IOException {
40+
Request startTrial = new Request("POST", "/_license/start_trial");
41+
startTrial.addParameter("acknowledge", "true");
42+
assertOK(client().performRequest(startTrial));
43+
}
44+
45+
}

‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsIndexModeRestTestIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,10 @@ protected static void waitForLogs(RestClient client) throws Exception {
3131
});
3232
}
3333

34-
protected static Response putComponentTemplate(final RestClient client, final String componentTemplate, final String contends)
34+
protected static Response putComponentTemplate(final RestClient client, final String componentTemplate, final String contents)
3535
throws IOException {
3636
final Request request = new Request("PUT", "/_component_template/" + componentTemplate);
37-
request.setJsonEntity(contends);
37+
request.setJsonEntity(contents);
3838
return client.performRequest(request);
3939
}
4040

‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,10 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException {
8181
boolean found = false;
8282
for (var feature : features) {
8383
if (feature.get("family") != null) {
84-
assertThat(feature.get("name"), anyOf(equalTo("synthetic-source"), equalTo("logsdb-routing-on-sort-fields")));
84+
assertThat(
85+
feature.get("name"),
86+
anyOf(equalTo("synthetic-source"), equalTo("logsdb-routing-on-sort-fields"), equalTo("patterned-text-templating"))
87+
);
8588
assertThat(feature.get("license_level"), equalTo("enterprise"));
8689
found = true;
8790
}

‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/DataStreamLicenseChangeIT.java‎ renamed to ‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/SourceModeLicenseChangeTestCase.java‎

Lines changed: 1 addition & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -11,28 +11,11 @@
1111
import org.elasticsearch.client.Response;
1212
import org.elasticsearch.client.RestClient;
1313
import org.elasticsearch.index.mapper.SourceFieldMapper;
14-
import org.elasticsearch.test.cluster.ElasticsearchCluster;
15-
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
16-
import org.junit.ClassRule;
1714

1815
import java.io.IOException;
1916
import java.util.List;
2017

21-
public abstract class DataStreamLicenseChangeIT extends LogsIndexModeRestTestIT {
22-
@ClassRule
23-
public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
24-
.distribution(DistributionType.DEFAULT)
25-
.module("data-streams")
26-
.module("x-pack-stack")
27-
.setting("cluster.logsdb.enabled", "true")
28-
.setting("xpack.security.enabled", "false")
29-
.setting("xpack.license.self_generated.type", "basic")
30-
.build();
31-
32-
@Override
33-
protected String getTestRestCluster() {
34-
return cluster.getHttpAddresses();
35-
}
18+
public abstract class SourceModeLicenseChangeTestCase extends DataStreamLicenseChangeTestCase {
3619

3720
protected interface TestCase {
3821
String dataStreamName();
@@ -88,18 +71,6 @@ public void testLicenseChange() throws IOException {
8871
}
8972
}
9073

91-
protected static void startBasic() throws IOException {
92-
Request startTrial = new Request("POST", "/_license/start_basic");
93-
startTrial.addParameter("acknowledge", "true");
94-
assertOK(client().performRequest(startTrial));
95-
}
96-
97-
protected static void startTrial() throws IOException {
98-
Request startTrial = new Request("POST", "/_license/start_trial");
99-
startTrial.addParameter("acknowledge", "true");
100-
assertOK(client().performRequest(startTrial));
101-
}
102-
10374
protected static Response removeComponentTemplate(final RestClient client, final String componentTemplate) throws IOException {
10475
final Request request = new Request("DELETE", "/_component_template/" + componentTemplate);
10576
return client.performRequest(request);

‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/DataStreamLicenceDowngradeIT.java‎ renamed to ‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/SourceModeLicenseDowngradeIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
import java.io.IOException;
1313
import java.util.List;
1414

15-
public class DataStreamLicenceDowngradeIT extends DataStreamLicenseChangeIT {
15+
public class SourceModeLicenseDowngradeIT extends SourceModeLicenseChangeTestCase {
1616
@Override
1717
protected void applyInitialLicense() throws IOException {
1818
startTrial();

‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/DataStreamLicenseUpgradeIT.java‎ renamed to ‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/SourceModeLicenseUpgradeIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
import java.io.IOException;
1313
import java.util.List;
1414

15-
public class DataStreamLicenseUpgradeIT extends DataStreamLicenseChangeIT {
15+
public class SourceModeLicenseUpgradeIT extends SourceModeLicenseChangeTestCase {
1616
@Override
1717
protected void applyInitialLicense() {}
1818

‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextLicenseDowngradeIT.java‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.logsdb.patternedtext;
9+
10+
import org.elasticsearch.Build;
11+
import org.elasticsearch.xpack.logsdb.DataStreamLicenseChangeTestCase;
12+
import org.junit.Before;
13+
14+
import java.io.IOException;
15+
import java.util.Map;
16+
17+
import static org.hamcrest.Matchers.hasEntry;
18+
import static org.hamcrest.Matchers.hasKey;
19+
import static org.hamcrest.Matchers.not;
20+
21+
public class PatternedTextLicenseDowngradeIT extends DataStreamLicenseChangeTestCase {
22+
@Before
23+
public void checkClusterFeature() {
24+
assumeTrue("[patterned_text] must be available", clusterHasFeature("mapper.patterned_text"));
25+
assumeTrue("[patterned_text] is only available in snapshot builds", Build.current().isSnapshot());
26+
}
27+
28+
private static final String patternedTextMapping = """
29+
{
30+
"template": {
31+
"mappings": {
32+
"properties": {
33+
"patterned_field": {
34+
"type": "patterned_text"
35+
}
36+
}
37+
}
38+
}
39+
}""";
40+
41+
@SuppressWarnings("unchecked")
42+
public void testLicenseDowngrade() throws IOException {
43+
final String dataStreamName = "logs-test-patterned-text";
44+
45+
startTrial();
46+
assertOK(putComponentTemplate(client(), "logs@custom", patternedTextMapping));
47+
assertOK(createDataStream(client(), dataStreamName));
48+
49+
String backingIndex0 = getDataStreamBackingIndex(client(), dataStreamName, 0);
50+
{
51+
assertEquals("false", getSetting(client(), backingIndex0, "index.mapping.patterned_text.disable_templating"));
52+
Map<String, Object> mapping = getMapping(client(), backingIndex0);
53+
Map<String, Object> patternedFieldMapping = (Map<String, Object>) ((Map<String, Object>) mapping.get("properties")).get(
54+
"patterned_field"
55+
);
56+
assertThat(patternedFieldMapping, not(hasKey("disable_templating")));
57+
}
58+
59+
startBasic();
60+
rolloverDataStream(client(), dataStreamName);
61+
62+
{
63+
assertEquals("false", getSetting(client(), backingIndex0, "index.mapping.patterned_text.disable_templating"));
64+
Map<String, Object> mapping = getMapping(client(), backingIndex0);
65+
Map<String, Object> patternedFieldMapping = (Map<String, Object>) ((Map<String, Object>) mapping.get("properties")).get(
66+
"patterned_field"
67+
);
68+
assertThat(patternedFieldMapping, not(hasKey("disable_templating")));
69+
}
70+
71+
String backingIndex1 = getDataStreamBackingIndex(client(), dataStreamName, 1);
72+
{
73+
assertEquals("true", getSetting(client(), backingIndex1, "index.mapping.patterned_text.disable_templating"));
74+
Map<String, Object> mapping = getMapping(client(), backingIndex1);
75+
Map<String, Object> patternedFieldMapping = (Map<String, Object>) ((Map<String, Object>) mapping.get("properties")).get(
76+
"patterned_field"
77+
);
78+
assertThat(patternedFieldMapping, hasEntry("disable_templating", true));
79+
}
80+
81+
}
82+
}

‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextLicenseUpgradeIT.java‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.logsdb.patternedtext;
9+
10+
import org.elasticsearch.Build;
11+
import org.elasticsearch.xpack.logsdb.DataStreamLicenseChangeTestCase;
12+
import org.junit.Before;
13+
14+
import java.io.IOException;
15+
import java.util.Map;
16+
17+
import static org.hamcrest.Matchers.hasEntry;
18+
import static org.hamcrest.Matchers.hasKey;
19+
import static org.hamcrest.Matchers.not;
20+
21+
public class PatternedTextLicenseUpgradeIT extends DataStreamLicenseChangeTestCase {
22+
@Before
23+
public void checkClusterFeature() {
24+
assumeTrue("[patterned_text] must be available", clusterHasFeature("mapper.patterned_text"));
25+
assumeTrue("[patterned_text] is only available in snapshot builds", Build.current().isSnapshot());
26+
}
27+
28+
private static final String patternedTextMapping = """
29+
{
30+
"template": {
31+
"mappings": {
32+
"properties": {
33+
"patterned_field": {
34+
"type": "patterned_text"
35+
}
36+
}
37+
}
38+
}
39+
}""";
40+
41+
@SuppressWarnings("unchecked")
42+
public void testLicenseUpgrade() throws IOException {
43+
final String dataStreamName = "logs-test-patterned-text";
44+
45+
assertOK(putComponentTemplate(client(), "logs@custom", patternedTextMapping));
46+
assertOK(createDataStream(client(), dataStreamName));
47+
48+
String backingIndex0 = getDataStreamBackingIndex(client(), dataStreamName, 0);
49+
{
50+
assertEquals("true", getSetting(client(), backingIndex0, "index.mapping.patterned_text.disable_templating"));
51+
Map<String, Object> mapping = getMapping(client(), backingIndex0);
52+
Map<String, Object> patternedFieldMapping = (Map<String, Object>) ((Map<String, Object>) mapping.get("properties")).get(
53+
"patterned_field"
54+
);
55+
assertThat(patternedFieldMapping, hasEntry("disable_templating", true));
56+
}
57+
58+
startTrial();
59+
rolloverDataStream(client(), dataStreamName);
60+
61+
{
62+
assertEquals("true", getSetting(client(), backingIndex0, "index.mapping.patterned_text.disable_templating"));
63+
Map<String, Object> mapping = getMapping(client(), backingIndex0);
64+
Map<String, Object> patternedFieldMapping = (Map<String, Object>) ((Map<String, Object>) mapping.get("properties")).get(
65+
"patterned_field"
66+
);
67+
assertThat(patternedFieldMapping, hasEntry("disable_templating", true));
68+
}
69+
70+
String backingIndex1 = getDataStreamBackingIndex(client(), dataStreamName, 1);
71+
{
72+
assertEquals("false", getSetting(client(), backingIndex1, "index.mapping.patterned_text.disable_templating"));
73+
Map<String, Object> mapping = getMapping(client(), backingIndex1);
74+
Map<String, Object> patternedFieldMapping = (Map<String, Object>) ((Map<String, Object>) mapping.get("properties")).get(
75+
"patterned_field"
76+
);
77+
assertThat(patternedFieldMapping, not(hasKey("disable_templating")));
78+
}
79+
80+
}
81+
}

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
import java.util.ArrayList;
2828
import java.util.Collection;
29+
import java.util.Collections;
2930
import java.util.List;
3031
import java.util.Map;
3132

@@ -93,7 +94,11 @@ public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(Index
9394

9495
@Override
9596
public List<Setting<?>> getSettings() {
96-
return List.of(FALLBACK_SETTING, CLUSTER_LOGSDB_ENABLED, LOGSDB_PRIOR_LOGS_USAGE);
97+
List<Setting<?>> settings = new ArrayList<>(List.of(FALLBACK_SETTING, CLUSTER_LOGSDB_ENABLED, LOGSDB_PRIOR_LOGS_USAGE));
98+
if (PatternedTextFieldMapper.PATTERNED_TEXT_MAPPER.isEnabled()) {
99+
settings.add(PatternedTextFieldMapper.DISABLE_TEMPLATING_SETTING);
100+
}
101+
return Collections.unmodifiableList(settings);
97102
}
98103

99104
@Override

0 commit comments

Comments
 (0)