diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java index 0637fc80f6644..db6c12c8bc565 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.hamcrest.Matchers; import org.junit.Before; import org.junit.ClassRule; @@ -357,6 +358,66 @@ public void testIgnoreMalformedSetting() throws IOException { } } + public void testIgnoreAboveSetting() throws IOException { + // with default template + { + assertOK(createDataStream(client, "logs-test-1")); + String logsIndex1 = getDataStreamBackingIndex(client, "logs-test-1", 0); + assertThat(getSetting(client, logsIndex1, "index.mapping.ignore_above"), equalTo("8191")); + for (String newValue : List.of("512", "2048", "12000", String.valueOf(Integer.MAX_VALUE))) { + closeIndex(logsIndex1); + updateIndexSettings(logsIndex1, Settings.builder().put("index.mapping.ignore_above", newValue)); + assertThat(getSetting(client, logsIndex1, "index.mapping.ignore_above"), equalTo(newValue)); + } + for (String newValue : List.of(String.valueOf((long) Integer.MAX_VALUE + 1), String.valueOf(Long.MAX_VALUE))) { + closeIndex(logsIndex1); + ResponseException ex = assertThrows( + ResponseException.class, + () -> updateIndexSettings(logsIndex1, Settings.builder().put("index.mapping.ignore_above", newValue)) + ); + assertThat( + ex.getMessage(), + Matchers.containsString("Failed to parse value [" + newValue + "] for setting [index.mapping.ignore_above]") + ); + } + } + // with override template + { + var template = """ + { + "template": { + "settings": { + "index": { + "mapping": { + "ignore_above": "128" + } + } + } + } + }"""; + assertOK(putComponentTemplate(client, "logs@custom", template)); + assertOK(createDataStream(client, "logs-custom-dev")); + String index = getDataStreamBackingIndex(client, "logs-custom-dev", 0); + assertThat(getSetting(client, index, "index.mapping.ignore_above"), equalTo("128")); + for (String newValue : List.of("64", "256", "12000", String.valueOf(Integer.MAX_VALUE))) { + closeIndex(index); + updateIndexSettings(index, Settings.builder().put("index.mapping.ignore_above", newValue)); + assertThat(getSetting(client, index, "index.mapping.ignore_above"), equalTo(newValue)); + } + } + // standard index + { + String index = "test-index"; + createIndex(index); + assertThat(getSetting(client, index, "index.mapping.ignore_above"), equalTo(Integer.toString(Integer.MAX_VALUE))); + for (String newValue : List.of("256", "512", "12000", String.valueOf(Integer.MAX_VALUE))) { + closeIndex(index); + updateIndexSettings(index, Settings.builder().put("index.mapping.ignore_above", newValue)); + assertThat(getSetting(client, index, "index.mapping.ignore_above"), equalTo(newValue)); + } + } + } + private static Map getMapping(final RestClient client, final String indexName) throws IOException { final Request request = new Request("GET", "/" + indexName + "/_mapping"); diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index 9a235bc4e162c..9481116187ba1 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -1366,6 +1366,16 @@ public static Setting intSetting(String key, int defaultValue, int minV return new Setting<>(key, Integer.toString(defaultValue), intParser(key, minValue, properties), properties); } + public static Setting intSetting( + String key, + Function defaultValueFn, + int minValue, + int maxValue, + Property... properties + ) { + return new Setting<>(key, defaultValueFn, intParser(key, minValue, maxValue, properties), properties); + } + private static Function intParser(String key, int minValue, Property[] properties) { final boolean isFiltered = isFiltered(properties); return s -> parseInt(s, minValue, key, isFiltered); diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index f6ad5e22b9ed7..e82f9eff7d5e0 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -701,7 +701,7 @@ public Iterator> settings() { /** * The `index.mapping.ignore_above` setting defines the maximum length for the content of a field that will be indexed * or stored. If the length of the field’s content exceeds this limit, the field value will be ignored during indexing. - * This setting is useful for `keyword`, `flattened`, and `wildcard` fields where very large values are undesirable. + * This setting is useful for `keyword`, `flattened`, and `wildcard` fields where very large values are undesirable. * It allows users to manage the size of indexed data by skipping fields with excessively long content. As an index-level * setting, it applies to all `keyword` and `wildcard` fields, as well as to keyword values within `flattened` fields. * When it comes to arrays, the `ignore_above` setting applies individually to each element of the array. If any element's @@ -713,14 +713,30 @@ public Iterator> settings() { * 
      * "index.mapping.ignore_above": 256
      *
+ *

+ * NOTE: The value for `ignore_above` is the _character count_, but Lucene counts + * bytes. Here we set the limit to `32766 / 4 = 8191` since UTF-8 characters may + * occupy at most 4 bytes. */ + public static final Setting IGNORE_ABOVE_SETTING = Setting.intSetting( "index.mapping.ignore_above", - Integer.MAX_VALUE, + IndexSettings::getIgnoreAboveDefaultValue, 0, + Integer.MAX_VALUE, Property.IndexScope, Property.ServerlessPublic ); + + private static String getIgnoreAboveDefaultValue(final Settings settings) { + if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB + && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB)) { + return "8191"; + } else { + return String.valueOf(Integer.MAX_VALUE); + } + } + public static final NodeFeature IGNORE_ABOVE_INDEX_LEVEL_SETTING = new NodeFeature("mapper.ignore_above_index_level_setting"); private final Index index; diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 9dde1a8d28e54..dedaac5042d71 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -117,7 +117,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion ENABLE_IGNORE_MALFORMED_LOGSDB = def(8_514_00_0, Version.LUCENE_9_11_1); public static final IndexVersion MERGE_ON_RECOVERY_VERSION = def(8_515_00_0, Version.LUCENE_9_11_1); public static final IndexVersion UPGRADE_TO_LUCENE_9_12 = def(8_516_00_0, Version.LUCENE_9_12_0); - + public static final IndexVersion ENABLE_IGNORE_ABOVE_LOGSDB = def(8_517_00_0, Version.LUCENE_9_12_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _