ignore_above default to 8191 for logsdb (#113442)

In LogsDB we would like to use a default value of `8191` for the index-level setting
`index.mapping.ignore_above`. The value for `ignore_above` is the _character count_,
but Lucene counts bytes. Here we set the limit to `32766 / 4 = 8191` since UTF-8
characters may occupy at most 4 bytes.

Author: salvatore-campagna

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.hamcrest.Matchers;
 import org.junit.Before;
 import org.junit.ClassRule;
 
@@ -357,6 +358,66 @@ public void testIgnoreMalformedSetting() throws IOException {
         }
     }
 
+    public void testIgnoreAboveSetting() throws IOException {
+        // with default template
+        {
+            assertOK(createDataStream(client, "logs-test-1"));
+            String logsIndex1 = getDataStreamBackingIndex(client, "logs-test-1", 0);
+            assertThat(getSetting(client, logsIndex1, "index.mapping.ignore_above"), equalTo("8191"));
+            for (String newValue : List.of("512", "2048", "12000", String.valueOf(Integer.MAX_VALUE))) {
+                closeIndex(logsIndex1);
+                updateIndexSettings(logsIndex1, Settings.builder().put("index.mapping.ignore_above", newValue));
+                assertThat(getSetting(client, logsIndex1, "index.mapping.ignore_above"), equalTo(newValue));
+            }
+            for (String newValue : List.of(String.valueOf((long) Integer.MAX_VALUE + 1), String.valueOf(Long.MAX_VALUE))) {
+                closeIndex(logsIndex1);
+                ResponseException ex = assertThrows(
+                    ResponseException.class,
+                    () -> updateIndexSettings(logsIndex1, Settings.builder().put("index.mapping.ignore_above", newValue))
+                );
+                assertThat(
+                    ex.getMessage(),
+                    Matchers.containsString("Failed to parse value [" + newValue + "] for setting [index.mapping.ignore_above]")
+                );
+            }
+        }
+        // with override template
+        {
+            var template = """
+                {
+                  "template": {
+                    "settings": {
+                      "index": {
+                        "mapping": {
+                          "ignore_above": "128"
+                        }
+                      }
+                    }
+                  }
+                }""";
+            assertOK(putComponentTemplate(client, "logs@custom", template));
+            assertOK(createDataStream(client, "logs-custom-dev"));
+            String index = getDataStreamBackingIndex(client, "logs-custom-dev", 0);
+            assertThat(getSetting(client, index, "index.mapping.ignore_above"), equalTo("128"));
+            for (String newValue : List.of("64", "256", "12000", String.valueOf(Integer.MAX_VALUE))) {
+                closeIndex(index);
+                updateIndexSettings(index, Settings.builder().put("index.mapping.ignore_above", newValue));
+                assertThat(getSetting(client, index, "index.mapping.ignore_above"), equalTo(newValue));
+            }
+        }
+        // standard index
+        {
+            String index = "test-index";
+            createIndex(index);
+            assertThat(getSetting(client, index, "index.mapping.ignore_above"), equalTo(Integer.toString(Integer.MAX_VALUE)));
+            for (String newValue : List.of("256", "512", "12000", String.valueOf(Integer.MAX_VALUE))) {
+                closeIndex(index);
+                updateIndexSettings(index, Settings.builder().put("index.mapping.ignore_above", newValue));
+                assertThat(getSetting(client, index, "index.mapping.ignore_above"), equalTo(newValue));
+            }
+        }
+    }
+
     private static Map<String, Object> getMapping(final RestClient client, final String indexName) throws IOException {
         final Request request = new Request("GET", "/" + indexName + "/_mapping");
 

server/src/main/java/org/elasticsearch/common/settings/Setting.java

@@ -1366,6 +1366,16 @@ public static Setting<Integer> intSetting(String key, int defaultValue, int minV
         return new Setting<>(key, Integer.toString(defaultValue), intParser(key, minValue, properties), properties);
     }
 
+    public static Setting<Integer> intSetting(
+        String key,
+        Function<Settings, String> defaultValueFn,
+        int minValue,
+        int maxValue,
+        Property... properties
+    ) {
+        return new Setting<>(key, defaultValueFn, intParser(key, minValue, maxValue, properties), properties);
+    }
+
     private static Function<String, Integer> intParser(String key, int minValue, Property[] properties) {
         final boolean isFiltered = isFiltered(properties);
         return s -> parseInt(s, minValue, key, isFiltered);

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -701,7 +701,7 @@ public Iterator<Setting<?>> settings() {
     /**
      * The `index.mapping.ignore_above` setting defines the maximum length for the content of a field that will be indexed
      * or stored. If the length of the field’s content exceeds this limit, the field value will be ignored during indexing.
-     * This setting is  useful for `keyword`, `flattened`, and `wildcard` fields where very large values are undesirable.
+     * This setting is useful for `keyword`, `flattened`, and `wildcard` fields where very large values are undesirable.
      * It allows users to manage the size of indexed data by skipping fields with excessively long content. As an index-level
      * setting, it applies to all `keyword` and `wildcard` fields, as well as to keyword values within `flattened` fields.
      * When it comes to arrays, the `ignore_above` setting applies individually to each element of the array. If any element's
@@ -713,14 +713,30 @@ public Iterator<Setting<?>> settings() {
      * <pre>
      * "index.mapping.ignore_above": 256
      * </pre>
+     * <p>
+     * NOTE: The value for `ignore_above` is the _character count_, but Lucene counts
+     * bytes. Here we set the limit to `32766 / 4 = 8191` since UTF-8 characters may
+     * occupy at most 4 bytes.
      */
+
     public static final Setting<Integer> IGNORE_ABOVE_SETTING = Setting.intSetting(
         "index.mapping.ignore_above",
-        Integer.MAX_VALUE,
+        IndexSettings::getIgnoreAboveDefaultValue,
         0,
+        Integer.MAX_VALUE,
         Property.IndexScope,
         Property.ServerlessPublic
     );
+
+    private static String getIgnoreAboveDefaultValue(final Settings settings) {
+        if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB
+            && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB)) {
+            return "8191";
+        } else {
+            return String.valueOf(Integer.MAX_VALUE);
+        }
+    }
+
     public static final NodeFeature IGNORE_ABOVE_INDEX_LEVEL_SETTING = new NodeFeature("mapper.ignore_above_index_level_setting");
 
     private final Index index;

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -117,7 +117,7 @@ private static IndexVersion def(int id, Version luceneVersion) {
     public static final IndexVersion ENABLE_IGNORE_MALFORMED_LOGSDB = def(8_514_00_0, Version.LUCENE_9_11_1);
     public static final IndexVersion MERGE_ON_RECOVERY_VERSION = def(8_515_00_0, Version.LUCENE_9_11_1);
     public static final IndexVersion UPGRADE_TO_LUCENE_9_12 = def(8_516_00_0, Version.LUCENE_9_12_0);
-
+    public static final IndexVersion ENABLE_IGNORE_ABOVE_LOGSDB = def(8_517_00_0, Version.LUCENE_9_12_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _