Optionally ignore field when indexed field name exceeds length limit

docs/changelog/136143.yaml

@@ -0,0 +1,6 @@
+pr: 136143
+summary: Optionally ignore field when indexed field name exceeds length limit
+area: Mapping
+type: enhancement
+issues:
+ - 135700

docs/reference/elasticsearch/index-settings/mapping-limit.md

@@ -46,6 +46,9 @@ $$$ignore-dynamic-beyond-limit$$$
 `index.mapping.field_name_length.limit`
 :   Setting for the maximum length of a field name. This setting isn’t really something that addresses mappings explosion but might still be useful if you want to limit the field length. It usually shouldn’t be necessary to set this setting. The default is okay unless a user starts to add a huge number of fields with really long names. Default is `Long.MAX_VALUE` (no limit).
 
+`index.mapping.field_name_length.ignore_dynamic_beyond_limit` {applies_to}`stack: ga 9.3`
+:   This setting determines what happens when a the name of a dynamically mapped field would exceed the configured maximum length. When set to `false` (the default), the index request of the document that tries to add a dynamic field to the mapping will fail with the message `Field name [x] is longer than the limit of [y] characters`. When set to `true`, the index request will not fail. Instead, fields that would exceed the limit are not added to the mapping, similar to [`dynamic: false`](/reference/elasticsearch/mapping-reference/dynamic.md). The fields that were not added to the mapping will be added to the [`_ignored` field](/reference/elasticsearch/mapping-reference/mapping-ignored-field.md). The default value is `false`.
+
 `index.mapping.dimension_fields.limit`
 :   (Dynamic, integer) Maximum number of [time series dimensions](docs-content://manage-data/data-store/data-streams/time-series-data-stream-tsds.md#time-series-dimension) for the index. Defaults to `32768`.
 

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/100_field_name_length_limit.yml

@@ -0,0 +1,213 @@
+setup:
+  - requires:
+      cluster_features: ["mapper.ignore_dynamic_field_names_beyond_limit"]
+      reason: "setting must be present to be tested"
+
+---
+
+"Test field name length limit":
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              mapping:
+                field_name_length:
+                  ignore_dynamic_beyond_limit: true
+                  limit: 45
+          mappings:
+            dynamic: true
+
+  - do:
+      index:
+        index: test_index
+        id: 1
+        refresh: true
+        body:
+          foo:
+            test_really_long_field_name_that_will_be_ignored: foo
+            field_name_not_ignored: 10
+
+  - do:
+      indices.get_mapping:
+        index: test_index
+
+  - match: { test_index.mappings.properties.foo.properties.test_really_long_field_name_that_will_be_ignored: null }
+  - match: { test_index.mappings.properties.foo.properties.field_name_not_ignored.type: long }
+
+  - do:
+      get:
+        index: test_index
+        id: 1
+        _source: true
+        stored_fields: [_ignored]
+  - match:
+      _source:
+        foo:
+          test_really_long_field_name_that_will_be_ignored: foo
+          field_name_not_ignored: 10
+  - match: { _ignored: [ "foo.test_really_long_field_name_that_will_be_ignored" ] }
+
+---
+
+"Test field name length limit array":
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              mapping:
+                field_name_length:
+                  ignore_dynamic_beyond_limit: true
+                  limit: 45
+          mappings:
+            dynamic: true
+
+  - do:
+      index:
+        index: test_index
+        id: 1
+        refresh: true
+        body:
+          foo:
+            test_really_long_field_name_that_will_be_ignored: [ foo, bar, baz ]
+            field_name_not_ignored: [ 10, 20, 30 ]
+
+  - do:
+      indices.get_mapping:
+        index: test_index
+
+  - match: { test_index.mappings.properties.foo.properties.test_really_long_field_name_that_will_be_ignored: null }
+  - match: { test_index.mappings.properties.foo.properties.field_name_not_ignored.type: long }
+
+  - do:
+      get:
+        index: test_index
+        id: 1
+        _source: true
+        stored_fields: [ _ignored ]
+
+  - match:
+      _source:
+        foo:
+          test_really_long_field_name_that_will_be_ignored: [ foo, bar, baz ]
+          field_name_not_ignored: [ 10, 20, 30 ]
+  - match: {_ignored:["foo.test_really_long_field_name_that_will_be_ignored"]}
+
+---
+
+"Test field name length limit synthetic source":
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              mapping:
+                source.mode: synthetic
+                field_name_length:
+                  ignore_dynamic_beyond_limit: true
+                  limit: 45
+          mappings:
+            dynamic: true
+
+  - do:
+      index:
+        index: test_index
+        id: 1
+        refresh: true
+        body:
+          foo:
+            test_really_long_field_name_that_will_be_ignored: foo
+            field_name_not_ignored: 10
+
+  - do:
+      indices.get_mapping:
+        index: test_index
+
+  - match: { test_index.mappings.properties.foo.properties.test_really_long_field_name_that_will_be_ignored: null }
+  - match: { test_index.mappings.properties.foo.properties.field_name_not_ignored.type: long }
+
+  - do:
+      get:
+        index: test_index
+        id: 1
+        _source: true
+        stored_fields: [ _ignored ]
+  - match:
+      _source:
+        foo:
+          test_really_long_field_name_that_will_be_ignored: foo
+          field_name_not_ignored: 10
+  - match: {_ignored:["foo.test_really_long_field_name_that_will_be_ignored"]}
+
+---
+
+"Test field name length limit array synthetic source":
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              mapping:
+                source.mode: synthetic
+                field_name_length:
+                  ignore_dynamic_beyond_limit: true
+                  limit: 45
+          mappings:
+            dynamic: true
+
+  - do:
+      index:
+        index: test_index
+        id: 1
+        refresh: true
+        body:
+          foo:
+            test_really_long_field_name_that_will_be_ignored: [ foo, bar, baz ]
+            field_name_not_ignored: [ 10, 20, 30 ]
+
+  - do:
+      indices.get_mapping:
+        index: test_index
+
+  - match: { test_index.mappings.properties.foo.properties.test_really_long_field_name_that_will_be_ignored: null }
+  - match: { test_index.mappings.properties.foo.properties.field_name_not_ignored.type: long }
+
+  - do:
+      get:
+        index: test_index
+        id: 1
+        _source: true
+        stored_fields: [ _ignored ]
+  - match:
+      _source:
+        foo:
+          test_really_long_field_name_that_will_be_ignored: [ foo, bar, baz ]
+          field_name_not_ignored: [ 10, 20, 30 ]
+  - match: {_ignored:["foo.test_really_long_field_name_that_will_be_ignored"]}
+
+---
+
+"Test field name length limit with static mapping":
+  - do:
+      catch: /illegal_argument_exception/
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            index:
+              mapping:
+                field_name_length:
+                  ignore_dynamic_beyond_limit: true
+                  limit: 45
+          mappings:
+            dynamic: false
+            properties:
+              test_really_long_field_name_that_will_be_ignored:
+                type: text
+
+  - match: {error.reason: "Field name [test_really_long_field_name_that_will_be_ignored] is longer than the limit of [45] characters"}

server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -168,6 +168,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING,
                 MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING,
                 MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING,
+                MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_FIELD_NAME_LENGTH_SETTING,
                 MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING,
                 MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING,
                 MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING,

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -52,6 +52,7 @@
 import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING;
 import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING;
 import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING;
+import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_FIELD_NAME_LENGTH_SETTING;
 import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING;
 import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING;
 import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING;
@@ -926,6 +927,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
     private volatile long mappingNestedDocsLimit;
     private volatile long mappingTotalFieldsLimit;
     private volatile boolean ignoreDynamicFieldsBeyondLimit;
+    private volatile boolean ignoreDynamicFieldNamesBeyondLimit;
     private volatile long mappingDepthLimit;
     private volatile long mappingFieldNameLengthLimit;
     private volatile long mappingDimensionFieldsLimit;
@@ -1101,6 +1103,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         mappingNestedDocsLimit = scopedSettings.get(INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING);
         mappingTotalFieldsLimit = scopedSettings.get(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING);
         ignoreDynamicFieldsBeyondLimit = scopedSettings.get(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING);
+        ignoreDynamicFieldNamesBeyondLimit = scopedSettings.get(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_FIELD_NAME_LENGTH_SETTING);
         mappingDepthLimit = scopedSettings.get(INDEX_MAPPING_DEPTH_LIMIT_SETTING);
         mappingFieldNameLengthLimit = scopedSettings.get(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING);
         mappingDimensionFieldsLimit = scopedSettings.get(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING);
@@ -1218,6 +1221,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
             INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING,
             this::setIgnoreDynamicFieldsBeyondLimit
         );
+        scopedSettings.addSettingsUpdateConsumer(
+            INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_FIELD_NAME_LENGTH_SETTING,
+            this::setIgnoreDynamicFieldNamesBeyondLimit
+        );
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, this::setMappingTotalFieldsLimit);
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit);
         scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit);
@@ -1765,10 +1772,18 @@ private void setIgnoreDynamicFieldsBeyondLimit(boolean ignoreDynamicFieldsBeyond
         this.ignoreDynamicFieldsBeyondLimit = ignoreDynamicFieldsBeyondLimit;
     }
 
+    private void setIgnoreDynamicFieldNamesBeyondLimit(boolean ignoreDynamicFieldNamesBeyondLimit) {
+        this.ignoreDynamicFieldNamesBeyondLimit = ignoreDynamicFieldNamesBeyondLimit;
+    }
+
     public boolean isIgnoreDynamicFieldsBeyondLimit() {
         return ignoreDynamicFieldsBeyondLimit;
     }
 
+    public boolean isIgnoreDynamicFieldNamesBeyondLimit() {
+        return ignoreDynamicFieldNamesBeyondLimit;
+    }
+
     public long getMappingDepthLimit() {
         return mappingDepthLimit;
     }

server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

@@ -679,6 +679,26 @@ private static void parseArrayDynamic(DocumentParserContext context, String curr
                 context.addIgnoredField(context.path().pathAsText(currentFieldName));
                 return;
             }
+
+            if (context.indexSettings().isIgnoreDynamicFieldNamesBeyondLimit()
+                && currentFieldName.length() > context.indexSettings().getMappingFieldNameLengthLimit()) {
+                if (context.canAddIgnoredField()) {
+                    try {
+                        context.addIgnoredField(
+                            IgnoredSourceFieldMapper.NameValue.fromContext(
+                                context,
+                                context.path().pathAsText(currentFieldName),
+                                context.encodeFlattenedToken()
+                            )
+                        );
+                    } catch (IOException e) {
+                        throw new IllegalArgumentException("failed to parse field [" + currentFieldName + "]", e);
+                    }
+                }
+                context.addIgnoredField(context.path().pathAsText(currentFieldName));
+                return;
+            }
+
             parseNonDynamicArray(context, objectMapperFromTemplate, currentFieldName, currentFieldName);
         } else {
             if (parsesArrayValue(objectMapperFromTemplate)) {

server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java

@@ -559,6 +559,24 @@ public final boolean addDynamicMapper(Mapper mapper) {
                 mappingLookup.checkFieldLimit(indexSettings().getMappingTotalFieldsLimit(), additionalFieldsToAdd);
             }
             dynamicMappersSize.add(mapperSize);
+
+            if (indexSettings().isIgnoreDynamicFieldNamesBeyondLimit()) {
+                if (mapper.leafName().length() > indexSettings().getMappingFieldNameLengthLimit()) {
+                    if (canAddIgnoredField()) {
+                        try {
+                            addIgnoredField(
+                                IgnoredSourceFieldMapper.NameValue.fromContext(this, mapper.fullPath(), encodeFlattenedToken())
+                            );
+                        } catch (IOException e) {
+                            throw new IllegalArgumentException("failed to parse field [" + mapper.fullPath() + "]", e);
+                        }
+                    }
+                    addIgnoredField(mapper.fullPath());
+                    return false;
+                } else {
+                    mappingLookup.checkFieldNameLengthLimit(indexSettings().getMappingFieldNameLengthLimit());
+                }
+            }
         }
         if (mapper instanceof ObjectMapper objectMapper) {
             dynamicObjectMappers.put(objectMapper.fullPath(), objectMapper);

server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java

@@ -54,6 +54,9 @@ public class MapperFeatures implements FeatureSpecification {
     static final NodeFeature PATTERN_TEXT_RENAME = new NodeFeature("mapper.pattern_text_rename");
     static final NodeFeature DISKBBQ_ON_DISK_RESCORING = new NodeFeature("mapper.vectors.diskbbq_on_disk_rescoring");
     static final NodeFeature PROVIDE_INDEX_SORT_SETTING_DEFAULTS = new NodeFeature("mapper.provide_index_sort_setting_defaults");
+    static final NodeFeature INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_FIELD_NAME_LIMIT = new NodeFeature(
+        "mapper.ignore_dynamic_field_names_beyond_limit"
+    );
 
     @Override
     public Set<NodeFeature> getTestFeatures() {
@@ -93,7 +96,8 @@ public Set<NodeFeature> getTestFeatures() {
             MATCH_ONLY_TEXT_BLOCK_LOADER_FIX,
             PATTERN_TEXT_RENAME,
             DISKBBQ_ON_DISK_RESCORING,
-            PROVIDE_INDEX_SORT_SETTING_DEFAULTS
+            PROVIDE_INDEX_SORT_SETTING_DEFAULTS,
+            INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_FIELD_NAME_LIMIT
         );
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -157,6 +157,12 @@ public boolean isAutoUpdate() {
         Property.Dynamic,
         Property.IndexScope
     );
+    public static final Setting<Boolean> INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_FIELD_NAME_LENGTH_SETTING = Setting.boolSetting(
+        "index.mapping.field_name_length.ignore_dynamic_beyond_limit",
+        false,
+        Property.Dynamic,
+        Property.IndexScope
+    );
     public static final Setting<Long> INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING = Setting.longSetting(
         "index.mapping.dimension_fields.limit",
         32_768,

server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java

@@ -347,7 +347,7 @@ static void checkObjectDepthLimit(long limit, String objectPath) {
         }
     }
 
-    private void checkFieldNameLengthLimit(long limit) {
+    void checkFieldNameLengthLimit(long limit) {
         validateMapperNameIn(objectMappers.values(), limit);
         validateMapperNameIn(fieldMappers.values(), limit);
     }

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java

@@ -321,6 +321,7 @@ public void testDynamicIndexSettingsAreClassified() {
         // be indexed in the follower index:
         replicatedSettings.add(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING);
         replicatedSettings.add(MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING);
+        replicatedSettings.add(MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_FIELD_NAME_LENGTH_SETTING);
         replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING);
         replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING);
         replicatedSettings.add(MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING);