diff --git a/docs/changelog/136143.yaml b/docs/changelog/136143.yaml new file mode 100644 index 0000000000000..90e09e34658b7 --- /dev/null +++ b/docs/changelog/136143.yaml @@ -0,0 +1,6 @@ +pr: 136143 +summary: Optionally ignore field when indexed field name exceeds length limit +area: Mapping +type: enhancement +issues: + - 135700 diff --git a/docs/reference/elasticsearch/index-settings/mapping-limit.md b/docs/reference/elasticsearch/index-settings/mapping-limit.md index 6159293a3ec9e..82f87515b60af 100644 --- a/docs/reference/elasticsearch/index-settings/mapping-limit.md +++ b/docs/reference/elasticsearch/index-settings/mapping-limit.md @@ -46,6 +46,9 @@ $$$ignore-dynamic-beyond-limit$$$ `index.mapping.field_name_length.limit` : Setting for the maximum length of a field name. This setting isn’t really something that addresses mappings explosion but might still be useful if you want to limit the field length. It usually shouldn’t be necessary to set this setting. The default is okay unless a user starts to add a huge number of fields with really long names. Default is `Long.MAX_VALUE` (no limit). +`index.mapping.field_name_length.ignore_dynamic_beyond_limit` {applies_to}`stack: ga 9.3` +: This setting determines what happens when a the name of a dynamically mapped field would exceed the configured maximum length. When set to `false` (the default), the index request of the document that tries to add a dynamic field to the mapping will fail with the message `Field name [x] is longer than the limit of [y] characters`. When set to `true`, the index request will not fail. Instead, fields that would exceed the limit are not added to the mapping, similar to [`dynamic: false`](/reference/elasticsearch/mapping-reference/dynamic.md). The fields that were not added to the mapping will be added to the [`_ignored` field](/reference/elasticsearch/mapping-reference/mapping-ignored-field.md). The default value is `false`. + `index.mapping.dimension_fields.limit` : (Dynamic, integer) Maximum number of [time series dimensions](docs-content://manage-data/data-store/data-streams/time-series-data-stream-tsds.md#time-series-dimension) for the index. Defaults to `32768`. diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/100_field_name_length_limit.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/100_field_name_length_limit.yml new file mode 100644 index 0000000000000..e6941f1325887 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/100_field_name_length_limit.yml @@ -0,0 +1,213 @@ +setup: + - requires: + cluster_features: ["mapper.ignore_dynamic_field_names_beyond_limit"] + reason: "setting must be present to be tested" + +--- + +"Test field name length limit": + - do: + indices.create: + index: test_index + body: + settings: + index: + mapping: + field_name_length: + ignore_dynamic_beyond_limit: true + limit: 45 + mappings: + dynamic: true + + - do: + index: + index: test_index + id: 1 + refresh: true + body: + foo: + test_really_long_field_name_that_will_be_ignored: foo + field_name_not_ignored: 10 + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.foo.properties.test_really_long_field_name_that_will_be_ignored: null } + - match: { test_index.mappings.properties.foo.properties.field_name_not_ignored.type: long } + + - do: + get: + index: test_index + id: 1 + _source: true + stored_fields: [_ignored] + - match: + _source: + foo: + test_really_long_field_name_that_will_be_ignored: foo + field_name_not_ignored: 10 + - match: { _ignored: [ "foo.test_really_long_field_name_that_will_be_ignored" ] } + +--- + +"Test field name length limit array": + - do: + indices.create: + index: test_index + body: + settings: + index: + mapping: + field_name_length: + ignore_dynamic_beyond_limit: true + limit: 45 + mappings: + dynamic: true + + - do: + index: + index: test_index + id: 1 + refresh: true + body: + foo: + test_really_long_field_name_that_will_be_ignored: [ foo, bar, baz ] + field_name_not_ignored: [ 10, 20, 30 ] + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.foo.properties.test_really_long_field_name_that_will_be_ignored: null } + - match: { test_index.mappings.properties.foo.properties.field_name_not_ignored.type: long } + + - do: + get: + index: test_index + id: 1 + _source: true + stored_fields: [ _ignored ] + + - match: + _source: + foo: + test_really_long_field_name_that_will_be_ignored: [ foo, bar, baz ] + field_name_not_ignored: [ 10, 20, 30 ] + - match: {_ignored:["foo.test_really_long_field_name_that_will_be_ignored"]} + +--- + +"Test field name length limit synthetic source": + - do: + indices.create: + index: test_index + body: + settings: + index: + mapping: + source.mode: synthetic + field_name_length: + ignore_dynamic_beyond_limit: true + limit: 45 + mappings: + dynamic: true + + - do: + index: + index: test_index + id: 1 + refresh: true + body: + foo: + test_really_long_field_name_that_will_be_ignored: foo + field_name_not_ignored: 10 + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.foo.properties.test_really_long_field_name_that_will_be_ignored: null } + - match: { test_index.mappings.properties.foo.properties.field_name_not_ignored.type: long } + + - do: + get: + index: test_index + id: 1 + _source: true + stored_fields: [ _ignored ] + - match: + _source: + foo: + test_really_long_field_name_that_will_be_ignored: foo + field_name_not_ignored: 10 + - match: {_ignored:["foo.test_really_long_field_name_that_will_be_ignored"]} + +--- + +"Test field name length limit array synthetic source": + - do: + indices.create: + index: test_index + body: + settings: + index: + mapping: + source.mode: synthetic + field_name_length: + ignore_dynamic_beyond_limit: true + limit: 45 + mappings: + dynamic: true + + - do: + index: + index: test_index + id: 1 + refresh: true + body: + foo: + test_really_long_field_name_that_will_be_ignored: [ foo, bar, baz ] + field_name_not_ignored: [ 10, 20, 30 ] + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.foo.properties.test_really_long_field_name_that_will_be_ignored: null } + - match: { test_index.mappings.properties.foo.properties.field_name_not_ignored.type: long } + + - do: + get: + index: test_index + id: 1 + _source: true + stored_fields: [ _ignored ] + - match: + _source: + foo: + test_really_long_field_name_that_will_be_ignored: [ foo, bar, baz ] + field_name_not_ignored: [ 10, 20, 30 ] + - match: {_ignored:["foo.test_really_long_field_name_that_will_be_ignored"]} + +--- + +"Test field name length limit with static mapping": + - do: + catch: /illegal_argument_exception/ + indices.create: + index: test_index + body: + settings: + index: + mapping: + field_name_length: + ignore_dynamic_beyond_limit: true + limit: 45 + mappings: + dynamic: false + properties: + test_really_long_field_name_that_will_be_ignored: + type: text + + - match: {error.reason: "Field name [test_really_long_field_name_that_will_be_ignored] is longer than the limit of [45] characters"} diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index aa3dfd4b37b24..dc54e8149e9e0 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -168,6 +168,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING, MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING, + MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_FIELD_NAME_LENGTH_SETTING, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING, MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING, diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index b396e1ca206e3..50f3925c425e7 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -52,6 +52,7 @@ import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING; +import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_FIELD_NAME_LENGTH_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING; @@ -926,6 +927,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private volatile long mappingNestedDocsLimit; private volatile long mappingTotalFieldsLimit; private volatile boolean ignoreDynamicFieldsBeyondLimit; + private volatile boolean ignoreDynamicFieldNamesBeyondLimit; private volatile long mappingDepthLimit; private volatile long mappingFieldNameLengthLimit; private volatile long mappingDimensionFieldsLimit; @@ -1101,6 +1103,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti mappingNestedDocsLimit = scopedSettings.get(INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING); mappingTotalFieldsLimit = scopedSettings.get(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING); ignoreDynamicFieldsBeyondLimit = scopedSettings.get(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING); + ignoreDynamicFieldNamesBeyondLimit = scopedSettings.get(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_FIELD_NAME_LENGTH_SETTING); mappingDepthLimit = scopedSettings.get(INDEX_MAPPING_DEPTH_LIMIT_SETTING); mappingFieldNameLengthLimit = scopedSettings.get(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING); mappingDimensionFieldsLimit = scopedSettings.get(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING); @@ -1218,6 +1221,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING, this::setIgnoreDynamicFieldsBeyondLimit ); + scopedSettings.addSettingsUpdateConsumer( + INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_FIELD_NAME_LENGTH_SETTING, + this::setIgnoreDynamicFieldNamesBeyondLimit + ); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, this::setMappingTotalFieldsLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit); @@ -1765,10 +1772,18 @@ private void setIgnoreDynamicFieldsBeyondLimit(boolean ignoreDynamicFieldsBeyond this.ignoreDynamicFieldsBeyondLimit = ignoreDynamicFieldsBeyondLimit; } + private void setIgnoreDynamicFieldNamesBeyondLimit(boolean ignoreDynamicFieldNamesBeyondLimit) { + this.ignoreDynamicFieldNamesBeyondLimit = ignoreDynamicFieldNamesBeyondLimit; + } + public boolean isIgnoreDynamicFieldsBeyondLimit() { return ignoreDynamicFieldsBeyondLimit; } + public boolean isIgnoreDynamicFieldNamesBeyondLimit() { + return ignoreDynamicFieldNamesBeyondLimit; + } + public long getMappingDepthLimit() { return mappingDepthLimit; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 51e0ba97a039b..0a95183652e24 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -679,6 +679,26 @@ private static void parseArrayDynamic(DocumentParserContext context, String curr context.addIgnoredField(context.path().pathAsText(currentFieldName)); return; } + + if (context.indexSettings().isIgnoreDynamicFieldNamesBeyondLimit() + && currentFieldName.length() > context.indexSettings().getMappingFieldNameLengthLimit()) { + if (context.canAddIgnoredField()) { + try { + context.addIgnoredField( + IgnoredSourceFieldMapper.NameValue.fromContext( + context, + context.path().pathAsText(currentFieldName), + context.encodeFlattenedToken() + ) + ); + } catch (IOException e) { + throw new IllegalArgumentException("failed to parse field [" + currentFieldName + "]", e); + } + } + context.addIgnoredField(context.path().pathAsText(currentFieldName)); + return; + } + parseNonDynamicArray(context, objectMapperFromTemplate, currentFieldName, currentFieldName); } else { if (parsesArrayValue(objectMapperFromTemplate)) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index b0e002d6c0aff..5f0d5a219bb32 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -559,6 +559,24 @@ public final boolean addDynamicMapper(Mapper mapper) { mappingLookup.checkFieldLimit(indexSettings().getMappingTotalFieldsLimit(), additionalFieldsToAdd); } dynamicMappersSize.add(mapperSize); + + if (indexSettings().isIgnoreDynamicFieldNamesBeyondLimit()) { + if (mapper.leafName().length() > indexSettings().getMappingFieldNameLengthLimit()) { + if (canAddIgnoredField()) { + try { + addIgnoredField( + IgnoredSourceFieldMapper.NameValue.fromContext(this, mapper.fullPath(), encodeFlattenedToken()) + ); + } catch (IOException e) { + throw new IllegalArgumentException("failed to parse field [" + mapper.fullPath() + "]", e); + } + } + addIgnoredField(mapper.fullPath()); + return false; + } else { + mappingLookup.checkFieldNameLengthLimit(indexSettings().getMappingFieldNameLengthLimit()); + } + } } if (mapper instanceof ObjectMapper objectMapper) { dynamicObjectMappers.put(objectMapper.fullPath(), objectMapper); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index e256453ac6d31..8dda9017ce2bc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -54,6 +54,9 @@ public class MapperFeatures implements FeatureSpecification { static final NodeFeature PATTERN_TEXT_RENAME = new NodeFeature("mapper.pattern_text_rename"); static final NodeFeature DISKBBQ_ON_DISK_RESCORING = new NodeFeature("mapper.vectors.diskbbq_on_disk_rescoring"); static final NodeFeature PROVIDE_INDEX_SORT_SETTING_DEFAULTS = new NodeFeature("mapper.provide_index_sort_setting_defaults"); + static final NodeFeature INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_FIELD_NAME_LIMIT = new NodeFeature( + "mapper.ignore_dynamic_field_names_beyond_limit" + ); @Override public Set getTestFeatures() { @@ -93,7 +96,8 @@ public Set getTestFeatures() { MATCH_ONLY_TEXT_BLOCK_LOADER_FIX, PATTERN_TEXT_RENAME, DISKBBQ_ON_DISK_RESCORING, - PROVIDE_INDEX_SORT_SETTING_DEFAULTS + PROVIDE_INDEX_SORT_SETTING_DEFAULTS, + INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_FIELD_NAME_LIMIT ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index af817f5827b17..d53d9c166aaea 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -157,6 +157,12 @@ public boolean isAutoUpdate() { Property.Dynamic, Property.IndexScope ); + public static final Setting INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_FIELD_NAME_LENGTH_SETTING = Setting.boolSetting( + "index.mapping.field_name_length.ignore_dynamic_beyond_limit", + false, + Property.Dynamic, + Property.IndexScope + ); public static final Setting INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING = Setting.longSetting( "index.mapping.dimension_fields.limit", 32_768, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index 93a157f435111..e36c9de0530b4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -347,7 +347,7 @@ static void checkObjectDepthLimit(long limit, String objectPath) { } } - private void checkFieldNameLengthLimit(long limit) { + void checkFieldNameLengthLimit(long limit) { validateMapperNameIn(objectMappers.values(), limit); validateMapperNameIn(fieldMappers.values(), limit); } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index beb43b1ee6c9b..a54e63d2a0a94 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -321,6 +321,7 @@ public void testDynamicIndexSettingsAreClassified() { // be indexed in the follower index: replicatedSettings.add(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING); replicatedSettings.add(MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING); + replicatedSettings.add(MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_FIELD_NAME_LENGTH_SETTING); replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING); replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING); replicatedSettings.add(MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING);