diff --git a/docs/reference/elasticsearch/mapping-reference/mapping-field-meta.md b/docs/reference/elasticsearch/mapping-reference/mapping-field-meta.md index 33a467e6fbc43..cdcafd43a0877 100644 --- a/docs/reference/elasticsearch/mapping-reference/mapping-field-meta.md +++ b/docs/reference/elasticsearch/mapping-reference/mapping-field-meta.md @@ -24,7 +24,8 @@ PUT my-index-000001 ``` ::::{note} -Field metadata enforces at most 5 entries, that keys have a length that is less than or equal to 20, and that values are strings whose length is less than or equal to 50. +Field metadata enforces at most 5 entries, that keys have a length that is less than or equal to 20, and that values are strings whose length is less than or equal to 500. +The value limit is configurable, with the index setting: `index.mapping.meta.length_limit`. :::: diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 93ddb5d3fc485..1873fc355a0a5 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -204,6 +204,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING, IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING, InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT, + IndexSettings.INDEX_MAPPING_META_LENGTH_LIMIT_SETTING, // validate that built-in similarities don't get redefined Setting.groupSetting("index.similarity.", (s) -> { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index a6335ca6666b0..8fc1d8432c72e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -856,6 +856,14 @@ private static String getIgnoreAboveDefaultValue(final Settings settings) { Property.ServerlessPublic ); + public static final Setting INDEX_MAPPING_META_LENGTH_LIMIT_SETTING = Setting.intSetting( + "index.mapping.meta.length_limit", + 500, + 0, + Property.Dynamic, + Property.IndexScope + ); + private final Index index; private final IndexVersion version; private final Logger logger; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index a43575b8f990c..847f4740e21fb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -1298,7 +1298,7 @@ public static Parameter> metaParam() { "meta", true, Map::of, - (n, c, o) -> TypeParsers.parseMeta(n, o), + (n, c, o) -> TypeParsers.parseMeta(n, o, c), m -> m.fieldType().meta(), XContentBuilder::stringStringMap, Objects::toString diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java index 7be9d658297ca..71d4b2eb8bbe9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TypeParsers.java @@ -24,6 +24,7 @@ import static org.elasticsearch.common.xcontent.support.XContentMapValues.isArray; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringValue; +import static org.elasticsearch.index.IndexSettings.INDEX_MAPPING_META_LENGTH_LIMIT_SETTING; public class TypeParsers { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(TypeParsers.class); @@ -31,7 +32,7 @@ public class TypeParsers { /** * Parse the {@code meta} key of the mapping. */ - public static Map parseMeta(String name, Object metaObject) { + public static Map parseMeta(String name, Object metaObject, MappingParserContext parserContext) { if (metaObject instanceof Map == false) { throw new MapperParsingException( "[meta] must be an object, got " + metaObject.getClass().getSimpleName() + "[" + metaObject + "] for field [" + name + "]" @@ -52,11 +53,18 @@ public static Map parseMeta(String name, Object metaObject) { ); } } + int metaValueLengthLimit = INDEX_MAPPING_META_LENGTH_LIMIT_SETTING.get(parserContext.getIndexSettings().getSettings()); for (Object value : meta.values()) { if (value instanceof String sValue) { - if (sValue.codePointCount(0, sValue.length()) > 50) { + if (sValue.codePointCount(0, sValue.length()) > metaValueLengthLimit) { throw new MapperParsingException( - "[meta] values can't be longer than 50 chars, but got [" + value + "] for field [" + name + "]" + "[meta] values can't be longer than " + + metaValueLengthLimit + + " chars, but got [" + + value + + "] for field [" + + name + + "]" ); } } else if (value == null) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java index 9a30e7d696b68..d18d43d327f5e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java @@ -35,6 +35,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.elasticsearch.index.IndexSettings.INDEX_MAPPING_META_LENGTH_LIMIT_SETTING; import static org.elasticsearch.index.analysis.AnalysisRegistry.DEFAULT_ANALYZER_NAME; import static org.elasticsearch.index.analysis.AnalysisRegistry.DEFAULT_SEARCH_ANALYZER_NAME; import static org.elasticsearch.index.analysis.AnalysisRegistry.DEFAULT_SEARCH_QUOTED_ANALYZER_NAME; @@ -52,48 +53,32 @@ private static Map defaultAnalyzers() { return analyzers; } - public void testMultiFieldWithinMultiField() throws IOException { - - XContentBuilder mapping = XContentFactory.jsonBuilder() - .startObject() - .field("type", "keyword") - .startObject("fields") - .startObject("sub-field") - .field("type", "keyword") - .startObject("fields") - .startObject("sub-sub-field") - .field("type", "keyword") - .endObject() - .endObject() - .endObject() - .endObject() - .endObject(); + private Settings buildSettings() { + return Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .build(); + } + private MappingParserContext createParserContext(Settings settings) { Mapper.TypeParser typeParser = KeywordFieldMapper.PARSER; MapperService mapperService = mock(MapperService.class); IndexAnalyzers indexAnalyzers = IndexAnalyzers.of(defaultAnalyzers()); when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); - Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .build(); IndexMetadata metadata = IndexMetadata.builder("test").settings(settings).build(); IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); when(mapperService.getIndexSettings()).thenReturn(indexSettings); - // For indices created in 8.0 or later, we should throw an error. - Map fieldNodeCopy = XContentHelper.convertToMap(BytesReference.bytes(mapping), true, mapping.contentType()).v2(); - IndexVersion version = IndexVersionUtils.randomVersionBetween(random(), IndexVersions.V_8_0_0, IndexVersion.current()); TransportVersion transportVersion = TransportVersionUtils.randomVersionBetween( random(), TransportVersions.V_8_0_0, TransportVersion.current() ); - MappingParserContext context = new MappingParserContext( + return new MappingParserContext( null, type -> typeParser, type -> null, @@ -108,6 +93,29 @@ public void testMultiFieldWithinMultiField() throws IOException { throw new UnsupportedOperationException(); } ); + } + + public void testMultiFieldWithinMultiField() throws IOException { + + XContentBuilder mapping = XContentFactory.jsonBuilder() + .startObject() + .field("type", "keyword") + .startObject("fields") + .startObject("sub-field") + .field("type", "keyword") + .startObject("fields") + .startObject("sub-sub-field") + .field("type", "keyword") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + + // For indices created in 8.0 or later, we should throw an error. + Map fieldNodeCopy = XContentHelper.convertToMap(BytesReference.bytes(mapping), true, mapping.contentType()).v2(); + + MappingParserContext context = createParserContext(buildSettings()); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { TextFieldMapper.PARSER.parse("textField", fieldNodeCopy, context); @@ -122,49 +130,80 @@ public void testMultiFieldWithinMultiField() throws IOException { } public void testParseMeta() { + MappingParserContext parserContext = createParserContext(buildSettings()); + { - MapperParsingException e = expectThrows(MapperParsingException.class, () -> TypeParsers.parseMeta("foo", 3)); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> TypeParsers.parseMeta("foo", 3, parserContext)); assertEquals("[meta] must be an object, got Integer[3] for field [foo]", e.getMessage()); } { MapperParsingException e = expectThrows( MapperParsingException.class, - () -> TypeParsers.parseMeta("foo", Map.of("veryloooooooooooongkey", 3L)) + () -> TypeParsers.parseMeta("foo", Map.of("veryloooooooooooongkey", 3L), parserContext) ); assertEquals("[meta] keys can't be longer than 20 chars, but got [veryloooooooooooongkey] for field [foo]", e.getMessage()); } { Map mapping = Map.of("foo1", 3L, "foo2", 4L, "foo3", 5L, "foo4", 6L, "foo5", 7L, "foo6", 8L); - MapperParsingException e = expectThrows(MapperParsingException.class, () -> TypeParsers.parseMeta("foo", mapping)); + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> TypeParsers.parseMeta("foo", mapping, parserContext) + ); assertEquals("[meta] can't have more than 5 entries, but got 6 on field [foo]", e.getMessage()); } { Map mapping = Map.of("foo", Map.of("bar", "baz")); - MapperParsingException e = expectThrows(MapperParsingException.class, () -> TypeParsers.parseMeta("foo", mapping)); + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> TypeParsers.parseMeta("foo", mapping, parserContext) + ); assertEquals("[meta] values can only be strings, but got Map1[{bar=baz}] for field [foo]", e.getMessage()); } { Map mapping = Map.of("bar", "baz", "foo", 3); - MapperParsingException e = expectThrows(MapperParsingException.class, () -> TypeParsers.parseMeta("foo", mapping)); + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> TypeParsers.parseMeta("foo", mapping, parserContext) + ); assertEquals("[meta] values can only be strings, but got Integer[3] for field [foo]", e.getMessage()); } { Map meta = new HashMap<>(); meta.put("foo", null); - MapperParsingException e = expectThrows(MapperParsingException.class, () -> TypeParsers.parseMeta("foo", meta)); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> TypeParsers.parseMeta("foo", meta, parserContext)); assertEquals("[meta] values can't be null (field [foo])", e.getMessage()); } { - String longString = IntStream.range(0, 51).mapToObj(Integer::toString).collect(Collectors.joining()); + String longString = IntStream.range(0, 501).mapToObj(Integer::toString).collect(Collectors.joining()); Map mapping = Map.of("foo", longString); - MapperParsingException e = expectThrows(MapperParsingException.class, () -> TypeParsers.parseMeta("foo", mapping)); - assertThat(e.getMessage(), Matchers.startsWith("[meta] values can't be longer than 50 chars")); + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> TypeParsers.parseMeta("foo", mapping, parserContext) + ); + assertThat(e.getMessage(), Matchers.startsWith("[meta] values can't be longer than 500 chars")); + } + + { + Settings otherSettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(INDEX_MAPPING_META_LENGTH_LIMIT_SETTING.getKey(), 300) + .build(); + MappingParserContext otherParserContext = createParserContext(otherSettings); + String longString = IntStream.range(0, 301).mapToObj(Integer::toString).collect(Collectors.joining()); + Map mapping = Map.of("foo", longString); + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> TypeParsers.parseMeta("foo", mapping, otherParserContext) + ); + assertThat(e.getMessage(), Matchers.startsWith("[meta] values can't be longer than 300 chars")); } } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java index b0be3e21bbc7c..731d600836b77 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java @@ -496,6 +496,7 @@ static String[] extractLeaderShardHistoryUUIDs(Map ccrIndexMetad IndexSettings.INDEX_FLUSH_AFTER_MERGE_THRESHOLD_SIZE_SETTING, IndexSettings.INDEX_GC_DELETES_SETTING, IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD, + IndexSettings.INDEX_MAPPING_META_LENGTH_LIMIT_SETTING, IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING, BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING, SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING,