From aa9afcead6d1bb20e4433f05e815b15f9e010160 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Tue, 17 Sep 2024 13:47:54 -0700 Subject: [PATCH] Add option to skip using _ignored_source field for synthetic source (#112963) --- .../common/settings/IndexScopedSettings.java | 3 + .../elasticsearch/index/IndexSettings.java | 26 ++++ .../index/mapper/DocumentParserContext.java | 4 +- .../mapper/IgnoredSourceFieldMapper.java | 93 ++++++++++- .../index/mapper/NestedObjectMapper.java | 33 +++- .../index/mapper/SourceLoader.java | 1 - .../FieldAliasMapperValidationTests.java | 5 +- ...edSourceFieldMapperConfigurationTests.java | 144 ++++++++++++++++++ .../index/mapper/NestedLookupTests.java | 5 +- .../index/mapper/NestedObjectMapperTests.java | 10 +- .../bucket/nested/NestedAggregatorTests.java | 5 +- .../search/sort/AbstractSortTestCase.java | 2 +- .../TransportResumeFollowActionTests.java | 3 + .../mapper/SemanticTextFieldMapper.java | 55 +++++-- 14 files changed, 348 insertions(+), 41 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index e4588e648318c..778136cbf5d31 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.FsDirectoryFactory; @@ -183,6 +184,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexSettings.PREFER_ILM_SETTING, DataStreamFailureStoreDefinition.FAILURE_STORE_DEFINITION_VERSION_SETTING, FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING, + IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING, + IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, // validate that built-in similarities don't get redefined Setting.groupSetting("index.similarity.", (s) -> { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 140133378ec06..41523c6dc2c7e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.ingest.IngestService; @@ -778,6 +779,8 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private volatile long mappingDepthLimit; private volatile long mappingFieldNameLengthLimit; private volatile long mappingDimensionFieldsLimit; + private volatile boolean skipIgnoredSourceWrite; + private volatile boolean skipIgnoredSourceRead; /** * The maximum number of refresh listeners allows on this shard. @@ -936,6 +939,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti indexRouting = IndexRouting.fromIndexMetadata(indexMetadata); sourceKeepMode = scopedSettings.get(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING); es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING); + skipIgnoredSourceWrite = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); + skipIgnoredSourceRead = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); scopedSettings.addSettingsUpdateConsumer( MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, @@ -1018,6 +1023,11 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING, this::setMappingDimensionFieldsLimit); + scopedSettings.addSettingsUpdateConsumer( + IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING, + this::setSkipIgnoredSourceWrite + ); + scopedSettings.addSettingsUpdateConsumer(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, this::setSkipIgnoredSourceRead); } private void setSearchIdleAfter(TimeValue searchIdleAfter) { @@ -1594,6 +1604,22 @@ private void setMappingDimensionFieldsLimit(long value) { this.mappingDimensionFieldsLimit = value; } + public boolean getSkipIgnoredSourceWrite() { + return skipIgnoredSourceWrite; + } + + private void setSkipIgnoredSourceWrite(boolean value) { + this.skipIgnoredSourceWrite = value; + } + + public boolean getSkipIgnoredSourceRead() { + return skipIgnoredSourceRead; + } + + private void setSkipIgnoredSourceRead(boolean value) { + this.skipIgnoredSourceRead = value; + } + /** * The bounds for {@code @timestamp} on this index or * {@code null} if there are no bounds. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 26db211345eae..38fa8d02db84e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -333,7 +333,7 @@ final boolean getClonedSource() { } public final boolean canAddIgnoredField() { - return mappingLookup.isSourceSynthetic() && clonedSource == false; + return mappingLookup.isSourceSynthetic() && clonedSource == false && indexSettings().getSkipIgnoredSourceWrite() == false; } Mapper.SourceKeepMode sourceKeepModeFromIndexSettings() { @@ -367,7 +367,7 @@ public boolean isFieldAppliedFromTemplate(String name) { public void markFieldAsCopyTo(String fieldName) { copyToFields.add(fieldName); - if (mappingLookup.isSourceSynthetic()) { + if (mappingLookup.isSourceSynthetic() && indexSettings().getSkipIgnoredSourceWrite() == false) { /* Mark this field as containing copied data meaning it should not be present in synthetic _source (to be consistent with stored _source). diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 35bbec6355762..94d862d5cc516 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -10,12 +10,15 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.LeafReader; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Tuple; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -27,6 +30,8 @@ import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; /** @@ -40,6 +45,7 @@ * if we can replace it for all use cases to avoid duplication, assuming that the storage tradeoff is favorable. */ public class IgnoredSourceFieldMapper extends MetadataFieldMapper { + private final IndexSettings indexSettings; // This factor is used to combine two offsets within the same integer: // - the offset of the end of the parent field within the field name (N / PARENT_OFFSET_IN_NAME_OFFSET) @@ -49,12 +55,32 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper { public static final String NAME = "_ignored_source"; - public static final IgnoredSourceFieldMapper INSTANCE = new IgnoredSourceFieldMapper(); - - public static final TypeParser PARSER = new FixedTypeParser(context -> INSTANCE); + public static final TypeParser PARSER = new FixedTypeParser(context -> new IgnoredSourceFieldMapper(context.getIndexSettings())); static final NodeFeature TRACK_IGNORED_SOURCE = new NodeFeature("mapper.track_ignored_source"); + /* + Setting to disable encoding and writing values for this field. + This is needed to unblock index functionality in case there is a bug on this code path. + */ + public static final Setting SKIP_IGNORED_SOURCE_WRITE_SETTING = Setting.boolSetting( + "index.mapping.synthetic_source.skip_ignored_source_write", + false, + Setting.Property.Dynamic, + Setting.Property.IndexScope + ); + + /* + Setting to disable reading and decoding values stored in this field. + This is needed to unblock search functionality in case there is a bug on this code path. + */ + public static final Setting SKIP_IGNORED_SOURCE_READ_SETTING = Setting.boolSetting( + "index.mapping.synthetic_source.skip_ignored_source_read", + false, + Setting.Property.Dynamic, + Setting.Property.IndexScope + ); + /* * Container for the ignored field data: * - the full name @@ -108,8 +134,9 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) } } - private IgnoredSourceFieldMapper() { + private IgnoredSourceFieldMapper(IndexSettings indexSettings) { super(IgnoredValuesFieldMapperType.INSTANCE); + this.indexSettings = indexSettings; } @Override @@ -151,6 +178,64 @@ static NameValue decode(Object field) { return new NameValue(name, parentOffset, value, null); } + // In rare cases decoding values stored in this field can fail leading to entire source + // not being available. + // We would like to have an option to lose some values in synthetic source + // but have search not fail. + public static Set ensureLoaded(Set fieldsToLoadForSyntheticSource, IndexSettings indexSettings) { + if (indexSettings.getSkipIgnoredSourceRead() == false) { + fieldsToLoadForSyntheticSource.add(NAME); + } + + return fieldsToLoadForSyntheticSource; + } + + @Override + protected SyntheticSourceSupport syntheticSourceSupport() { + // This loader controls if this field is loaded in scope of synthetic source constructions. + // In rare cases decoding values stored in this field can fail leading to entire source + // not being available. + // We would like to have an option to lose some values in synthetic source + // but have search not fail. + return new SyntheticSourceSupport.Native(new SourceLoader.SyntheticFieldLoader() { + @Override + public Stream> storedFieldLoaders() { + if (indexSettings.getSkipIgnoredSourceRead()) { + return Stream.empty(); + } + + // Values are handled in `SourceLoader`. + return Stream.of(Map.entry(NAME, (v) -> {})); + } + + @Override + public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException { + return null; + } + + @Override + public boolean hasValue() { + return false; + } + + @Override + public void write(XContentBuilder b) throws IOException { + + } + + @Override + public String fieldName() { + // Does not really matter. + return NAME; + } + + @Override + public void reset() { + + } + }); + } + public record MappedNameValue(NameValue nameValue, XContentType type, Map map) {} /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java index adf1b329d9e83..fc5f28dd51c9d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; @@ -48,11 +49,18 @@ public static class Builder extends ObjectMapper.Builder { private Explicit includeInParent = Explicit.IMPLICIT_FALSE; private final IndexVersion indexCreatedVersion; private final Function bitSetProducer; + private final IndexSettings indexSettings; - public Builder(String name, IndexVersion indexCreatedVersion, Function bitSetProducer) { + public Builder( + String name, + IndexVersion indexCreatedVersion, + Function bitSetProducer, + IndexSettings indexSettings + ) { super(name, Optional.empty()); this.indexCreatedVersion = indexCreatedVersion; this.bitSetProducer = bitSetProducer; + this.indexSettings = indexSettings; } Builder includeInRoot(boolean includeInRoot) { @@ -113,7 +121,8 @@ public NestedObjectMapper build(MapperBuilderContext context) { parentTypeFilter, nestedTypePath, nestedTypeFilter, - bitSetProducer + bitSetProducer, + indexSettings ); } } @@ -128,7 +137,8 @@ public Mapper.Builder parse(String name, Map node, MappingParser NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder( name, parserContext.indexVersionCreated(), - parserContext::bitSetProducer + parserContext::bitSetProducer, + parserContext.getIndexSettings() ); parseNested(name, node, builder); parseObjectFields(node, parserContext, builder); @@ -195,6 +205,7 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) { private final Query nestedTypeFilter; // Function to create a bitset for identifying parent documents private final Function bitsetProducer; + private final IndexSettings indexSettings; NestedObjectMapper( String name, @@ -208,7 +219,8 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) { Query parentTypeFilter, String nestedTypePath, Query nestedTypeFilter, - Function bitsetProducer + Function bitsetProducer, + IndexSettings indexSettings ) { super(name, fullPath, enabled, Optional.empty(), storeArraySource, dynamic, mappers); this.parentTypeFilter = parentTypeFilter; @@ -217,6 +229,7 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) { this.includeInParent = includeInParent; this.includeInRoot = includeInRoot; this.bitsetProducer = bitsetProducer; + this.indexSettings = indexSettings; } public Query parentTypeFilter() { @@ -254,7 +267,7 @@ public Map getChildren() { @Override public ObjectMapper.Builder newBuilder(IndexVersion indexVersionCreated) { - NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(leafName(), indexVersionCreated, bitsetProducer); + NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(leafName(), indexVersionCreated, bitsetProducer, indexSettings); builder.enabled = enabled; builder.dynamic = dynamic; builder.includeInRoot = includeInRoot; @@ -276,7 +289,8 @@ NestedObjectMapper withoutMappers() { parentTypeFilter, nestedTypePath, nestedTypeFilter, - bitsetProducer + bitsetProducer, + indexSettings ); } @@ -351,7 +365,8 @@ public ObjectMapper merge(Mapper mergeWith, MapperMergeContext parentMergeContex parentTypeFilter, nestedTypePath, nestedTypeFilter, - bitsetProducer + bitsetProducer, + indexSettings ); } @@ -384,7 +399,9 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { } SourceLoader sourceLoader = new SourceLoader.Synthetic(() -> super.syntheticFieldLoader(mappers.values().stream(), true), NOOP); - var storedFieldLoader = org.elasticsearch.index.fieldvisitor.StoredFieldLoader.create(false, sourceLoader.requiredStoredFields()); + // Some synthetic source use cases require using _ignored_source field + var requiredStoredFields = IgnoredSourceFieldMapper.ensureLoaded(sourceLoader.requiredStoredFields(), indexSettings); + var storedFieldLoader = org.elasticsearch.index.fieldvisitor.StoredFieldLoader.create(false, requiredStoredFields); return new NestedSyntheticFieldLoader( storedFieldLoader, sourceLoader, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index 43bf6f2bd83dd..baff3835d104b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -118,7 +118,6 @@ public Synthetic(Supplier fieldLoaderSupplier, SourceField .storedFieldLoaders() .map(Map.Entry::getKey) .collect(Collectors.toSet()); - this.requiredStoredFields.add(IgnoredSourceFieldMapper.NAME); this.metrics = metrics; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index f303171c7e465..d48c5550631cd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -186,9 +186,8 @@ private static ObjectMapper createObjectMapper(String name) { } private static NestedObjectMapper createNestedObjectMapper(String name) { - return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }).build( - MapperBuilderContext.root(false, false) - ); + return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }, null) + .build(MapperBuilderContext.root(false, false)); } private static MappingLookup createMappingLookup( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java new file mode 100644 index 0000000000000..e08ace01e88e8 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.DirectoryReader; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Map; + +public class IgnoredSourceFieldMapperConfigurationTests extends MapperServiceTestCase { + public void testDisableIgnoredSourceRead() throws IOException { + var mapperService = mapperServiceWithCustomSettings( + Map.of(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING.getKey(), true), + b -> { + b.startObject("fallback_field"); + { + b.field("type", "long").field("doc_values", "false"); + } + b.endObject(); + b.startObject("disabled_object"); + { + b.field("enabled", "false"); + b.startObject("properties"); + { + b.startObject("field").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + ); + + CheckedConsumer inputDocument = b -> { + b.field("fallback_field", 111); + b.startObject("disabled_object"); + { + b.field("field", "hey"); + } + b.endObject(); + }; + + var doc = mapperService.documentMapper().parse(source(inputDocument)); + // Field was written. + assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); + + String syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); + // Values are not loaded. + assertEquals("{}", syntheticSource); + + mapperService.getIndexSettings() + .getScopedSettings() + .applySettings(Settings.builder().put(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING.getKey(), false).build()); + + doc = mapperService.documentMapper().parse(source(inputDocument)); + // Field was written. + assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); + + syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); + // Values are loaded. + assertEquals("{\"disabled_object\":{\"field\":\"hey\"},\"fallback_field\":111}", syntheticSource); + } + + public void testDisableIgnoredSourceWrite() throws IOException { + var mapperService = mapperServiceWithCustomSettings( + Map.of(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING.getKey(), true), + b -> { + b.startObject("fallback_field"); + { + b.field("type", "long").field("doc_values", "false"); + } + b.endObject(); + b.startObject("disabled_object"); + { + b.field("enabled", "false"); + b.startObject("properties"); + { + b.startObject("field").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + ); + + CheckedConsumer inputDocument = b -> { + b.field("fallback_field", 111); + b.startObject("disabled_object"); + { + b.field("field", "hey"); + } + b.endObject(); + }; + + var doc = mapperService.documentMapper().parse(source(inputDocument)); + // Field is not written. + assertNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); + + String syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); + // Values are not loaded. + assertEquals("{}", syntheticSource); + + mapperService.getIndexSettings() + .getScopedSettings() + .applySettings(Settings.builder().put(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING.getKey(), false).build()); + + doc = mapperService.documentMapper().parse(source(inputDocument)); + // Field was written. + assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); + + syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); + // Values are loaded. + assertEquals("{\"disabled_object\":{\"field\":\"hey\"},\"fallback_field\":111}", syntheticSource); + } + + private MapperService mapperServiceWithCustomSettings( + Map customSettings, + CheckedConsumer mapping + ) throws IOException { + var settings = Settings.builder(); + for (var entry : customSettings.entrySet()) { + settings.put(entry.getKey(), entry.getValue()); + } + + return createMapperService(settings.build(), syntheticSourceMapping(mapping)); + } + + protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) + throws IOException { + // Disabling this field via index settings leads to some values not being present in source and assertReaderEquals validation to + // fail as a result. + // This is expected, these settings are introduced only as a safety net when related logic blocks ingestion or search + // and we would rather lose some part of source but unblock the workflow. + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedLookupTests.java index 4953a330107b4..d209d08b48469 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedLookupTests.java @@ -65,9 +65,8 @@ public void testMultiLevelParents() throws IOException { } private static NestedObjectMapper buildMapper(String name) { - return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }).build( - MapperBuilderContext.root(false, false) - ); + return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }, null) + .build(MapperBuilderContext.root(false, false)); } public void testAllParentFilters() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index 37971a1908fa2..0a954115e77f6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -1505,10 +1505,10 @@ public void testIndexTemplatesMergeIncludes() throws IOException { public void testMergeNested() { NestedObjectMapper firstMapper = new NestedObjectMapper.Builder("nested1", IndexVersion.current(), query -> { throw new UnsupportedOperationException(); - }).includeInParent(true).includeInRoot(true).build(MapperBuilderContext.root(false, false)); + }, null).includeInParent(true).includeInRoot(true).build(MapperBuilderContext.root(false, false)); NestedObjectMapper secondMapper = new NestedObjectMapper.Builder("nested1", IndexVersion.current(), query -> { throw new UnsupportedOperationException(); - }).includeInParent(false).includeInRoot(true).build(MapperBuilderContext.root(false, false)); + }, null).includeInParent(false).includeInRoot(true).build(MapperBuilderContext.root(false, false)); MapperException e = expectThrows( MapperException.class, @@ -1855,7 +1855,7 @@ public void testNestedMapperBuilderContextConstructor() { MergeReason mergeReason = randomFrom(MergeReason.values()); MapperBuilderContext mapperBuilderContext = MapperBuilderContext.root(isSourceSynthetic, isDataStream, mergeReason); mapperBuilderContext = mapperBuilderContext.createChildContext("name", parentContainsDimensions, randomFrom(Dynamic.values())); - NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null); + NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null, null); builder.add(new Mapper.Builder("name") { @Override public Mapper build(MapperBuilderContext context) { @@ -1876,7 +1876,7 @@ public void testNestedMapperMergeContextRootConstructor() { MergeReason mergeReason = randomFrom(MergeReason.values()); { MapperBuilderContext mapperBuilderContext = MapperBuilderContext.root(false, false, mergeReason); - NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null); + NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null, null); NestedObjectMapper nestedObjectMapper = builder.build(mapperBuilderContext); MapperMergeContext mapperMergeContext = MapperMergeContext.root(isSourceSynthetic, isDataStream, mergeReason, randomLong()); MapperMergeContext childMergeContext = nestedObjectMapper.createChildContext(mapperMergeContext, "name"); @@ -1907,7 +1907,7 @@ public void testNestedMapperMergeContextFromConstructor() { MergeReason mergeReason = randomFrom(MergeReason.values()); MapperBuilderContext mapperBuilderContext = MapperBuilderContext.root(isSourceSynthetic, isDataStream, mergeReason); mapperBuilderContext = mapperBuilderContext.createChildContext("name", parentContainsDimensions, randomFrom(Dynamic.values())); - NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null); + NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null, null); NestedObjectMapper nestedObjectMapper = builder.build(mapperBuilderContext); MapperMergeContext mapperMergeContext = MapperMergeContext.from(mapperBuilderContext, randomLong()); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java index 78943ed6ccdd7..c7e9f02c283ab 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java @@ -913,8 +913,7 @@ protected List objectMappers() { ); public static NestedObjectMapper nestedObject(String path) { - return new NestedObjectMapper.Builder(path, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }).build( - MapperBuilderContext.root(false, false) - ); + return new NestedObjectMapper.Builder(path, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }, null) + .build(MapperBuilderContext.root(false, false)); } } diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 0e9ca00702b68..583cdf302ad65 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -197,7 +197,7 @@ protected final SearchExecutionContext createMockSearchExecutionContext(IndexSea }; NestedLookup nestedLookup = NestedLookup.build(List.of(new NestedObjectMapper.Builder("path", IndexVersion.current(), query -> { throw new UnsupportedOperationException(); - }).build(MapperBuilderContext.root(false, false)))); + }, null).build(MapperBuilderContext.root(false, false)))); return new SearchExecutionContext( 0, 0, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index 7de0d775ba150..b4be0b33a464e 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ccr.Ccr; @@ -331,6 +332,8 @@ public void testDynamicIndexSettingsAreClassified() { replicatedSettings.add(IndexSettings.MAX_SHINGLE_DIFF_SETTING); replicatedSettings.add(IndexSettings.TIME_SERIES_END_TIME); replicatedSettings.add(IndexSettings.PREFER_ILM_SETTING); + replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); + replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); for (Setting setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) { // removed settings have no effect, they are only there for BWC diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 1577fbc4a642a..81dfba769136b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -80,13 +81,16 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFieldMapper { public static final String CONTENT_TYPE = "semantic_text"; + private final IndexSettings indexSettings; + public static final TypeParser PARSER = new TypeParser( - (n, c) -> new Builder(n, c.indexVersionCreated(), c::bitSetProducer), + (n, c) -> new Builder(n, c.indexVersionCreated(), c::bitSetProducer, c.getIndexSettings()), List.of(notInMultiFields(CONTENT_TYPE), notFromDynamicTemplates(CONTENT_TYPE)) ); public static class Builder extends FieldMapper.Builder { private final IndexVersion indexVersionCreated; + private final IndexSettings indexSettings; private final Parameter inferenceId = Parameter.stringParam( "inference_id", @@ -113,10 +117,22 @@ public static class Builder extends FieldMapper.Builder { private Function inferenceFieldBuilder; - public Builder(String name, IndexVersion indexVersionCreated, Function bitSetProducer) { + public Builder( + String name, + IndexVersion indexVersionCreated, + Function bitSetProducer, + IndexSettings indexSettings + ) { super(name); this.indexVersionCreated = indexVersionCreated; - this.inferenceFieldBuilder = c -> createInferenceField(c, indexVersionCreated, modelSettings.get(), bitSetProducer); + this.indexSettings = indexSettings; + this.inferenceFieldBuilder = c -> createInferenceField( + c, + indexVersionCreated, + modelSettings.get(), + bitSetProducer, + indexSettings + ); } public Builder setInferenceId(String id) { @@ -170,13 +186,20 @@ public SemanticTextFieldMapper build(MapperBuilderContext context) { indexVersionCreated, meta.getValue() ), - builderParams(this, context) + builderParams(this, context), + indexSettings ); } } - private SemanticTextFieldMapper(String simpleName, MappedFieldType mappedFieldType, BuilderParams builderParams) { + private SemanticTextFieldMapper( + String simpleName, + MappedFieldType mappedFieldType, + BuilderParams builderParams, + IndexSettings indexSettings + ) { super(simpleName, mappedFieldType, builderParams); + this.indexSettings = indexSettings; } @Override @@ -188,7 +211,9 @@ public Iterator iterator() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), fieldType().indexVersionCreated, fieldType().getChunksField().bitsetProducer()).init(this); + return new Builder(leafName(), fieldType().indexVersionCreated, fieldType().getChunksField().bitsetProducer(), indexSettings).init( + this + ); } @Override @@ -229,7 +254,8 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio Builder builder = (Builder) new Builder( leafName(), fieldType().indexVersionCreated, - fieldType().getChunksField().bitsetProducer() + fieldType().getChunksField().bitsetProducer(), + indexSettings ).init(this); try { mapper = builder.setModelSettings(field.inference().modelSettings()) @@ -473,19 +499,26 @@ private static ObjectMapper createInferenceField( MapperBuilderContext context, IndexVersion indexVersionCreated, @Nullable SemanticTextField.ModelSettings modelSettings, - Function bitSetProducer + Function bitSetProducer, + IndexSettings indexSettings ) { return new ObjectMapper.Builder(INFERENCE_FIELD, Optional.of(ObjectMapper.Subobjects.ENABLED)).dynamic(ObjectMapper.Dynamic.FALSE) - .add(createChunksField(indexVersionCreated, modelSettings, bitSetProducer)) + .add(createChunksField(indexVersionCreated, modelSettings, bitSetProducer, indexSettings)) .build(context); } private static NestedObjectMapper.Builder createChunksField( IndexVersion indexVersionCreated, @Nullable SemanticTextField.ModelSettings modelSettings, - Function bitSetProducer + Function bitSetProducer, + IndexSettings indexSettings ) { - NestedObjectMapper.Builder chunksField = new NestedObjectMapper.Builder(CHUNKS_FIELD, indexVersionCreated, bitSetProducer); + NestedObjectMapper.Builder chunksField = new NestedObjectMapper.Builder( + CHUNKS_FIELD, + indexVersionCreated, + bitSetProducer, + indexSettings + ); chunksField.dynamic(ObjectMapper.Dynamic.FALSE); KeywordFieldMapper.Builder chunkTextField = new KeywordFieldMapper.Builder(CHUNKED_TEXT_FIELD, indexVersionCreated).indexed(false) .docValues(false);