diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java index 848ee6e556dc1..55b8c18138f46 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java @@ -63,7 +63,7 @@ public void setup() throws IOException { ); includesSet = Set.of(fetchContext.includes()); excludesSet = Set.of(fetchContext.excludes()); - parserConfig = XContentParserConfiguration.EMPTY.withFiltering(includesSet, excludesSet, false); + parserConfig = XContentParserConfiguration.EMPTY.withFiltering(null, includesSet, excludesSet, false); } private BytesReference read300BytesExample() throws IOException { diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java index 334f5ef153048..aa9236e9f314f 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java @@ -170,7 +170,7 @@ private XContentParserConfiguration buildParseConfig(boolean matchDotsInFieldNam includes = null; excludes = filters; } - return XContentParserConfiguration.EMPTY.withFiltering(includes, excludes, matchDotsInFieldNames); + return XContentParserConfiguration.EMPTY.withFiltering(null, includes, excludes, matchDotsInFieldNames); } private BytesReference filter(XContentParserConfiguration contentParserConfiguration) throws IOException { diff --git a/docs/changelog/113827.yaml b/docs/changelog/113827.yaml new file mode 100644 index 0000000000000..2c05f3eeb5d6a --- /dev/null +++ b/docs/changelog/113827.yaml @@ -0,0 +1,5 @@ +pr: 113827 +summary: Add Optional Source Filtering to Source Loaders +area: Mapping +type: enhancement +issues: [] diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentParserConfigurationImpl.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentParserConfigurationImpl.java index a8e039deb38be..70adc59b9c6a9 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentParserConfigurationImpl.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/XContentParserConfigurationImpl.java @@ -19,6 +19,8 @@ import org.elasticsearch.xcontent.provider.filtering.FilterPathBasedFilter; import org.elasticsearch.xcontent.support.filtering.FilterPath; +import java.util.ArrayList; +import java.util.List; import java.util.Set; public class XContentParserConfigurationImpl implements XContentParserConfiguration { @@ -106,12 +108,41 @@ public XContentParserConfiguration withFiltering( Set excludeStrings, boolean filtersMatchFieldNamesWithDots ) { + return withFiltering(null, includeStrings, excludeStrings, filtersMatchFieldNamesWithDots); + } + + public XContentParserConfiguration withFiltering( + String prefixPath, + Set includeStrings, + Set excludeStrings, + boolean filtersMatchFieldNamesWithDots + ) { + FilterPath[] includePaths = FilterPath.compile(includeStrings); + FilterPath[] excludePaths = FilterPath.compile(excludeStrings); + + if (prefixPath != null) { + if (includePaths != null) { + List includeFilters = new ArrayList<>(); + for (var incl : includePaths) { + incl.matches(prefixPath, includeFilters, true); + } + includePaths = includeFilters.isEmpty() ? null : includeFilters.toArray(FilterPath[]::new); + } + + if (excludePaths != null) { + List excludeFilters = new ArrayList<>(); + for (var excl : excludePaths) { + excl.matches(prefixPath, excludeFilters, true); + } + excludePaths = excludeFilters.isEmpty() ? null : excludeFilters.toArray(FilterPath[]::new); + } + } return new XContentParserConfigurationImpl( registry, deprecationHandler, restApiVersion, - FilterPath.compile(includeStrings), - FilterPath.compile(excludeStrings), + includePaths, + excludePaths, filtersMatchFieldNamesWithDots ); } diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java index a8e45e821c220..59e5cd5d6485c 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentParserConfiguration.java @@ -49,10 +49,27 @@ public interface XContentParserConfiguration { RestApiVersion restApiVersion(); + // TODO: Remove when serverless uses the new API + XContentParserConfiguration withFiltering( + Set includeStrings, + Set excludeStrings, + boolean filtersMatchFieldNamesWithDots + ); + /** * Replace the configured filtering. + * + * @param prefixPath The path to be prepended to each sub-path before applying the include/exclude rules. + * Specify {@code null} if parsing starts from the root. + * @param includeStrings A set of strings representing paths to include during filtering. + * If specified, only these paths will be included in parsing. + * @param excludeStrings A set of strings representing paths to exclude during filtering. + * If specified, these paths will be excluded from parsing. + * @param filtersMatchFieldNamesWithDots Indicates whether filters should match field names containing dots ('.') + * as part of the field name. */ XContentParserConfiguration withFiltering( + String prefixPath, Set includeStrings, Set excludeStrings, boolean filtersMatchFieldNamesWithDots diff --git a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java index 453a4473e54c8..481a62a2cd7b9 100644 --- a/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java +++ b/libs/x-content/src/test/java/org/elasticsearch/xcontent/support/filtering/AbstractXContentFilteringTestCase.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.HashSet; import java.util.Set; import java.util.stream.IntStream; @@ -332,6 +333,24 @@ protected final void testFilter(Builder expected, Builder sample, Collection includes, Set excludes, boolean matchFieldNamesWithDots) throws IOException { assertFilterResult(expected.apply(createBuilder()), filter(sample, includes, excludes, matchFieldNamesWithDots)); + + String rootPrefix = "root.path.random"; + if (includes != null) { + Set rootIncludes = new HashSet<>(); + for (var incl : includes) { + rootIncludes.add(rootPrefix + (randomBoolean() ? "." : "*.") + incl); + } + includes = rootIncludes; + } + + if (excludes != null) { + Set rootExcludes = new HashSet<>(); + for (var excl : excludes) { + rootExcludes.add(rootPrefix + (randomBoolean() ? "." : "*.") + excl); + } + excludes = rootExcludes; + } + assertFilterResult(expected.apply(createBuilder()), filterSub(sample, rootPrefix, includes, excludes, matchFieldNamesWithDots)); } public void testArrayWithEmptyObjectInInclude() throws IOException { @@ -413,21 +432,36 @@ private XContentBuilder filter(Builder sample, Set includes, Set && matchFieldNamesWithDots == false) { return filterOnBuilder(sample, includes, excludes); } - return filterOnParser(sample, includes, excludes, matchFieldNamesWithDots); + return filterOnParser(sample, null, includes, excludes, matchFieldNamesWithDots); + } + + private XContentBuilder filterSub( + Builder sample, + String root, + Set includes, + Set excludes, + boolean matchFieldNamesWithDots + ) throws IOException { + return filterOnParser(sample, root, includes, excludes, matchFieldNamesWithDots); } private XContentBuilder filterOnBuilder(Builder sample, Set includes, Set excludes) throws IOException { return sample.apply(XContentBuilder.builder(getXContentType(), includes, excludes)); } - private XContentBuilder filterOnParser(Builder sample, Set includes, Set excludes, boolean matchFieldNamesWithDots) - throws IOException { + private XContentBuilder filterOnParser( + Builder sample, + String rootPath, + Set includes, + Set excludes, + boolean matchFieldNamesWithDots + ) throws IOException { try (XContentBuilder builtSample = sample.apply(createBuilder())) { BytesReference sampleBytes = BytesReference.bytes(builtSample); try ( XContentParser parser = getXContentType().xContent() .createParser( - XContentParserConfiguration.EMPTY.withFiltering(includes, excludes, matchFieldNamesWithDots), + XContentParserConfiguration.EMPTY.withFiltering(rootPath, includes, excludes, matchFieldNamesWithDots), sampleBytes.streamInput() ) ) { diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java index d32e102b2e18b..a645c156b63c7 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java @@ -35,6 +35,7 @@ import org.elasticsearch.script.UpdateScript; import org.elasticsearch.script.UpsertCtxMap; import org.elasticsearch.search.lookup.Source; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; @@ -340,8 +341,9 @@ public static GetResult extractGetResult( return null; } BytesReference sourceFilteredAsBytes = sourceAsBytes; - if (request.fetchSource().hasFilter()) { - sourceFilteredAsBytes = Source.fromMap(source, sourceContentType).filter(request.fetchSource().filter()).internalSourceRef(); + SourceFilter sourceFilter = request.fetchSource().filter(); + if (sourceFilter != null) { + sourceFilteredAsBytes = Source.fromMap(source, sourceContentType).filter(sourceFilter).internalSourceRef(); } // TODO when using delete/none, we can still return the source as bytes by generating it (using the sourceContentType) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 1c6206a4815eb..7745ec9cc75b2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -1361,6 +1361,7 @@ public DataStream getParentDataStream() { } public static final XContentParserConfiguration TS_EXTRACT_CONFIG = XContentParserConfiguration.EMPTY.withFiltering( + null, Set.of(TIMESTAMP_FIELD_NAME), null, false diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index be0e3429a2ce4..a6d9b6ed3767f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -287,7 +287,7 @@ public static class ExtractFromSource extends IndexRouting { trackTimeSeriesRoutingHash = metadata.getCreationVersion().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID); List routingPaths = metadata.getRoutingPaths(); isRoutingPath = Regex.simpleMatcher(routingPaths.toArray(String[]::new)); - this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.copyOf(routingPaths), null, true); + this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(null, Set.copyOf(routingPaths), null, true); } public boolean matchesField(String fieldName) { diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java index 9464ccbcc7aa3..c0eaee071b76c 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java @@ -192,7 +192,7 @@ public static Tuple> convertToMap( ) throws ElasticsearchParseException { XContentParserConfiguration config = XContentParserConfiguration.EMPTY; if (include != null || exclude != null) { - config = config.withFiltering(include, exclude, false); + config = config.withFiltering(null, include, exclude, false); } return parseToType(ordered ? XContentParser::mapOrdered : XContentParser::map, bytes, xContentType, config); } @@ -267,7 +267,10 @@ public static Map convertToMap( @Nullable Set exclude ) throws ElasticsearchParseException { try ( - XContentParser parser = xContent.createParser(XContentParserConfiguration.EMPTY.withFiltering(include, exclude, false), input) + XContentParser parser = xContent.createParser( + XContentParserConfiguration.EMPTY.withFiltering(null, include, exclude, false), + input + ) ) { return ordered ? parser.mapOrdered() : parser.map(); } catch (IOException e) { @@ -302,7 +305,7 @@ public static Map convertToMap( ) throws ElasticsearchParseException { try ( XContentParser parser = xContent.createParser( - XContentParserConfiguration.EMPTY.withFiltering(include, exclude, false), + XContentParserConfiguration.EMPTY.withFiltering(null, include, exclude, false), bytes, offset, length diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java b/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java index c4b03c712c272..a1ba3759c7854 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java @@ -274,24 +274,8 @@ public static Map filter(Map map, String[] inclu */ public static Function, Map> filter(String[] includes, String[] excludes) { CharacterRunAutomaton matchAllAutomaton = new CharacterRunAutomaton(Automata.makeAnyString()); - - CharacterRunAutomaton include; - if (includes == null || includes.length == 0) { - include = matchAllAutomaton; - } else { - Automaton includeA = Regex.simpleMatchToAutomaton(includes); - includeA = Operations.determinize(makeMatchDotsInFieldNames(includeA), MAX_DETERMINIZED_STATES); - include = new CharacterRunAutomaton(includeA); - } - - Automaton excludeA; - if (excludes == null || excludes.length == 0) { - excludeA = Automata.makeEmpty(); - } else { - excludeA = Regex.simpleMatchToAutomaton(excludes); - excludeA = Operations.determinize(makeMatchDotsInFieldNames(excludeA), MAX_DETERMINIZED_STATES); - } - CharacterRunAutomaton exclude = new CharacterRunAutomaton(excludeA); + CharacterRunAutomaton include = compileAutomaton(includes, matchAllAutomaton); + CharacterRunAutomaton exclude = compileAutomaton(excludes, new CharacterRunAutomaton(Automata.makeEmpty())); // NOTE: We cannot use Operations.minus because of the special case that // we want all sub properties to match as soon as an object matches @@ -299,6 +283,15 @@ public static Function, Map> filter(String[] return (map) -> filter(map, include, 0, exclude, 0, matchAllAutomaton); } + public static CharacterRunAutomaton compileAutomaton(String[] patterns, CharacterRunAutomaton defaultValue) { + if (patterns == null || patterns.length == 0) { + return defaultValue; + } + var aut = Regex.simpleMatchToAutomaton(patterns); + aut = Operations.determinize(makeMatchDotsInFieldNames(aut), MAX_DETERMINIZED_STATES); + return new CharacterRunAutomaton(aut); + } + /** Make matches on objects also match dots in field names. * For instance, if the original simple regex is `foo`, this will translate * it into `foo` OR `foo.*`. */ diff --git a/server/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java b/server/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java index 3d3d2f6f66d56..f21a3c06ab015 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java +++ b/server/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java @@ -80,7 +80,7 @@ public LuceneSyntheticSourceChangesSnapshot( assert mappingLookup.isSourceSynthetic(); // ensure we can buffer at least one document this.maxMemorySizeInBytes = maxMemorySizeInBytes > 0 ? maxMemorySizeInBytes : 1; - this.sourceLoader = mappingLookup.newSourceLoader(SourceFieldMetrics.NOOP); + this.sourceLoader = mappingLookup.newSourceLoader(null, SourceFieldMetrics.NOOP); Set storedFields = sourceLoader.requiredStoredFields(); assert mappingLookup.isSourceSynthetic() : "synthetic source must be enabled for proper functionality."; this.storedFieldLoader = StoredFieldLoader.create(false, storedFields); diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java index d41f65fd68fc2..52e9830037832 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java @@ -53,17 +53,24 @@ public static StoredFieldLoader fromSpec(StoredFieldsSpec spec) { return create(spec.requiresSource(), spec.requiredStoredFields()); } + public static StoredFieldLoader create(boolean loadSource, Set fields) { + return create(loadSource, fields, false); + } + /** * Creates a new StoredFieldLoader - * @param loadSource should this loader load the _source field - * @param fields a set of additional fields the loader should load + * + * @param loadSource indicates whether this loader should load the {@code _source} field. + * @param fields a set of additional fields that the loader should load. + * @param forceSequentialReader if {@code true}, forces the use of a sequential leaf reader; + * otherwise, uses the heuristic defined in {@link StoredFieldLoader#reader(LeafReaderContext, int[])}. */ - public static StoredFieldLoader create(boolean loadSource, Set fields) { + public static StoredFieldLoader create(boolean loadSource, Set fields, boolean forceSequentialReader) { List fieldsToLoad = fieldsToLoad(loadSource, fields); return new StoredFieldLoader() { @Override public LeafStoredFieldLoader getLoader(LeafReaderContext ctx, int[] docs) throws IOException { - return new ReaderStoredFieldLoader(reader(ctx, docs), loadSource, fields); + return new ReaderStoredFieldLoader(forceSequentialReader ? sequentialReader(ctx) : reader(ctx, docs), loadSource, fields); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java index 41879c64c3338..43b5d2c7d3f78 100644 --- a/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/server/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -306,9 +306,14 @@ private GetResult innerGetFetch( Map documentFields = null; Map metadataFields = null; DocIdAndVersion docIdAndVersion = get.docIdAndVersion(); + var sourceFilter = fetchSourceContext.filter(); SourceLoader loader = forceSyntheticSource - ? new SourceLoader.Synthetic(mappingLookup.getMapping()::syntheticFieldLoader, mapperMetrics.sourceFieldMetrics()) - : mappingLookup.newSourceLoader(mapperMetrics.sourceFieldMetrics()); + ? new SourceLoader.Synthetic( + sourceFilter, + () -> mappingLookup.getMapping().syntheticFieldLoader(sourceFilter), + mapperMetrics.sourceFieldMetrics() + ) + : mappingLookup.newSourceLoader(fetchSourceContext.filter(), mapperMetrics.sourceFieldMetrics()); StoredFieldLoader storedFieldLoader = buildStoredFieldLoader(storedFields, fetchSourceContext, loader); LeafStoredFieldLoader leafStoredFieldLoader = storedFieldLoader.getLoader(docIdAndVersion.reader.getContext(), null); try { @@ -367,10 +372,6 @@ private GetResult innerGetFetch( if (mapperService.mappingLookup().isSourceEnabled() && fetchSourceContext.fetchSource()) { Source source = loader.leaf(docIdAndVersion.reader, new int[] { docIdAndVersion.docId }) .source(leafStoredFieldLoader, docIdAndVersion.docId); - - if (fetchSourceContext.hasFilter()) { - source = source.filter(fetchSourceContext.filter()); - } sourceBytes = source.internalSourceRef(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index ecc4b92f369d6..a99fa3f93679b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -155,7 +155,7 @@ public void validate(IndexSettings settings, boolean checkLimits) { * with the source loading strategy declared on the source field mapper. */ try { - sourceMapper().newSourceLoader(mapping(), mapperMetrics.sourceFieldMetrics()); + mappingLookup.newSourceLoader(null, mapperMetrics.sourceFieldMetrics()); } catch (IllegalArgumentException e) { mapperMetrics.sourceFieldMetrics().recordSyntheticSourceIncompatibleMapping(); throw e; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java index 57e1ffa322302..7ce955a441f6d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java @@ -156,9 +156,4 @@ public FieldAliasMapper build(MapperBuilderContext context) { return new FieldAliasMapper(leafName(), fullName, path); } } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 4802fb5a28b58..7238127571fed 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -31,6 +31,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; @@ -484,7 +485,7 @@ final SyntheticSourceMode syntheticSourceMode() { /** * Returns synthetic field loader for the mapper. * If mapper does not support synthetic source, it is handled using generic implementation - * in {@link DocumentParser#parseObjectOrField} and {@link ObjectMapper#syntheticFieldLoader()}. + * in {@link DocumentParser#parseObjectOrField} and {@link ObjectMapper#syntheticFieldLoader(SourceFilter)}. *
* * This method is final in order to support common use cases like fallback synthetic source. @@ -492,7 +493,6 @@ final SyntheticSourceMode syntheticSourceMode() { * * @return implementation of {@link SourceLoader.SyntheticFieldLoader} */ - @Override public final SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { if (hasScript()) { return SourceLoader.SyntheticFieldLoader.NOTHING; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 0ecd3cc588d5b..6bc63bdbcceaf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -172,18 +172,6 @@ public final String leafName() { */ public abstract void validate(MappingLookup mappers); - /** - * Create a {@link SourceLoader.SyntheticFieldLoader} to populate synthetic source. - * - * @throws IllegalArgumentException if the field is configured in a way that doesn't - * support synthetic source. This translates nicely into a 400 error when - * users configure synthetic source in the mapping without configuring all - * fields properly. - */ - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - throw new IllegalArgumentException("field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source"); - } - @Override public String toString() { return Strings.toString(this); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java index 52bc48004ccda..1278ebf0a393a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java @@ -13,7 +13,9 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.MapperService.MergeReason; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; @@ -22,6 +24,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; import java.util.stream.Stream; /** @@ -126,9 +129,9 @@ private boolean isSourceSynthetic() { return sfm != null && sfm.isSynthetic(); } - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - var stream = Stream.concat(Stream.of(metadataMappers), root.mappers.values().stream()); - return root.syntheticFieldLoader(stream); + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader(@Nullable SourceFilter filter) { + var mappers = Stream.concat(Stream.of(metadataMappers), root.mappers.values().stream()).collect(Collectors.toList()); + return root.syntheticFieldLoader(filter, mappers, false); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index ce3f8cfb53184..ed02e5fc29617 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -11,10 +11,12 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.InferenceFieldMetadata; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.inference.InferenceService; +import org.elasticsearch.search.lookup.SourceFilter; import java.util.ArrayList; import java.util.Collection; @@ -480,9 +482,11 @@ public boolean isSourceSynthetic() { /** * Build something to load source {@code _source}. */ - public SourceLoader newSourceLoader(SourceFieldMetrics metrics) { - SourceFieldMapper sfm = mapping.getMetadataMapperByClass(SourceFieldMapper.class); - return sfm == null ? SourceLoader.FROM_STORED_SOURCE : sfm.newSourceLoader(mapping, metrics); + public SourceLoader newSourceLoader(@Nullable SourceFilter filter, SourceFieldMetrics metrics) { + if (isSourceSynthetic()) { + return new SourceLoader.Synthetic(filter, () -> mapping.syntheticFieldLoader(filter), metrics); + } + return filter == null ? SourceLoader.FROM_STORED_SOURCE : new SourceLoader.Stored(filter); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java index d0e0dcb6b97ba..03818f7b5c83f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java @@ -23,10 +23,12 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Locale; import java.util.Map; @@ -403,16 +405,18 @@ protected MapperMergeContext createChildContext(MapperMergeContext mapperMergeCo } @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + SourceLoader.SyntheticFieldLoader syntheticFieldLoader(SourceFilter filter, Collection mappers, boolean isFragment) { + // IgnoredSourceFieldMapper integration takes care of writing the source for nested objects that enabled store_array_source. if (sourceKeepMode.orElse(SourceKeepMode.NONE) == SourceKeepMode.ALL) { // IgnoredSourceFieldMapper integration takes care of writing the source for the nested object. return SourceLoader.SyntheticFieldLoader.NOTHING; } - SourceLoader sourceLoader = new SourceLoader.Synthetic(() -> super.syntheticFieldLoader(mappers.values().stream(), true), NOOP); + SourceLoader sourceLoader = new SourceLoader.Synthetic(filter, () -> super.syntheticFieldLoader(filter, mappers, true), NOOP); // Some synthetic source use cases require using _ignored_source field var requiredStoredFields = IgnoredSourceFieldMapper.ensureLoaded(sourceLoader.requiredStoredFields(), indexSettings); - var storedFieldLoader = org.elasticsearch.index.fieldvisitor.StoredFieldLoader.create(false, requiredStoredFields); + // force sequential access since nested fields are indexed per block + var storedFieldLoader = org.elasticsearch.index.fieldvisitor.StoredFieldLoader.create(false, requiredStoredFields, true); return new NestedSyntheticFieldLoader( storedFieldLoader, sourceLoader, @@ -504,5 +508,10 @@ public void write(XContentBuilder b) throws IOException { public String fieldName() { return NestedObjectMapper.this.fullPath(); } + + @Override + public void reset() { + children.clear(); + } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 023f6fcea0bfe..46b70193ba0e8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -24,8 +24,10 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.MapperService.MergeReason; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParserConfiguration; import java.io.IOException; import java.util.ArrayList; @@ -39,6 +41,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.TreeMap; import java.util.stream.Stream; @@ -888,24 +891,40 @@ ObjectMapper findParentMapper(String leafFieldPath) { return null; } - protected SourceLoader.SyntheticFieldLoader syntheticFieldLoader(Stream mappers, boolean isFragment) { - var fields = mappers.sorted(Comparator.comparing(Mapper::fullPath)) - .map(Mapper::syntheticFieldLoader) + SourceLoader.SyntheticFieldLoader syntheticFieldLoader(SourceFilter filter, Collection mappers, boolean isFragment) { + var fields = mappers.stream() + .sorted(Comparator.comparing(Mapper::fullPath)) + .map(m -> innerSyntheticFieldLoader(filter, m)) .filter(l -> l != SourceLoader.SyntheticFieldLoader.NOTHING) .toList(); - return new SyntheticSourceFieldLoader(fields, isFragment); + return new SyntheticSourceFieldLoader(filter, fields, isFragment); } - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader(Stream mappers) { - return syntheticFieldLoader(mappers, false); + final SourceLoader.SyntheticFieldLoader syntheticFieldLoader(@Nullable SourceFilter filter) { + return syntheticFieldLoader(filter, mappers.values(), false); } - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return syntheticFieldLoader(mappers.values().stream()); + private SourceLoader.SyntheticFieldLoader innerSyntheticFieldLoader(SourceFilter filter, Mapper mapper) { + if (mapper instanceof MetadataFieldMapper metaMapper) { + return metaMapper.syntheticFieldLoader(); + } + if (filter != null && filter.isPathFiltered(mapper.fullPath(), mapper instanceof ObjectMapper)) { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } + + if (mapper instanceof ObjectMapper objectMapper) { + return objectMapper.syntheticFieldLoader(filter); + } + + if (mapper instanceof FieldMapper fieldMapper) { + return fieldMapper.syntheticFieldLoader(); + } + return SourceLoader.SyntheticFieldLoader.NOTHING; } private class SyntheticSourceFieldLoader implements SourceLoader.SyntheticFieldLoader { + private final SourceFilter filter; + private final XContentParserConfiguration parserConfig; private final List fields; private final boolean isFragment; @@ -921,9 +940,19 @@ private class SyntheticSourceFieldLoader implements SourceLoader.SyntheticFieldL // Use an ordered map between field names and writers to order writing by field name. private TreeMap currentWriters; - private SyntheticSourceFieldLoader(List fields, boolean isFragment) { + private SyntheticSourceFieldLoader(SourceFilter filter, List fields, boolean isFragment) { this.fields = fields; this.isFragment = isFragment; + this.filter = filter; + String fullPath = ObjectMapper.this.isRoot() ? null : fullPath(); + this.parserConfig = filter == null + ? XContentParserConfiguration.EMPTY + : XContentParserConfiguration.EMPTY.withFiltering( + fullPath, + filter.getIncludes() != null ? Set.of(filter.getIncludes()) : null, + filter.getExcludes() != null ? Set.of(filter.getExcludes()) : null, + true + ); } @Override @@ -994,7 +1023,7 @@ public void prepare() { var existing = currentWriters.get(value.name()); if (existing == null) { - currentWriters.put(value.name(), new FieldWriter.IgnoredSource(value)); + currentWriters.put(value.name(), new FieldWriter.IgnoredSource(filter, value)); } else if (existing instanceof FieldWriter.IgnoredSource isw) { isw.mergeWith(value); } @@ -1031,7 +1060,10 @@ public void write(XContentBuilder b) throws IOException { // If the root object mapper is disabled, it is expected to contain // the source encapsulated within a single ignored source value. assert ignoredValues.size() == 1 : ignoredValues.size(); - XContentDataHelper.decodeAndWrite(b, ignoredValues.get(0).value()); + var value = ignoredValues.get(0).value(); + var type = XContentDataHelper.decodeType(value); + assert type.isPresent(); + XContentDataHelper.decodeAndWriteXContent(parserConfig, b, type.get(), ignoredValues.get(0).value()); softReset(); return; } @@ -1109,11 +1141,20 @@ public boolean hasValue() { } class IgnoredSource implements FieldWriter { + private final XContentParserConfiguration parserConfig; private final String fieldName; private final String leafName; private final List encodedValues; - IgnoredSource(IgnoredSourceFieldMapper.NameValue initialValue) { + IgnoredSource(SourceFilter filter, IgnoredSourceFieldMapper.NameValue initialValue) { + parserConfig = filter == null + ? XContentParserConfiguration.EMPTY + : XContentParserConfiguration.EMPTY.withFiltering( + initialValue.name(), + filter.getIncludes() != null ? Set.of(filter.getIncludes()) : null, + filter.getExcludes() != null ? Set.of(filter.getExcludes()) : null, + true + ); this.fieldName = initialValue.name(); this.leafName = initialValue.getFieldName(); this.encodedValues = new ArrayList<>(); @@ -1124,7 +1165,7 @@ class IgnoredSource implements FieldWriter { @Override public void writeTo(XContentBuilder builder) throws IOException { - XContentDataHelper.writeMerged(builder, leafName, encodedValues); + XContentDataHelper.writeMerged(parserConfig, builder, leafName, encodedValues); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index d491eb9de5886..85f4217811a84 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -454,16 +454,6 @@ public FieldMapper.Builder getMergeBuilder() { return new Builder(null, Settings.EMPTY, false, serializeMode).init(this); } - /** - * Build something to load source {@code _source}. - */ - public SourceLoader newSourceLoader(Mapping mapping, SourceFieldMetrics metrics) { - if (mode == Mode.SYNTHETIC) { - return new SourceLoader.Synthetic(mapping::syntheticFieldLoader, metrics); - } - return SourceLoader.FROM_STORED_SOURCE; - } - public boolean isSynthetic() { return mode == Mode.SYNTHETIC; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index ec255a53e7c5a..27b4f4eb0ae76 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -11,9 +11,11 @@ import org.apache.lucene.index.LeafReader; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.search.lookup.Source; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; @@ -71,7 +73,15 @@ interface Leaf { /** * Load {@code _source} from a stored field. */ - SourceLoader FROM_STORED_SOURCE = new SourceLoader() { + SourceLoader FROM_STORED_SOURCE = new Stored(null); + + class Stored implements SourceLoader { + final SourceFilter filter; + + public Stored(@Nullable SourceFilter filter) { + this.filter = filter; + } + @Override public boolean reordersFieldValues() { return false; @@ -82,7 +92,8 @@ public Leaf leaf(LeafReader reader, int[] docIdsInLeaf) { return new Leaf() { @Override public Source source(LeafStoredFieldLoader storedFields, int docId) throws IOException { - return Source.fromBytes(storedFields.source()); + var res = Source.fromBytes(storedFields.source()); + return filter == null ? res : res.filter(filter); } @Override @@ -97,28 +108,31 @@ public void write(LeafStoredFieldLoader storedFields, int docId, XContentBuilder public Set requiredStoredFields() { return Set.of(); } - }; + } /** * Reconstructs {@code _source} from doc values anf stored fields. */ class Synthetic implements SourceLoader { + private final SourceFilter filter; private final Supplier syntheticFieldLoaderLeafSupplier; private final Set requiredStoredFields; private final SourceFieldMetrics metrics; /** * Creates a {@link SourceLoader} to reconstruct {@code _source} from doc values anf stored fields. + * @param filter An optional filter to include/exclude fields. * @param fieldLoaderSupplier A supplier to create {@link SyntheticFieldLoader}, one for each leaf. * @param metrics Metrics for profiling. */ - public Synthetic(Supplier fieldLoaderSupplier, SourceFieldMetrics metrics) { + public Synthetic(@Nullable SourceFilter filter, Supplier fieldLoaderSupplier, SourceFieldMetrics metrics) { this.syntheticFieldLoaderLeafSupplier = fieldLoaderSupplier; this.requiredStoredFields = syntheticFieldLoaderLeafSupplier.get() .storedFieldLoaders() .map(Map.Entry::getKey) .collect(Collectors.toSet()); this.metrics = metrics; + this.filter = filter; } @Override @@ -134,7 +148,7 @@ public Set requiredStoredFields() { @Override public Leaf leaf(LeafReader reader, int[] docIdsInLeaf) throws IOException { SyntheticFieldLoader loader = syntheticFieldLoaderLeafSupplier.get(); - return new LeafWithMetrics(new SyntheticLeaf(loader, loader.docValuesLoader(reader, docIdsInLeaf)), metrics); + return new LeafWithMetrics(new SyntheticLeaf(filter, loader, loader.docValuesLoader(reader, docIdsInLeaf)), metrics); } private record LeafWithMetrics(Leaf leaf, SourceFieldMetrics metrics) implements Leaf { @@ -163,11 +177,13 @@ public void write(LeafStoredFieldLoader storedFields, int docId, XContentBuilder } private static class SyntheticLeaf implements Leaf { + private final SourceFilter filter; private final SyntheticFieldLoader loader; private final SyntheticFieldLoader.DocValuesLoader docValuesLoader; private final Map storedFieldLoaders; - private SyntheticLeaf(SyntheticFieldLoader loader, SyntheticFieldLoader.DocValuesLoader docValuesLoader) { + private SyntheticLeaf(SourceFilter filter, SyntheticFieldLoader loader, SyntheticFieldLoader.DocValuesLoader docValuesLoader) { + this.filter = filter; this.loader = loader; this.docValuesLoader = docValuesLoader; this.storedFieldLoaders = Map.copyOf( @@ -199,6 +215,11 @@ public void write(LeafStoredFieldLoader storedFieldLoader, int docId, XContentBu objectsWithIgnoredFields = new HashMap<>(); } IgnoredSourceFieldMapper.NameValue nameValue = IgnoredSourceFieldMapper.decode(value); + if (filter != null + && filter.isPathFiltered(nameValue.name(), XContentDataHelper.isEncodedObject(nameValue.value()))) { + // This path is filtered by the include/exclude rules + continue; + } objectsWithIgnoredFields.computeIfAbsent(nameValue.getParentFieldName(), k -> new ArrayList<>()).add(nameValue); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java index dee5ff92040a9..646368b96a4c5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.CheckedFunction; +import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.Tuple; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -109,29 +110,53 @@ static void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { } } + /** + * Determines if the given {@link BytesRef}, encoded with {@link XContentDataHelper#encodeToken(XContentParser)}, + * is an encoded object. + */ + static boolean isEncodedObject(BytesRef encoded) { + return switch ((char) encoded.bytes[encoded.offset]) { + case CBOR_OBJECT_ENCODING, YAML_OBJECT_ENCODING, JSON_OBJECT_ENCODING, SMILE_OBJECT_ENCODING -> true; + default -> false; + }; + } + + static Optional decodeType(BytesRef encodedValue) { + return switch ((char) encodedValue.bytes[encodedValue.offset]) { + case CBOR_OBJECT_ENCODING, JSON_OBJECT_ENCODING, YAML_OBJECT_ENCODING, SMILE_OBJECT_ENCODING -> Optional.of( + getXContentType(encodedValue) + ); + default -> Optional.empty(); + }; + } + /** * Writes encoded values to provided builder. If there are multiple values they are merged into * a single resulting array. * * Note that this method assumes all encoded parts have values that need to be written (are not VOID encoded). + * @param parserConfig The configuration for the parsing of the provided {@code encodedParts}. * @param b destination * @param fieldName name of the field that is written * @param encodedParts subset of field data encoded using methods of this class. Can contain arrays which will be flattened. * @throws IOException */ - static void writeMerged(XContentBuilder b, String fieldName, List encodedParts) throws IOException { + static void writeMerged(XContentParserConfiguration parserConfig, XContentBuilder b, String fieldName, List encodedParts) + throws IOException { if (encodedParts.isEmpty()) { return; } - if (encodedParts.size() == 1) { - b.field(fieldName); - XContentDataHelper.decodeAndWrite(b, encodedParts.get(0)); - return; - } - - b.startArray(fieldName); + boolean isArray = encodedParts.size() > 1; + // xcontent filtering can remove all values so we delay the start of the field until we have an actual value to write. + CheckedRunnable startField = () -> { + if (isArray) { + b.startArray(fieldName); + } else { + b.field(fieldName); + } + }; for (var encodedValue : encodedParts) { Optional encodedXContentType = switch ((char) encodedValue.bytes[encodedValue.offset]) { case CBOR_OBJECT_ENCODING, JSON_OBJECT_ENCODING, YAML_OBJECT_ENCODING, SMILE_OBJECT_ENCODING -> Optional.of( @@ -140,27 +165,33 @@ static void writeMerged(XContentBuilder b, String fieldName, List enco default -> Optional.empty(); }; if (encodedXContentType.isEmpty()) { + if (startField != null) { + // first value to write + startField.run(); + startField = null; + } // This is a plain value, we can just write it XContentDataHelper.decodeAndWrite(b, encodedValue); } else { - // Encoded value could be an array which needs to be flattened - // since we are already inside an array. + // Encoded value could be an object or an array of objects that needs + // to be filtered or flattened. try ( XContentParser parser = encodedXContentType.get() .xContent() - .createParser( - XContentParserConfiguration.EMPTY, - encodedValue.bytes, - encodedValue.offset + 1, - encodedValue.length - 1 - ) + .createParser(parserConfig, encodedValue.bytes, encodedValue.offset + 1, encodedValue.length - 1) ) { - if (parser.currentToken() == null) { - parser.nextToken(); + if ((parser.currentToken() == null) && (parser.nextToken() == null)) { + // the entire content is filtered by include/exclude rules + continue; } - // It's an array, we will flatten it. - if (parser.currentToken() == XContentParser.Token.START_ARRAY) { + if (startField != null) { + // first value to write + startField.run(); + startField = null; + } + if (isArray && parser.currentToken() == XContentParser.Token.START_ARRAY) { + // Encoded value is an array which needs to be flattened since we are already inside an array. while (parser.nextToken() != XContentParser.Token.END_ARRAY) { b.copyCurrentStructure(parser); } @@ -171,8 +202,9 @@ static void writeMerged(XContentBuilder b, String fieldName, List enco } } } - - b.endArray(); + if (isArray) { + b.endArray(); + } } public static boolean isDataPresent(BytesRef encoded) { @@ -509,10 +541,10 @@ byte[] encode(XContentParser parser) throws IOException { @Override void decodeAndWrite(XContentBuilder b, BytesRef r) throws IOException { switch ((char) r.bytes[r.offset]) { - case CBOR_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.CBOR, r); - case JSON_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.JSON, r); - case SMILE_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.SMILE, r); - case YAML_OBJECT_ENCODING -> decodeAndWriteXContent(b, XContentType.YAML, r); + case CBOR_OBJECT_ENCODING -> decodeAndWriteXContent(XContentParserConfiguration.EMPTY, b, XContentType.CBOR, r); + case JSON_OBJECT_ENCODING -> decodeAndWriteXContent(XContentParserConfiguration.EMPTY, b, XContentType.JSON, r); + case SMILE_OBJECT_ENCODING -> decodeAndWriteXContent(XContentParserConfiguration.EMPTY, b, XContentType.SMILE, r); + case YAML_OBJECT_ENCODING -> decodeAndWriteXContent(XContentParserConfiguration.EMPTY, b, XContentType.YAML, r); default -> throw new IllegalArgumentException("Can't decode " + r); } } @@ -606,11 +638,15 @@ static byte[] encode(XContentBuilder builder) throws IOException { assert position == encoded.length; return encoded; } + } - static void decodeAndWriteXContent(XContentBuilder b, XContentType type, BytesRef r) throws IOException { - try ( - XContentParser parser = type.xContent().createParser(XContentParserConfiguration.EMPTY, r.bytes, r.offset + 1, r.length - 1) - ) { + public static void decodeAndWriteXContent(XContentParserConfiguration parserConfig, XContentBuilder b, XContentType type, BytesRef r) + throws IOException { + try (XContentParser parser = type.xContent().createParser(parserConfig, r.bytes, r.offset + 1, r.length - 1)) { + if ((parser.currentToken() == null) && (parser.nextToken() == null)) { + // This can occur when all fields in a sub-object or all entries in an array of objects have been filtered out. + b.startObject().endObject(); + } else { b.copyCurrentStructure(parser); } } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index d5e48a6a54daa..fbc3696d40221 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -439,9 +439,13 @@ public boolean isSourceSynthetic() { */ public SourceLoader newSourceLoader(boolean forceSyntheticSource) { if (forceSyntheticSource) { - return new SourceLoader.Synthetic(mappingLookup.getMapping()::syntheticFieldLoader, mapperMetrics.sourceFieldMetrics()); + return new SourceLoader.Synthetic( + null, + () -> mappingLookup.getMapping().syntheticFieldLoader(null), + mapperMetrics.sourceFieldMetrics() + ); } - return mappingLookup.newSourceLoader(mapperMetrics.sourceFieldMetrics()); + return mappingLookup.newSourceLoader(null, mapperMetrics.sourceFieldMetrics()); } /** @@ -501,7 +505,7 @@ public SearchLookup lookup() { public SourceProvider createSourceProvider() { return isSourceSynthetic() - ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping(), mapperMetrics.sourceFieldMetrics()) + ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping(), null, mapperMetrics.sourceFieldMetrics()) : SourceProvider.fromStoredFields(); } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceContext.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceContext.java index 126c7aa28f4d1..0594fa4909783 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceContext.java @@ -85,12 +85,15 @@ public String[] excludes() { return this.excludes; } - public boolean hasFilter() { + private boolean hasFilter() { return this.includes.length > 0 || this.excludes.length > 0; } + /** + * Returns a {@link SourceFilter} if filtering is enabled, {@code null} otherwise. + */ public SourceFilter filter() { - return new SourceFilter(includes, excludes); + return hasFilter() ? new SourceFilter(includes, excludes) : null; } public static FetchSourceContext parseFromRestRequest(RestRequest request) { diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java index e151f0fc2e090..79e51036a91be 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java @@ -29,7 +29,7 @@ public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) { } assert fetchSourceContext.fetchSource(); SourceFilter sourceFilter = fetchSourceContext.filter(); - final boolean filterExcludesAll = sourceFilter.excludesAll(); + final boolean filterExcludesAll = sourceFilter != null && sourceFilter.excludesAll(); return new FetchSubPhaseProcessor() { private int fastPath; @@ -47,22 +47,22 @@ public StoredFieldsSpec storedFieldsSpec() { public void process(HitContext hitContext) { String index = fetchContext.getIndexName(); if (fetchContext.getSearchExecutionContext().isSourceEnabled() == false) { - if (fetchSourceContext.hasFilter()) { + if (sourceFilter != null) { throw new IllegalArgumentException( "unable to fetch fields from _source field: _source is disabled in the mappings for index [" + index + "]" ); } return; } - hitExecute(fetchSourceContext, hitContext); + hitExecute(hitContext); } - private void hitExecute(FetchSourceContext fetchSourceContext, HitContext hitContext) { + private void hitExecute(HitContext hitContext) { final boolean nestedHit = hitContext.hit().getNestedIdentity() != null; Source source = hitContext.source(); // If this is a parent document and there are no source filters, then add the source as-is. - if (nestedHit == false && fetchSourceContext.hasFilter() == false) { + if (nestedHit == false && sourceFilter == null) { hitContext.hit().sourceRef(source.internalSourceRef()); fastPath++; return; @@ -73,7 +73,7 @@ private void hitExecute(FetchSourceContext fetchSourceContext, HitContext hitCon source = Source.empty(source.sourceContentType()); } else { // Otherwise, filter the source and add it to the hit. - source = source.filter(sourceFilter); + source = sourceFilter != null ? source.filter(sourceFilter) : source; } if (nestedHit) { source = extractNested(source, hitContext.hit().getNestedIdentity()); diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceFilter.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceFilter.java index d3951700f3c9f..90034ef447c92 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceFilter.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceFilter.java @@ -9,6 +9,8 @@ package org.elasticsearch.search.lookup; +import org.apache.lucene.util.automaton.Automata; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -40,6 +42,8 @@ public final class SourceFilter { private final boolean empty; private final String[] includes; private final String[] excludes; + private CharacterRunAutomaton includeAut; + private CharacterRunAutomaton excludeAut; /** * Construct a new filter based on a list of includes and excludes @@ -56,6 +60,53 @@ public SourceFilter(String[] includes, String[] excludes) { this.empty = CollectionUtils.isEmpty(this.includes) && CollectionUtils.isEmpty(this.excludes); } + public String[] getIncludes() { + return includes; + } + + public String[] getExcludes() { + return excludes; + } + + /** + * Determines whether the given full path should be filtered out. + * + * @param fullPath The full path to evaluate. + * @param isObject Indicates if the path represents an object. + * @return {@code true} if the path should be filtered out, {@code false} otherwise. + */ + public boolean isPathFiltered(String fullPath, boolean isObject) { + final boolean included; + if (includes != null) { + if (includeAut == null) { + includeAut = XContentMapValues.compileAutomaton(includes, new CharacterRunAutomaton(Automata.makeAnyString())); + } + int state = step(includeAut, fullPath, 0); + included = state != -1 && (isObject || includeAut.isAccept(state)); + } else { + included = true; + } + + if (excludes != null) { + if (excludeAut == null) { + excludeAut = XContentMapValues.compileAutomaton(excludes, new CharacterRunAutomaton(Automata.makeEmpty())); + } + int state = step(excludeAut, fullPath, 0); + if (state != -1 && excludeAut.isAccept(state)) { + return true; + } + } + + return included == false; + } + + private static int step(CharacterRunAutomaton automaton, String key, int state) { + for (int i = 0; state != -1 && i < key.length(); ++i) { + state = automaton.step(state, key.charAt(i)); + } + return state; + } + /** * Filter a Source using its map representation */ @@ -87,6 +138,7 @@ private Function buildBytesFilter() { return this::filterMap; } final XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withFiltering( + null, Set.copyOf(Arrays.asList(includes)), Set.copyOf(Arrays.asList(excludes)), true diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java index e232aec5d1f6c..4696ef2299fd7 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java @@ -48,7 +48,7 @@ static SourceProvider fromStoredFields() { * but it is not safe to use this to access documents from the same segment across * multiple threads. */ - static SourceProvider fromSyntheticSource(Mapping mapping, SourceFieldMetrics metrics) { - return new SyntheticSourceProvider(new SourceLoader.Synthetic(mapping::syntheticFieldLoader, metrics)); + static SourceProvider fromSyntheticSource(Mapping mapping, SourceFilter filter, SourceFieldMetrics metrics) { + return new SyntheticSourceProvider(new SourceLoader.Synthetic(filter, () -> mapping.syntheticFieldLoader(filter), metrics)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java index 4101828d4cd24..84f17c2fc3d6a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocCountFieldMapperTests.java @@ -98,7 +98,7 @@ public void testSyntheticSourceMany() throws IOException { iw.addDocument(mapper.documentMapper().parse(source(b -> b.field("doc", doc).field(CONTENT_TYPE, c))).rootDoc()); } }, reader -> { - SourceLoader loader = mapper.mappingLookup().newSourceLoader(SourceFieldMetrics.NOOP); + SourceLoader loader = mapper.mappingLookup().newSourceLoader(null, SourceFieldMetrics.NOOP); assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_source")); for (LeafReaderContext leaf : reader.leaves()) { int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray(); @@ -130,7 +130,7 @@ public void testSyntheticSourceManyDoNotHave() throws IOException { })).rootDoc()); } }, reader -> { - SourceLoader loader = mapper.mappingLookup().newSourceLoader(SourceFieldMetrics.NOOP); + SourceLoader loader = mapper.mappingLookup().newSourceLoader(null, SourceFieldMetrics.NOOP); assertThat(loader.requiredStoredFields(), Matchers.contains("_ignored_source")); for (LeafReaderContext leaf : reader.leaves()) { int[] docIds = IntStream.range(0, leaf.reader().maxDoc()).toArray(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index b43371594d57b..14902aa419b9f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -12,6 +12,8 @@ import org.apache.lucene.index.DirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.test.FieldMaskingReader; import org.elasticsearch.xcontent.XContentBuilder; import org.hamcrest.Matchers; @@ -46,8 +48,15 @@ private ParsedDocument getParsedDocumentWithFieldLimit(CheckedConsumer build) throws IOException { + return getSyntheticSourceWithFieldLimit(null, build); + } + + private String getSyntheticSourceWithFieldLimit( + @Nullable SourceFilter sourceFilter, + CheckedConsumer build + ) throws IOException { DocumentMapper documentMapper = getDocumentMapperWithFieldLimit(); - return syntheticSource(documentMapper, build); + return syntheticSource(documentMapper, sourceFilter, build); } private MapperService createMapperServiceWithStoredArraySource(XContentBuilder mappings) throws IOException { @@ -62,36 +71,120 @@ private MapperService createMapperServiceWithStoredArraySource(XContentBuilder m public void testIgnoredBoolean() throws IOException { boolean value = randomBoolean(); assertEquals("{\"my_value\":" + value + "}", getSyntheticSourceWithFieldLimit(b -> b.field("my_value", value))); + assertEquals( + "{\"my_value\":" + value + "}", + getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_value" }, null), b -> b.field("my_value", value)) + ); + assertEquals( + "{}", + getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_value" }), b -> b.field("my_value", value)) + ); + } + + public void testIgnoredBooleanArray() throws IOException { + assertEquals( + "{\"my_value\":[false,true,false]}", + getSyntheticSourceWithFieldLimit(b -> b.field("my_value", new boolean[] { false, true, false })) + ); + assertEquals( + "{\"my_value\":[false,true,false]}", + getSyntheticSourceWithFieldLimit( + new SourceFilter(new String[] { "my_value" }, null), + b -> b.array("my_value", new boolean[] { false, true, false }) + ) + ); + assertEquals( + "{}", + getSyntheticSourceWithFieldLimit( + new SourceFilter(null, new String[] { "my_value" }), + b -> b.field("my_value", new boolean[] { false, true, false }) + ) + ); + assertEquals( + "{}", + getSyntheticSourceWithFieldLimit( + new SourceFilter(new String[] { "my_value.object" }, null), + b -> b.array("my_value", new boolean[] { false, true, false }) + ) + ); } public void testIgnoredString() throws IOException { String value = randomAlphaOfLength(5); assertEquals("{\"my_value\":\"" + value + "\"}", getSyntheticSourceWithFieldLimit(b -> b.field("my_value", value))); + assertEquals( + "{\"my_value\":\"" + value + "\"}", + getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_value" }, null), b -> b.field("my_value", value)) + ); + assertEquals( + "{}", + getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_value" }), b -> b.field("my_value", value)) + ); } public void testIgnoredInt() throws IOException { int value = randomInt(); assertEquals("{\"my_value\":" + value + "}", getSyntheticSourceWithFieldLimit(b -> b.field("my_value", value))); + assertEquals( + "{\"my_value\":" + value + "}", + getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_value" }, null), b -> b.field("my_value", value)) + ); + assertEquals( + "{}", + getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_value" }), b -> b.field("my_value", value)) + ); } public void testIgnoredLong() throws IOException { long value = randomLong(); assertEquals("{\"my_value\":" + value + "}", getSyntheticSourceWithFieldLimit(b -> b.field("my_value", value))); + assertEquals( + "{\"my_value\":" + value + "}", + getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_value" }, null), b -> b.field("my_value", value)) + ); + assertEquals( + "{}", + getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_value" }), b -> b.field("my_value", value)) + ); } public void testIgnoredFloat() throws IOException { float value = randomFloat(); assertEquals("{\"my_value\":" + value + "}", getSyntheticSourceWithFieldLimit(b -> b.field("my_value", value))); + assertEquals( + "{\"my_value\":" + value + "}", + getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_value" }, null), b -> b.field("my_value", value)) + ); + assertEquals( + "{}", + getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_value" }), b -> b.field("my_value", value)) + ); } public void testIgnoredDouble() throws IOException { double value = randomDouble(); assertEquals("{\"my_value\":" + value + "}", getSyntheticSourceWithFieldLimit(b -> b.field("my_value", value))); + assertEquals( + "{\"my_value\":" + value + "}", + getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_value" }, null), b -> b.field("my_value", value)) + ); + assertEquals( + "{}", + getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_value" }), b -> b.field("my_value", value)) + ); } public void testIgnoredBigInteger() throws IOException { BigInteger value = randomBigInteger(); assertEquals("{\"my_value\":" + value + "}", getSyntheticSourceWithFieldLimit(b -> b.field("my_value", value))); + assertEquals( + "{\"my_value\":" + value + "}", + getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_value" }, null), b -> b.field("my_value", value)) + ); + assertEquals( + "{}", + getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_value" }), b -> b.field("my_value", value)) + ); } public void testIgnoredBytes() throws IOException { @@ -100,6 +193,14 @@ public void testIgnoredBytes() throws IOException { "{\"my_value\":\"" + Base64.getEncoder().encodeToString(value) + "\"}", getSyntheticSourceWithFieldLimit(b -> b.field("my_value", value)) ); + assertEquals( + "{\"my_value\":\"" + Base64.getEncoder().encodeToString(value) + "\"}", + getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_value" }, null), b -> b.field("my_value", value)) + ); + assertEquals( + "{}", + getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_value" }), b -> b.field("my_value", value)) + ); } public void testIgnoredObjectBoolean() throws IOException { @@ -107,15 +208,124 @@ public void testIgnoredObjectBoolean() throws IOException { assertEquals("{\"my_object\":{\"my_value\":" + value + "}}", getSyntheticSourceWithFieldLimit(b -> { b.startObject("my_object").field("my_value", value).endObject(); })); + + assertEquals( + "{\"my_object\":{\"my_value\":" + value + "}}", + getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_object" }, null), b -> { + b.startObject("my_object").field("my_value", value).endObject(); + }) + ); + + assertEquals( + "{\"my_object\":{\"my_value\":" + value + "}}", + getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_object.my_value" }, null), b -> { + b.startObject("my_object").field("my_value", value).endObject(); + }) + ); + + assertEquals("{}", getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_object" }), b -> { + b.startObject("my_object").field("my_value", value).endObject(); + })); + + assertEquals("{}", getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_object.my_value" }), b -> { + b.startObject("my_object").field("my_value", value).endObject(); + })); + + assertEquals( + "{\"my_object\":{\"another_value\":\"0\"}}", + getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_object.my_value" }), b -> { + b.startObject("my_object").field("my_value", value).field("another_value", "0").endObject(); + }) + ); + } + + public void testIgnoredArrayOfObjects() throws IOException { + boolean value = randomBoolean(); + int another_value = randomInt(); + + assertEquals( + "{\"my_object\":[{\"my_value\":" + value + "},{\"another_value\":" + another_value + "}]}", + getSyntheticSourceWithFieldLimit(b -> { + b.startArray("my_object"); + b.startObject().field("my_value", value).endObject(); + b.startObject().field("another_value", another_value).endObject(); + b.endArray(); + }) + ); + + assertEquals( + "{\"my_object\":[{\"another_value\":" + another_value + "}]}", + getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_object.my_value" }), b -> { + b.startArray("my_object"); + b.startObject().field("my_value", value).endObject(); + b.startObject().field("another_value", another_value).endObject(); + b.endArray(); + }) + ); + + assertEquals( + "{\"my_object\":[{\"my_value\":" + value + "}]}", + getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_object.another_value" }), b -> { + b.startArray("my_object"); + b.startObject().field("my_value", value).endObject(); + b.startObject().field("another_value", another_value).endObject(); + b.endArray(); + }) + ); + + assertEquals( + "{}", + getSyntheticSourceWithFieldLimit( + new SourceFilter(null, new String[] { "my_object.another_value", "my_object.my_value" }), + b -> { + b.startArray("my_object"); + b.startObject().field("my_value", value).endObject(); + b.startObject().field("another_value", another_value).endObject(); + b.endArray(); + } + ) + ); + + assertEquals( + "{\"my_object\":[{\"another_field2\":2}]}", + getSyntheticSourceWithFieldLimit( + new SourceFilter(null, new String[] { "my_object.another_field1", "my_object.my_value" }), + b -> { + b.startArray("my_object"); + b.startObject().field("my_value", value).endObject(); + b.startObject().field("another_field1", 1).endObject(); + b.startObject().field("another_field2", 2).endObject(); + b.endArray(); + } + ) + ); + + assertEquals("{}", getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_object" }), b -> { + b.startArray("my_object"); + b.startObject().field("my_value", value).endObject(); + b.startObject().field("another_value", another_value).endObject(); + b.endArray(); + })); } public void testIgnoredArray() throws IOException { - assertEquals("{\"my_array\":[{\"int_value\":10},{\"int_value\":20}]}", getSyntheticSourceWithFieldLimit(b -> { + assertEquals( + "{\"my_array\":[{\"int_value\":10},{\"int_value\":20}]}", + getSyntheticSourceWithFieldLimit(new SourceFilter(new String[] { "my_array" }, null), b -> { + b.startArray("my_array"); + b.startObject().field("int_value", 10).endObject(); + b.startObject().field("int_value", 20).endObject(); + b.endArray(); + }) + ); + + assertEquals("{}", getSyntheticSourceWithFieldLimit(new SourceFilter(null, new String[] { "my_array" }), b -> { b.startArray("my_array"); b.startObject().field("int_value", 10).endObject(); b.startObject().field("int_value", 20).endObject(); b.endArray(); })); + } public void testEncodeFieldToMap() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index 527d7497a8418..911fe6d4b9337 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -654,8 +654,8 @@ public void testSubobjectsAutoRootWithInnerNested() throws IOException { public void testSyntheticSourceDocValuesEmpty() throws IOException { DocumentMapper mapper = createDocumentMapper(mapping(b -> b.startObject("o").field("type", "object").endObject())); ObjectMapper o = (ObjectMapper) mapper.mapping().getRoot().getMapper("o"); - assertThat(o.syntheticFieldLoader().docValuesLoader(null, null), nullValue()); - assertThat(mapper.mapping().getRoot().syntheticFieldLoader().docValuesLoader(null, null), nullValue()); + assertThat(o.syntheticFieldLoader(null).docValuesLoader(null, null), nullValue()); + assertThat(mapper.mapping().getRoot().syntheticFieldLoader(null).docValuesLoader(null, null), nullValue()); } /** @@ -680,8 +680,8 @@ public void testSyntheticSourceDocValuesFieldWithout() throws IOException { b.endObject().endObject(); })); ObjectMapper o = (ObjectMapper) mapper.mapping().getRoot().getMapper("o"); - assertThat(o.syntheticFieldLoader().docValuesLoader(null, null), nullValue()); - assertThat(mapper.mapping().getRoot().syntheticFieldLoader().docValuesLoader(null, null), nullValue()); + assertThat(o.syntheticFieldLoader(null).docValuesLoader(null, null), nullValue()); + assertThat(mapper.mapping().getRoot().syntheticFieldLoader(null).docValuesLoader(null, null), nullValue()); } public void testStoreArraySourceinSyntheticSourceMode() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index 3a091bf539229..c36a126479e87 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -408,7 +408,7 @@ protected Source getSourceFor(CheckedConsumer mapp iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { - SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), SourceFieldMetrics.NOOP); + SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), null, SourceFieldMetrics.NOOP); Source syntheticSource = provider.getSource(getOnlyLeafReader(reader).getContext(), 0); return syntheticSource; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java index c640cea16487b..ea9f8f6ae28a7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java @@ -47,6 +47,7 @@ public void testSyntheticSourceLoadLatency() throws IOException { try (DirectoryReader reader = DirectoryReader.open(directory)) { SourceProvider provider = SourceProvider.fromSyntheticSource( mapper.mapping(), + null, createTestMapperMetrics().sourceFieldMetrics() ); Source synthetic = provider.getSource(getOnlyLeafReader(reader).getContext(), 0); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java index c2e49759cdfde..8b4176d6b9631 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java @@ -21,7 +21,7 @@ public void testNonSynthetic() throws IOException { b.startObject("o").field("type", "object").endObject(); b.startObject("kwd").field("type", "keyword").endObject(); })); - assertFalse(mapper.mappers().newSourceLoader(SourceFieldMetrics.NOOP).reordersFieldValues()); + assertFalse(mapper.mappers().newSourceLoader(null, SourceFieldMetrics.NOOP).reordersFieldValues()); } public void testEmptyObject() throws IOException { @@ -29,7 +29,7 @@ public void testEmptyObject() throws IOException { b.startObject("o").field("type", "object").endObject(); b.startObject("kwd").field("type", "keyword").endObject(); })).documentMapper(); - assertTrue(mapper.mappers().newSourceLoader(SourceFieldMetrics.NOOP).reordersFieldValues()); + assertTrue(mapper.mappers().newSourceLoader(null, SourceFieldMetrics.NOOP).reordersFieldValues()); assertThat(syntheticSource(mapper, b -> b.field("kwd", "foo")), equalTo(""" {"kwd":"foo"}""")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 7f9474f5bab83..32cbcfc2441a1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1354,7 +1354,7 @@ private void testBlockLoaderFromParent(boolean columnReader, boolean syntheticSo XContentBuilder mapping = mapping(buildFields); MapperService mapper = syntheticSource ? createSytheticSourceMapperService(mapping) : createMapperService(mapping); BlockReaderSupport blockReaderSupport = getSupportedReaders(mapper, "field.sub"); - var sourceLoader = mapper.mappingLookup().newSourceLoader(SourceFieldMetrics.NOOP); + var sourceLoader = mapper.mappingLookup().newSourceLoader(null, SourceFieldMetrics.NOOP); testBlockLoader(columnReader, example, blockReaderSupport, sourceLoader); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java index f4e114da1fa51..ecf59b611080b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/XContentDataHelperTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; @@ -29,6 +30,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Stream; import static org.hamcrest.Matchers.equalTo; @@ -57,6 +59,7 @@ private String encodeAndDecodeCustom(XContentType type, Object value) throws IOE parser.nextToken(); var encoded = XContentDataHelper.encodeToken(parser); + assertThat(XContentDataHelper.isEncodedObject(encoded), equalTo(value instanceof Map)); var decoded = XContentFactory.jsonBuilder(); XContentDataHelper.decodeAndWrite(decoded, encoded); @@ -124,6 +127,7 @@ public void testEmbeddedObject() throws IOException { assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken()); parser.nextToken(); var encoded = XContentDataHelper.encodeToken(parser); + assertFalse(XContentDataHelper.isEncodedObject(encoded)); var decoded = XContentFactory.jsonBuilder(); XContentDataHelper.decodeAndWrite(decoded, encoded); @@ -132,6 +136,7 @@ public void testEmbeddedObject() throws IOException { } var encoded = XContentDataHelper.encodeXContentBuilder(builder); + assertTrue(XContentDataHelper.isEncodedObject(encoded)); var decoded = XContentFactory.jsonBuilder(); XContentDataHelper.decodeAndWrite(decoded, encoded); @@ -147,7 +152,9 @@ public void testObject() throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); builder.humanReadable(true); - XContentDataHelper.decodeAndWrite(builder, XContentDataHelper.encodeToken(p)); + var encoded = XContentDataHelper.encodeToken(p); + assertTrue(XContentDataHelper.isEncodedObject(encoded)); + XContentDataHelper.decodeAndWrite(builder, encoded); assertEquals(object, Strings.toString(builder)); XContentBuilder builder2 = XContentFactory.jsonBuilder(); @@ -156,6 +163,62 @@ public void testObject() throws IOException { assertEquals(object, Strings.toString(builder2)); } + public void testObjectWithFilter() throws IOException { + String object = "{\"name\":\"foo\",\"path\":{\"filter\":{\"keep\":[0],\"field\":\"value\"}}}"; + String filterObject = "{\"name\":\"foo\",\"path\":{\"filter\":{\"keep\":[0]}}}"; + + XContentParser p = createParser(JsonXContent.jsonXContent, object); + assertThat(p.nextToken(), equalTo(XContentParser.Token.START_OBJECT)); + XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withFiltering( + null, + null, + Set.of("path.filter.field"), + true + ); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.humanReadable(true); + XContentDataHelper.decodeAndWriteXContent(parserConfig, builder, XContentType.JSON, XContentDataHelper.encodeToken(p)); + assertEquals(filterObject, Strings.toString(builder)); + + XContentBuilder builder2 = XContentFactory.jsonBuilder(); + builder2.humanReadable(true); + XContentDataHelper.decodeAndWriteXContent( + parserConfig, + builder2, + XContentType.JSON, + XContentDataHelper.encodeXContentBuilder(builder) + ); + assertEquals(filterObject, Strings.toString(builder2)); + } + + public void testObjectWithFilterRootPath() throws IOException { + String object = "{\"name\":\"foo\",\"path\":{\"filter\":{\"keep\":[0],\"field\":\"value\"}}}"; + String filterObject = "{\"path\":{\"filter\":{\"keep\":[0]}}}"; + + XContentParser p = createParser(JsonXContent.jsonXContent, object); + assertThat(p.nextToken(), equalTo(XContentParser.Token.START_OBJECT)); + XContentParserConfiguration parserConfig = XContentParserConfiguration.EMPTY.withFiltering( + "root.obj.sub_obj", + Set.of("root.obj.sub_obj.path"), + Set.of("root.obj.sub_obj.path.filter.field"), + true + ); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.humanReadable(true); + XContentDataHelper.decodeAndWriteXContent(parserConfig, builder, XContentType.JSON, XContentDataHelper.encodeToken(p)); + assertEquals(filterObject, Strings.toString(builder)); + + XContentBuilder builder2 = XContentFactory.jsonBuilder(); + builder2.humanReadable(true); + XContentDataHelper.decodeAndWriteXContent( + parserConfig, + builder2, + XContentType.JSON, + XContentDataHelper.encodeXContentBuilder(builder) + ); + assertEquals(filterObject, Strings.toString(builder2)); + } + public void testArrayInt() throws IOException { String values = "[" + String.join(",", List.of(Integer.toString(randomInt()), Integer.toString(randomInt()), Integer.toString(randomInt()))) @@ -252,7 +315,7 @@ private Map executeWriteMergedOnTwoEncodedValues(Object first, O var destination = XContentFactory.contentBuilder(xContentType); destination.startObject(); - XContentDataHelper.writeMerged(destination, "foo", List.of(firstEncoded, secondEncoded)); + XContentDataHelper.writeMerged(XContentParserConfiguration.EMPTY, destination, "foo", List.of(firstEncoded, secondEncoded)); destination.endObject(); return XContentHelper.convertToMap(BytesReference.bytes(destination), false, xContentType).v2(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 66d87f3532cbd..b9356bc4b5633 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -70,6 +70,7 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.internal.SubSearchContext; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.lookup.SourceProvider; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.BucketedSort.ExtraData; @@ -798,6 +799,14 @@ protected RandomIndexWriter indexWriterForSyntheticSource(Directory directory) t } protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer build) throws IOException { + return syntheticSource(mapper, null, build); + } + + protected final String syntheticSource( + DocumentMapper mapper, + @Nullable SourceFilter sourceFilter, + CheckedConsumer build + ) throws IOException { try (Directory directory = newDirectory()) { RandomIndexWriter iw = indexWriterForSyntheticSource(directory); ParsedDocument doc = mapper.parse(source(build)); @@ -806,9 +815,10 @@ protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer mapper.mapping().syntheticFieldLoader(filter), + SourceFieldMetrics.NOOP + ); var sourceLeafLoader = sourceLoader.leaf(getOnlyLeafReader(reader), docIds); var storedFieldLoader = StoredFieldLoader.create(false, sourceLoader.requiredStoredFields()) .getLoader(leafReader.getContext(), docIds); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 7dcbbce9fa8e0..2da2c5a08c177 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -64,6 +64,7 @@ import org.elasticsearch.search.lookup.LeafStoredFieldsLookup; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.Source; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.lookup.SourceProvider; import org.elasticsearch.test.ListMatcher; import org.elasticsearch.xcontent.ToXContent; @@ -1169,6 +1170,11 @@ private void assertSyntheticSource(SyntheticSourceExample example) throws IOExce b.endObject(); })).documentMapper(); assertThat(syntheticSource(mapper, example::buildInput), equalTo(example.expected())); + assertThat( + syntheticSource(mapper, new SourceFilter(new String[] { "field" }, null), example::buildInput), + equalTo(example.expected()) + ); + assertThat(syntheticSource(mapper, new SourceFilter(null, new String[] { "field" }), example::buildInput), equalTo("{}")); } private void assertSyntheticSourceWithTranslogSnapshot(SyntheticSourceSupport support, boolean doIndexSort) throws IOException { @@ -1292,7 +1298,7 @@ public final void testSyntheticSourceMany() throws IOException { } try (DirectoryReader reader = DirectoryReader.open(directory)) { int i = 0; - SourceLoader loader = mapper.sourceMapper().newSourceLoader(mapper.mapping(), SourceFieldMetrics.NOOP); + SourceLoader loader = mapper.mappers().newSourceLoader(null, SourceFieldMetrics.NOOP); StoredFieldLoader storedFieldLoader = loader.requiredStoredFields().isEmpty() ? StoredFieldLoader.empty() : StoredFieldLoader.create(false, loader.requiredStoredFields()); @@ -1335,6 +1341,18 @@ public final void testSyntheticSourceInObject() throws IOException { syntheticSourceExample.buildInput(b); b.endObject(); }), equalTo("{\"obj\":" + syntheticSourceExample.expected() + "}")); + + assertThat(syntheticSource(mapper, new SourceFilter(new String[] { "obj.field" }, null), b -> { + b.startObject("obj"); + syntheticSourceExample.buildInput(b); + b.endObject(); + }), equalTo("{\"obj\":" + syntheticSourceExample.expected() + "}")); + + assertThat(syntheticSource(mapper, new SourceFilter(null, new String[] { "obj.field" }), b -> { + b.startObject("obj"); + syntheticSourceExample.buildInput(b); + b.endObject(); + }), equalTo("{}")); } public final void testSyntheticEmptyList() throws IOException { @@ -1465,7 +1483,7 @@ private void testBlockLoader(boolean syntheticSource, boolean columnReader) thro blockReaderSupport.syntheticSource ); } - var sourceLoader = mapper.mappingLookup().newSourceLoader(SourceFieldMetrics.NOOP); + var sourceLoader = mapper.mappingLookup().newSourceLoader(null, SourceFieldMetrics.NOOP); testBlockLoader(columnReader, example, blockReaderSupport, sourceLoader); } @@ -1592,7 +1610,8 @@ private void assertNoDocValueLoader(CheckedConsumer { + b.startObject("obj"); + syntheticSourceExample.buildInput(b); + b.endObject(); + }), equalTo("{\"obj\":" + syntheticSourceExample.expected() + "}")); + + assertThat(syntheticSource(mapper, new SourceFilter(null, new String[] { "obj.field" }), b -> { + b.startObject("obj"); + syntheticSourceExample.buildInput(b); + b.endObject(); + }), equalTo("{\"obj\":{}}")); + + assertThat(syntheticSource(mapper, new SourceFilter(null, new String[] { "obj" }), b -> { + b.startObject("obj"); + syntheticSourceExample.buildInput(b); + b.endObject(); + }), equalTo("{}")); } protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed) { diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java index 6618f3199debf..3035bb98a3a93 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java @@ -68,6 +68,7 @@ public class TransportPutRollupJobAction extends AcknowledgedTransportMasterNode private static final Logger LOGGER = LogManager.getLogger(TransportPutRollupJobAction.class); private static final XContentParserConfiguration PARSER_CONFIGURATION = XContentParserConfiguration.EMPTY.withFiltering( + null, Set.of("_doc._meta._rollup"), null, false