From e711269ca7fa4eb6b6fa9b108e5f4441bba93a69 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Tue, 20 May 2025 17:57:17 +0100 Subject: [PATCH 1/5] Refactor SourceProvider creation to consistently use MappingLookup This change updates the code to always create SourceProvider instances via MappingLookup, avoiding direct exposure to the underlying source format (synthetic or stored). It also aligns source filtering behavior between SourceProvider and SourceLoader, ensuring consistent application of filters. This change is needed to enable source filtering to occur earlier in the fetch phase, for example, when constructing a synthetic source. --- .../script/ScriptScoreBenchmark.java | 4 +- .../murmur3/Murmur3FieldMapperTests.java | 3 +- .../query/FilteredSearchExecutionContext.java | 6 +- .../index/query/SearchExecutionContext.java | 12 ++-- .../search/DefaultSearchContext.java | 5 +- .../search/fetch/FetchContext.java | 4 +- .../search/fetch/FetchPhase.java | 5 +- .../internal/FilteredSearchContext.java | 6 +- .../search/internal/SearchContext.java | 3 +- .../ConcurrentSegmentSourceProvider.java | 71 +++++++++++++++++++ .../search/lookup/SourceProvider.java | 23 ++---- .../search/rank/RankSearchContext.java | 4 +- .../mapper/CompositeRuntimeFieldTests.java | 2 +- .../index/mapper/DateFieldScriptTests.java | 6 +- .../index/mapper/DoubleFieldScriptTests.java | 6 +- .../index/mapper/IpFieldScriptTests.java | 6 +- .../index/mapper/LongFieldScriptTests.java | 6 +- .../mapper/PlaceHolderFieldMapperTests.java | 2 +- .../mapper/ProvidedIdFieldMapperTests.java | 2 +- .../index/mapper/RangeFieldMapperTests.java | 2 +- .../index/mapper/SourceFieldMetricsTests.java | 6 +- .../index/mapper/StringFieldScriptTests.java | 12 +++- .../index/mapper/TextFieldMapperTests.java | 2 +- .../vectors/DenseVectorFieldMapperTests.java | 3 +- .../search/fetch/StoredFieldsSpecTests.java | 9 ++- .../fetch/subphase/FieldFetcherTests.java | 9 ++- .../search/lookup/SourceProviderTests.java | 6 +- ...tScriptFieldDistanceFeatureQueryTests.java | 8 ++- ...gScriptFieldDistanceFeatureQueryTests.java | 8 ++- .../AbstractScriptFieldTypeTestCase.java | 6 +- .../index/mapper/MapperServiceTestCase.java | 2 +- .../index/mapper/MapperTestCase.java | 7 +- .../aggregations/AggregatorTestCase.java | 2 +- .../search/fetch/HighlighterTestCase.java | 2 +- .../elasticsearch/test/TestSearchContext.java | 6 +- .../planner/EsPhysicalOperationProviders.java | 2 +- .../mapper/OffsetSourceFieldMapperTests.java | 3 +- .../mapper/RankVectorsFieldMapperTests.java | 3 +- 38 files changed, 196 insertions(+), 78 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/search/lookup/ConcurrentSegmentSourceProvider.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java index e61171aeff027..b94af73a6b5f2 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java @@ -30,8 +30,10 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.index.mapper.SourceFieldMetrics; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.plugins.PluginsLoader; @@ -90,7 +92,7 @@ public class ScriptScoreBenchmark { private final SearchLookup lookup = new SearchLookup( fieldTypes::get, (mft, lookup, fdo) -> mft.fielddataBuilder(FieldDataContext.noRuntimeFields("benchmark")).build(fieldDataCache, breakerService), - SourceProvider.fromStoredFields() + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) ); @Param({ "expression", "metal", "painless_cast", "painless_def" }) diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index cd02c34a064b0..039f8c049e0c5 100644 --- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -107,7 +107,8 @@ protected void assertFetch(MapperService mapperService, String field, Object val ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); ParsedDocument doc = mapperService.documentMapper().parse(source); withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), ir -> { - Source s = SourceProvider.fromStoredFields().getSource(ir.leaves().get(0), 0); + Source s = SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) + .getSource(ir.leaves().get(0), 0); docValueFetcher.setNextReader(ir.leaves().get(0)); nativeFetcher.setNextReader(ir.leaves().get(0)); List fromDocValues = docValueFetcher.fetchValues(s, 0, new ArrayList<>()); diff --git a/server/src/main/java/org/elasticsearch/index/query/FilteredSearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/FilteredSearchExecutionContext.java index 8d53552a11bfe..5e26c6cbc99c8 100644 --- a/server/src/main/java/org/elasticsearch/index/query/FilteredSearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/FilteredSearchExecutionContext.java @@ -19,6 +19,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; @@ -39,6 +40,7 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.lookup.LeafFieldLookupProvider; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.lookup.SourceProvider; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -162,8 +164,8 @@ public boolean isSourceSynthetic() { } @Override - public SourceLoader newSourceLoader(boolean forceSyntheticSource) { - return in.newSourceLoader(forceSyntheticSource); + public SourceLoader newSourceLoader(@Nullable SourceFilter filter, boolean forceSyntheticSource) { + return in.newSourceLoader(filter, forceSyntheticSource); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index bf1ac90101d86..b2c0cdab8d16e 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSortConfig; @@ -57,6 +58,7 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.lookup.LeafFieldLookupProvider; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.lookup.SourceProvider; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -439,15 +441,15 @@ public boolean isSourceSynthetic() { /** * Build something to load source {@code _source}. */ - public SourceLoader newSourceLoader(boolean forceSyntheticSource) { + public SourceLoader newSourceLoader(@Nullable SourceFilter filter, boolean forceSyntheticSource) { if (forceSyntheticSource) { return new SourceLoader.Synthetic( - null, + filter, () -> mappingLookup.getMapping().syntheticFieldLoader(null), mapperMetrics.sourceFieldMetrics() ); } - return mappingLookup.newSourceLoader(null, mapperMetrics.sourceFieldMetrics()); + return mappingLookup.newSourceLoader(filter, mapperMetrics.sourceFieldMetrics()); } /** @@ -506,9 +508,7 @@ public SearchLookup lookup() { } public SourceProvider createSourceProvider() { - return isSourceSynthetic() - ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping(), null, mapperMetrics.sourceFieldMetrics()) - : SourceProvider.fromStoredFields(); + return SourceProvider.fromLookup(mappingLookup, null, mapperMetrics.sourceFieldMetrics()); } /** diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 983132ac9061a..10be76f1d9429 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -70,6 +70,7 @@ import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.ShardSearchContextId; import org.elasticsearch.search.internal.ShardSearchRequest; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.profile.Profilers; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext; @@ -943,8 +944,8 @@ public ReaderContext readerContext() { } @Override - public SourceLoader newSourceLoader() { - return searchExecutionContext.newSourceLoader(request.isForceSyntheticSource()); + public SourceLoader newSourceLoader(SourceFilter filter) { + return searchExecutionContext.newSourceLoader(filter, request.isForceSyntheticSource()); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java index 0bbbff3a5d5f4..4801c53ec0f1e 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java @@ -45,9 +45,9 @@ public class FetchContext { /** * Create a FetchContext based on a SearchContext */ - public FetchContext(SearchContext searchContext) { + public FetchContext(SearchContext searchContext, SourceLoader sourceLoader) { this.searchContext = searchContext; - this.sourceLoader = searchContext.newSourceLoader(); + this.sourceLoader = sourceLoader; this.storedFieldsContext = buildStoredFieldsContext(searchContext); this.fetchSourceContext = buildFetchSourceContext(searchContext); } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index e61450fb7019a..d9a8ee72b47c0 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -111,9 +111,8 @@ public Source getSource(LeafReaderContext ctx, int doc) { } private SearchHits buildSearchHits(SearchContext context, int[] docIdsToLoad, Profiler profiler, RankDocShardInfo rankDocs) { - - FetchContext fetchContext = new FetchContext(context); - SourceLoader sourceLoader = context.newSourceLoader(); + SourceLoader sourceLoader = context.newSourceLoader(null); + FetchContext fetchContext = new FetchContext(context, sourceLoader); PreloadedSourceProvider sourceProvider = new PreloadedSourceProvider(); PreloadedFieldLookupProvider fieldLookupProvider = new PreloadedFieldLookupProvider(); diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java index 3ed2bf0fff5c0..04368b7414aeb 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java @@ -14,6 +14,7 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.mapper.IdLoader; @@ -33,6 +34,7 @@ import org.elasticsearch.search.fetch.subphase.InnerHitsContext; import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext; import org.elasticsearch.search.fetch.subphase.highlight.SearchHighlightContext; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.profile.Profilers; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext; @@ -453,8 +455,8 @@ public ReaderContext readerContext() { } @Override - public SourceLoader newSourceLoader() { - return in.newSourceLoader(); + public SourceLoader newSourceLoader(@Nullable SourceFilter filter) { + return in.newSourceLoader(filter); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 9b1df285cd11c..580fb5efc7222 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -41,6 +41,7 @@ import org.elasticsearch.search.fetch.subphase.InnerHitsContext; import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext; import org.elasticsearch.search.fetch.subphase.highlight.SearchHighlightContext; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.profile.Profilers; import org.elasticsearch.search.query.QueryPhase; import org.elasticsearch.search.query.QuerySearchResult; @@ -441,7 +442,7 @@ public String toString() { /** * Build something to load source {@code _source}. */ - public abstract SourceLoader newSourceLoader(); + public abstract SourceLoader newSourceLoader(@Nullable SourceFilter sourceFilter); public abstract IdLoader newIdLoader(); } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/ConcurrentSegmentSourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/ConcurrentSegmentSourceProvider.java new file mode 100644 index 0000000000000..72ab4b79caeeb --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/lookup/ConcurrentSegmentSourceProvider.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.search.lookup; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; +import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; +import org.elasticsearch.index.mapper.SourceLoader; + +import java.io.IOException; +import java.util.Map; + +/** + * A {@link SourceProvider} that loads _source from a concurrent search. + * + * NOTE: This is written under the assumption that individual segments are accessed by a single + * thread, even if separate segments may be searched concurrently. If we ever implement + * within-segment concurrency this will have to work entirely differently. + * **/ +class ConcurrentSegmentSourceProvider implements SourceProvider { + private final SourceLoader sourceLoader; + private final StoredFieldLoader storedFieldLoader; + private final Map leaves = ConcurrentCollections.newConcurrentMap(); + + ConcurrentSegmentSourceProvider(SourceLoader loader, boolean loadSource) { + this.sourceLoader = loader; + this.storedFieldLoader = StoredFieldLoader.create(loadSource, sourceLoader.requiredStoredFields()); + } + + @Override + public Source getSource(LeafReaderContext ctx, int doc) throws IOException { + final Object id = ctx.id(); + var leaf = leaves.get(id); + if (leaf == null) { + leaf = new Leaf(sourceLoader.leaf(ctx.reader(), null), storedFieldLoader.getLoader(ctx, null)); + var existing = leaves.put(id, leaf); + assert existing == null : "unexpected source provider [" + existing + "]"; + } + return leaf.getSource(ctx, doc); + } + + private static class Leaf implements SourceProvider { + private final SourceLoader.Leaf sourceLoader; + private final LeafStoredFieldLoader storedFieldLoader; + int doc = -1; + Source source = null; + + private Leaf(SourceLoader.Leaf sourceLoader, LeafStoredFieldLoader storedFieldLoader) { + this.sourceLoader = sourceLoader; + this.storedFieldLoader = storedFieldLoader; + } + + @Override + public Source getSource(LeafReaderContext ctx, int doc) throws IOException { + if (this.doc == doc) { + return source; + } + this.doc = doc; + storedFieldLoader.advanceTo(doc); + return source = sourceLoader.source(storedFieldLoader, doc); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java index 4696ef2299fd7..954930a9437c9 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java @@ -10,10 +10,8 @@ package org.elasticsearch.search.lookup; import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; -import org.elasticsearch.index.mapper.Mapping; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.SourceFieldMetrics; -import org.elasticsearch.index.mapper.SourceLoader; import java.io.IOException; @@ -28,27 +26,14 @@ public interface SourceProvider { Source getSource(LeafReaderContext ctx, int doc) throws IOException; /** - * A SourceProvider that loads source from stored fields + * A SourceProvider that delegate loading source to the provided {@link MappingLookup}. * * The returned SourceProvider is thread-safe across segments, in that it may be * safely used by a searcher that searches different segments on different threads, * but it is not safe to use this to access documents from the same segment across * multiple threads. */ - static SourceProvider fromStoredFields() { - StoredFieldLoader storedFieldLoader = StoredFieldLoader.sequentialSource(); - return new StoredFieldSourceProvider(storedFieldLoader); - } - - /** - * A SourceProvider that loads source from synthetic source - * - * The returned SourceProvider is thread-safe across segments, in that it may be - * safely used by a searcher that searches different segments on different threads, - * but it is not safe to use this to access documents from the same segment across - * multiple threads. - */ - static SourceProvider fromSyntheticSource(Mapping mapping, SourceFilter filter, SourceFieldMetrics metrics) { - return new SyntheticSourceProvider(new SourceLoader.Synthetic(filter, () -> mapping.syntheticFieldLoader(filter), metrics)); + static SourceProvider fromLookup(MappingLookup lookup, SourceFilter filter, SourceFieldMetrics metrics) { + return new ConcurrentSegmentSourceProvider(lookup.newSourceLoader(filter, metrics), lookup.isSourceSynthetic() == false); } } diff --git a/server/src/main/java/org/elasticsearch/search/rank/RankSearchContext.java b/server/src/main/java/org/elasticsearch/search/rank/RankSearchContext.java index 57a9b9282023f..c8c6dc942c148 100644 --- a/server/src/main/java/org/elasticsearch/search/rank/RankSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/rank/RankSearchContext.java @@ -14,6 +14,7 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.mapper.IdLoader; @@ -41,6 +42,7 @@ import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.ShardSearchContextId; import org.elasticsearch.search.internal.ShardSearchRequest; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.profile.Profilers; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext; @@ -536,7 +538,7 @@ public ReaderContext readerContext() { } @Override - public SourceLoader newSourceLoader() { + public SourceLoader newSourceLoader(@Nullable SourceFilter filter) { throw new UnsupportedOperationException(); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java index 723980721b76b..7b356b68abf4a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java @@ -346,7 +346,7 @@ public void testParseDocumentSubFieldAccess() throws IOException { (mft, lookupSupplier, fdo) -> mft.fielddataBuilder( new FieldDataContext("test", null, lookupSupplier, mapperService.mappingLookup()::sourcePaths, fdo) ).build(null, null), - SourceProvider.fromStoredFields() + SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) ); LeafSearchLookup leafSearchLookup = searchLookup.getLeafSearchLookup(reader.leaves().get(0)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java index d443fee5e9d15..b3b166690fc15 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java @@ -106,7 +106,11 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { DateFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup, fdt) -> null, SourceProvider.fromStoredFields()), + new SearchLookup( + field -> null, + (ft, lookup, fdt) -> null, + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) + ), DateFormatter.forPattern("epoch_millis"), OnScriptError.FAIL ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java index f23c5e608aba1..48f5db4fd7e25 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java @@ -103,7 +103,11 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { DoubleFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup, fdt) -> null, SourceProvider.fromStoredFields()), + new SearchLookup( + field -> null, + (ft, lookup, fdt) -> null, + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) + ), OnScriptError.FAIL ); DoubleFieldScript doubleFieldScript = leafFactory.newInstance(reader.leaves().get(0)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java index e6c78f506717e..0226aee272787 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java @@ -104,7 +104,11 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { IpFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup, fdt) -> null, SourceProvider.fromStoredFields()), + new SearchLookup( + field -> null, + (ft, lookup, fdt) -> null, + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) + ), OnScriptError.FAIL ); IpFieldScript ipFieldScript = leafFactory.newInstance(reader.leaves().get(0)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java index c6ad7c40e9a75..6d8363d97c734 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java @@ -103,7 +103,11 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { LongFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup, fdt) -> null, SourceProvider.fromStoredFields()), + new SearchLookup( + field -> null, + (ft, lookup, fdt) -> null, + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) + ), OnScriptError.FAIL ); LongFieldScript longFieldScript = leafFactory.newInstance(reader.leaves().get(0)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java index e9c75772637b4..800a428596b36 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java @@ -67,7 +67,7 @@ public void testFetchValue() throws Exception { SearchLookup lookup = new SearchLookup( mapperService::fieldType, fieldDataLookup(mapperService), - SourceProvider.fromStoredFields() + SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); FieldFetcher fieldFetcher = FieldFetcher.create( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java index 453ac0620e760..93846149b6daa 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java @@ -77,7 +77,7 @@ public void testFetchIdFieldValue() throws IOException { SearchLookup lookup = new SearchLookup( mapperService::fieldType, fieldDataLookup(mapperService), - SourceProvider.fromStoredFields() + SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(searchExecutionContext.lookup()).thenReturn(lookup); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index c36a126479e87..7555d1cd7fb48 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -408,7 +408,7 @@ protected Source getSourceFor(CheckedConsumer mapp iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { - SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), null, SourceFieldMetrics.NOOP); + SourceProvider provider = SourceProvider.fromLookup(mapper.mappers(), null, SourceFieldMetrics.NOOP); Source syntheticSource = provider.getSource(getOnlyLeafReader(reader).getContext(), 0); return syntheticSource; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java index ea9f8f6ae28a7..ac4842f0fc6a5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java @@ -45,11 +45,7 @@ public void testSyntheticSourceLoadLatency() throws IOException { iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { - SourceProvider provider = SourceProvider.fromSyntheticSource( - mapper.mapping(), - null, - createTestMapperMetrics().sourceFieldMetrics() - ); + SourceProvider provider = SourceProvider.fromLookup(mapper.mappers(), null, createTestMapperMetrics().sourceFieldMetrics()); Source synthetic = provider.getSource(getOnlyLeafReader(reader).getContext(), 0); assertEquals(synthetic.source().get("kwd"), "foo"); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java index 36f71c74838e6..7ad33b59122ae 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java @@ -134,7 +134,11 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { StringFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup, fdt) -> null, SourceProvider.fromStoredFields()), + new SearchLookup( + field -> null, + (ft, lookup, fdt) -> null, + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) + ), OnScriptError.FAIL ); StringFieldScript stringFieldScript = leafFactory.newInstance(reader.leaves().get(0)); @@ -164,7 +168,11 @@ public final void testFromSourceDoesNotEnforceCharsLimit() throws IOException { StringFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup, fdt) -> null, SourceProvider.fromStoredFields()), + new SearchLookup( + field -> null, + (ft, lookup, fdt) -> null, + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) + ), OnScriptError.FAIL ); StringFieldScript stringFieldScript = leafFactory.newInstance(reader.leaves().get(0)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index f74af767e9410..4670738c1d210 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1300,7 +1300,7 @@ public void testEmpty() throws Exception { MappedFieldType ft = mapperService.fieldType("field"); SourceProvider sourceProvider = mapperService.mappingLookup().isSourceSynthetic() ? (ctx, doc) -> { throw new IllegalArgumentException("Can't load source in scripts in synthetic mode"); - } : SourceProvider.fromStoredFields(); + } : SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()); SearchLookup searchLookup = new SearchLookup(null, null, sourceProvider); IndexFieldData sfd = ft.fielddataBuilder( new FieldDataContext("", null, () -> searchLookup, Set::of, MappedFieldType.FielddataOperation.SCRIPT) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index c1c21ccda580a..1ce916533a2c9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -1760,7 +1760,8 @@ protected void assertFetch(MapperService mapperService, String field, Object val ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); ParsedDocument doc = mapperService.documentMapper().parse(source); withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), ir -> { - Source s = SourceProvider.fromStoredFields().getSource(ir.leaves().get(0), 0); + Source s = SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) + .getSource(ir.leaves().get(0), 0); nativeFetcher.setNextReader(ir.leaves().get(0)); List fromNative = nativeFetcher.fetchValues(s, 0, new ArrayList<>()); DenseVectorFieldType denseVectorFieldType = (DenseVectorFieldType) ft; diff --git a/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java b/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java index 37fd33cf302b2..7cbef28bd9532 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java @@ -29,8 +29,9 @@ public class StoredFieldsSpecTests extends ESTestCase { public void testDefaults() { SearchSourceBuilder search = new SearchSourceBuilder(); + var context = searchContext(search); // defaults - return source and metadata fields - FetchContext fc = new FetchContext(searchContext(search)); + FetchContext fc = new FetchContext(context, context.newSourceLoader(null)); FetchSubPhaseProcessor sourceProcessor = new FetchSourcePhase().getProcessor(fc); assertNotNull(sourceProcessor); @@ -52,7 +53,8 @@ public void testDefaults() { public void testStoredFieldsDisabled() { SearchSourceBuilder search = new SearchSourceBuilder(); search.storedField("_none_"); - FetchContext fc = new FetchContext(searchContext(search)); + var context = searchContext(search); + FetchContext fc = new FetchContext(context, context.newSourceLoader(null)); assertNull(new StoredFieldsPhase().getProcessor(fc)); assertNull(new FetchSourcePhase().getProcessor(fc)); @@ -61,7 +63,8 @@ public void testStoredFieldsDisabled() { public void testScriptFieldsEnableMetadata() { SearchSourceBuilder search = new SearchSourceBuilder(); search.scriptField("field", new Script("script")); - FetchContext fc = new FetchContext(searchContext(search)); + var context = searchContext(search); + FetchContext fc = new FetchContext(context, null); FetchSubPhaseProcessor subPhaseProcessor = new ScriptFieldsPhase().getProcessor(fc); assertNotNull(subPhaseProcessor); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index c5f1efe561c22..5f26b046e0c8e 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -253,7 +253,8 @@ public void testMetadataFields() throws IOException { LeafReaderContext readerContext = searcher.getIndexReader().leaves().get(0); fieldFetcher.setNextReader(readerContext); - Source s = SourceProvider.fromStoredFields().getSource(readerContext, 0); + Source s = SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) + .getSource(readerContext, 0); Map fetchedFields = fieldFetcher.fetch(s, 0); assertThat(fetchedFields.size(), equalTo(5)); @@ -1539,7 +1540,11 @@ public void testFetchRuntimeFieldWithSourceDisabled() throws IOException { IndexSearcher searcher = newSearcher(iw); LeafReaderContext readerContext = searcher.getIndexReader().leaves().get(0); fieldFetcher.setNextReader(readerContext); - Source source = SourceProvider.fromStoredFields().getSource(readerContext, 0); + Source source = SourceProvider.fromLookup( + mapperService.mappingLookup(), + null, + mapperService.getMapperMetrics().sourceFieldMetrics() + ).getSource(readerContext, 0); Map fields = fieldFetcher.fetch(source, 0); assertEquals(1, fields.size()); DocumentField field = fields.get("runtime_field"); diff --git a/server/src/test/java/org/elasticsearch/search/lookup/SourceProviderTests.java b/server/src/test/java/org/elasticsearch/search/lookup/SourceProviderTests.java index b7ff79f7abd30..a899be8b070c2 100644 --- a/server/src/test/java/org/elasticsearch/search/lookup/SourceProviderTests.java +++ b/server/src/test/java/org/elasticsearch/search/lookup/SourceProviderTests.java @@ -27,6 +27,8 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.MappingLookup; +import org.elasticsearch.index.mapper.SourceFieldMetrics; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -46,7 +48,7 @@ public void testStoredFieldsSourceProvider() throws IOException { try (IndexReader reader = iw.getReader()) { LeafReaderContext readerContext = reader.leaves().get(0); - SourceProvider sourceProvider = SourceProvider.fromStoredFields(); + SourceProvider sourceProvider = SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP); Source source = sourceProvider.getSource(readerContext, 0); assertNotNull(source.internalSourceRef()); @@ -121,7 +123,7 @@ public ScoreMode scoreMode() { } private static CollectorManager assertingCollectorManager() { - SourceProvider sourceProvider = SourceProvider.fromStoredFields(); + SourceProvider sourceProvider = SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP); return new CollectorManager<>() { @Override public SourceAssertingCollector newCollector() { diff --git a/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java b/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java index d1fa8613ea393..cd8188689a760 100644 --- a/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java @@ -26,7 +26,9 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.index.mapper.GeoPointScriptFieldType; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.OnScriptError; +import org.elasticsearch.index.mapper.SourceFieldMetrics; import org.elasticsearch.script.AbstractLongFieldScript; import org.elasticsearch.script.GeoPointFieldScript; import org.elasticsearch.script.Script; @@ -93,7 +95,11 @@ public void testMatches() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - SearchLookup searchLookup = new SearchLookup(null, null, SourceProvider.fromStoredFields()); + SearchLookup searchLookup = new SearchLookup( + null, + null, + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) + ); Function leafFactory = ctx -> new GeoPointFieldScript( "test", Map.of(), diff --git a/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java b/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java index 0627a88284ac0..ac0a111cb0a52 100644 --- a/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java @@ -18,7 +18,9 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.OnScriptError; +import org.elasticsearch.index.mapper.SourceFieldMetrics; import org.elasticsearch.script.AbstractLongFieldScript; import org.elasticsearch.script.DateFieldScript; import org.elasticsearch.script.Script; @@ -71,7 +73,11 @@ public void testMatches() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181351]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - SearchLookup searchLookup = new SearchLookup(null, null, SourceProvider.fromStoredFields()); + SearchLookup searchLookup = new SearchLookup( + null, + null, + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) + ); Function leafFactory = ctx -> new DateFieldScript( "test", Map.of(), diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java index 47a227cebc956..1c785d58f9804 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java @@ -287,7 +287,11 @@ protected boolean supportsRangeQueries() { } protected static SearchExecutionContext mockContext(boolean allowExpensiveQueries, MappedFieldType mappedFieldType) { - return mockContext(allowExpensiveQueries, mappedFieldType, SourceProvider.fromStoredFields()); + return mockContext( + allowExpensiveQueries, + mappedFieldType, + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) + ); } protected static SearchExecutionContext mockContext( diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index ddcb3e6a42d80..4a1d33595eb79 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -858,7 +858,7 @@ protected static String syntheticSource(DocumentMapper mapper, SourceFilter filt final String synthetic1; final XContent xContent; { - SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), filter, SourceFieldMetrics.NOOP); + SourceProvider provider = SourceProvider.fromLookup(mapper.mappers(), filter, SourceFieldMetrics.NOOP); var source = provider.getSource(leafReader.getContext(), docId); synthetic1 = source.internalSourceRef().utf8ToString(); xContent = source.sourceContentType().xContent(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 52bf6484de80c..7e127ba307942 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -545,7 +545,7 @@ protected final List fetchFromDocValues(MapperService mapperService, MappedFi SearchLookup lookup = new SearchLookup( mapperService::fieldType, fieldDataLookup(mapperService), - SourceProvider.fromStoredFields() + SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) ); ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.getForField(ft, MappedFieldType.FielddataOperation.SEARCH)); IndexSearcher searcher = newSearcher(iw); @@ -566,7 +566,7 @@ protected static void assertScriptDocValues(MapperService mapperService, Object MappedFieldType ft = mapperService.fieldType("field"); SourceProvider sourceProvider = mapperService.mappingLookup().isSourceSynthetic() ? (ctx, doc) -> { throw new IllegalArgumentException("Can't load source in scripts in synthetic mode"); - } : SourceProvider.fromStoredFields(); + } : SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()); SearchLookup searchLookup = new SearchLookup(null, null, sourceProvider); IndexFieldData sfd = ft.fielddataBuilder( new FieldDataContext( @@ -865,7 +865,8 @@ protected void assertFetch(MapperService mapperService, String field, Object val ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); ParsedDocument doc = mapperService.documentMapper().parse(source); withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), ir -> { - Source s = SourceProvider.fromStoredFields().getSource(ir.leaves().get(0), 0); + Source s = SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) + .getSource(ir.leaves().get(0), 0); docValueFetcher.setNextReader(ir.leaves().get(0)); nativeFetcher.setNextReader(ir.leaves().get(0)); List fromDocValues = docValueFetcher.fetchValues(s, 0, new ArrayList<>()); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index e0c4be506969e..2559966ba5e7e 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -523,7 +523,7 @@ private SubSearchContext buildSubSearchContext( when(indexShard.shardId()).thenReturn(new ShardId("test", "test", 0)); when(indexShard.indexSettings()).thenReturn(indexSettings); when(ctx.indexShard()).thenReturn(indexShard); - when(ctx.newSourceLoader()).thenAnswer(inv -> searchExecutionContext.newSourceLoader(false)); + when(ctx.newSourceLoader(null)).thenAnswer(inv -> searchExecutionContext.newSourceLoader(null, false)); when(ctx.newIdLoader()).thenReturn(IdLoader.fromLeafStoredFieldLoader()); var res = new SubSearchContext(ctx); releasables.add(res); // TODO: nasty workaround for not getting the standard resource handling behavior of a real search context diff --git a/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java index 5c629075e6475..72417f5e61b27 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java @@ -114,7 +114,7 @@ private static FetchContext fetchContext(SearchExecutionContext context, SearchS when(fetchContext.highlight()).thenReturn(search.highlighter().build(context)); when(fetchContext.parsedQuery()).thenReturn(new ParsedQuery(search.query().toQuery(context))); when(fetchContext.getSearchExecutionContext()).thenReturn(context); - when(fetchContext.sourceLoader()).thenReturn(context.newSourceLoader(false)); + when(fetchContext.sourceLoader()).thenReturn(context.newSourceLoader(null, false)); return fetchContext; } diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index bf612ebc70bc4..8ac52c3d48a0d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -13,6 +13,7 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; @@ -42,6 +43,7 @@ import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.ShardSearchContextId; import org.elasticsearch.search.internal.ShardSearchRequest; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.profile.Profilers; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext; @@ -552,8 +554,8 @@ public ReaderContext readerContext() { } @Override - public SourceLoader newSourceLoader() { - return searchExecutionContext.newSourceLoader(false); + public SourceLoader newSourceLoader(@Nullable SourceFilter filter) { + return searchExecutionContext.newSourceLoader(filter, false); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 31a859d0c1951..ef0b9aa931267 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -380,7 +380,7 @@ public String shardIdentifier() { @Override public SourceLoader newSourceLoader() { - return ctx.newSourceLoader(false); + return ctx.newSourceLoader(null, false); } @Override diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java index 47c4c8b23f42d..b3ee52c8dede5 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java @@ -167,7 +167,8 @@ protected void assertFetch(MapperService mapperService, String field, Object val ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); ParsedDocument doc = mapperService.documentMapper().parse(source); withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), ir -> { - Source s = SourceProvider.fromStoredFields().getSource(ir.leaves().get(0), 0); + Source s = SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) + .getSource(ir.leaves().get(0), 0); nativeFetcher.setNextReader(ir.leaves().get(0)); List fromNative = nativeFetcher.fetchValues(s, 0, new ArrayList<>()); assertThat(fromNative.size(), equalTo(1)); diff --git a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java index ac7ed8cf1c07f..69f95ae4bf52f 100644 --- a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java @@ -376,7 +376,8 @@ protected void assertFetch(MapperService mapperService, String field, Object val ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); ParsedDocument doc = mapperService.documentMapper().parse(source); withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), ir -> { - Source s = SourceProvider.fromStoredFields().getSource(ir.leaves().get(0), 0); + Source s = SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) + .getSource(ir.leaves().get(0), 0); nativeFetcher.setNextReader(ir.leaves().get(0)); List fromNative = nativeFetcher.fetchValues(s, 0, new ArrayList<>()); RankVectorsFieldMapper.RankVectorsFieldType denseVectorFieldType = (RankVectorsFieldMapper.RankVectorsFieldType) ft; From 7e549d548719f19b0ccb27c8757b15a33c679d9d Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Tue, 20 May 2025 18:00:01 +0100 Subject: [PATCH 2/5] Update docs/changelog/128213.yaml --- docs/changelog/128213.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/128213.yaml diff --git a/docs/changelog/128213.yaml b/docs/changelog/128213.yaml new file mode 100644 index 0000000000000..a1f108cbbec8e --- /dev/null +++ b/docs/changelog/128213.yaml @@ -0,0 +1,5 @@ +pr: 128213 +summary: Refactor `SourceProvider` creation to consistently use `MappingLookup` +area: Mapping +type: enhancement +issues: [] From e72210603b0accc959df8ebad7da8dfc87eaff8c Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Wed, 21 May 2025 09:40:32 +0100 Subject: [PATCH 3/5] address review comment --- .../lookup/StoredFieldSourceProvider.java | 63 ------------------- .../lookup/SyntheticSourceProvider.java | 61 ------------------ 2 files changed, 124 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/search/lookup/StoredFieldSourceProvider.java delete mode 100644 server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java diff --git a/server/src/main/java/org/elasticsearch/search/lookup/StoredFieldSourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/StoredFieldSourceProvider.java deleted file mode 100644 index aa3caa6865bdb..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/lookup/StoredFieldSourceProvider.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.search.lookup; - -import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; -import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; -import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; - -import java.io.IOException; -import java.util.Map; - -// NB This is written under the assumption that individual segments are accessed by a single -// thread, even if separate segments may be searched concurrently. If we ever implement -// within-segment concurrency this will have to work entirely differently. -class StoredFieldSourceProvider implements SourceProvider { - - private final StoredFieldLoader storedFieldLoader; - private final Map leaves = ConcurrentCollections.newConcurrentMap(); - - StoredFieldSourceProvider(StoredFieldLoader storedFieldLoader) { - this.storedFieldLoader = storedFieldLoader; - } - - @Override - public Source getSource(LeafReaderContext ctx, int doc) throws IOException { - final Object id = ctx.id(); - var provider = leaves.get(id); - if (provider == null) { - provider = new LeafStoredFieldSourceProvider(storedFieldLoader.getLoader(ctx, null)); - var existing = leaves.put(id, provider); - assert existing == null : "unexpected source provider [" + existing + "]"; - } - return provider.getSource(doc); - } - - private static class LeafStoredFieldSourceProvider { - - final LeafStoredFieldLoader leafStoredFieldLoader; - int doc = -1; - Source source; - - private LeafStoredFieldSourceProvider(LeafStoredFieldLoader leafStoredFieldLoader) { - this.leafStoredFieldLoader = leafStoredFieldLoader; - } - - Source getSource(int doc) throws IOException { - if (this.doc == doc) { - return source; - } - this.doc = doc; - leafStoredFieldLoader.advanceTo(doc); - return source = Source.fromBytes(leafStoredFieldLoader.source()); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java deleted file mode 100644 index 8078f4cb9cb8e..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.search.lookup; - -import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; -import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; -import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; -import org.elasticsearch.index.mapper.SourceLoader; - -import java.io.IOException; -import java.util.Map; - -// NB This is written under the assumption that individual segments are accessed by a single -// thread, even if separate segments may be searched concurrently. If we ever implement -// within-segment concurrency this will have to work entirely differently. -class SyntheticSourceProvider implements SourceProvider { - - private final SourceLoader sourceLoader; - private final Map leaves = ConcurrentCollections.newConcurrentMap(); - - SyntheticSourceProvider(SourceLoader sourceLoader) { - this.sourceLoader = sourceLoader; - } - - @Override - public Source getSource(LeafReaderContext ctx, int doc) throws IOException { - final Object id = ctx.id(); - var provider = leaves.get(id); - if (provider == null) { - provider = new SyntheticSourceLeafLoader(ctx); - var existing = leaves.put(id, provider); - assert existing == null : "unexpected source provider [" + existing + "]"; - } - return provider.getSource(doc); - } - - private class SyntheticSourceLeafLoader { - private final LeafStoredFieldLoader leafLoader; - private final SourceLoader.Leaf leaf; - - SyntheticSourceLeafLoader(LeafReaderContext ctx) throws IOException { - this.leafLoader = (sourceLoader.requiredStoredFields().isEmpty()) - ? StoredFieldLoader.empty().getLoader(ctx, null) - : StoredFieldLoader.create(false, sourceLoader.requiredStoredFields()).getLoader(ctx, null); - this.leaf = sourceLoader.leaf(ctx.reader(), null); - } - - Source getSource(int doc) throws IOException { - leafLoader.advanceTo(doc); - return leaf.source(leafLoader, doc); - } - } -} From ec220d4005ed71d90fd310c771f25c795089193e Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Thu, 22 May 2025 09:40:58 +0100 Subject: [PATCH 4/5] iter --- .../elasticsearch/index/fieldvisitor/StoredFieldLoader.java | 3 +++ .../java/org/elasticsearch/search/DefaultSearchContext.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java index 52e9830037832..a02a8da9e629e 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java @@ -66,6 +66,9 @@ public static StoredFieldLoader create(boolean loadSource, Set fields) { * otherwise, uses the heuristic defined in {@link StoredFieldLoader#reader(LeafReaderContext, int[])}. */ public static StoredFieldLoader create(boolean loadSource, Set fields, boolean forceSequentialReader) { + if (loadSource == false && fields.isEmpty()) { + return StoredFieldLoader.empty(); + } List fieldsToLoad = fieldsToLoad(loadSource, fields); return new StoredFieldLoader() { @Override diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 10be76f1d9429..c2b526128a9bc 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -944,7 +944,7 @@ public ReaderContext readerContext() { } @Override - public SourceLoader newSourceLoader(SourceFilter filter) { + public SourceLoader newSourceLoader(@Nullable SourceFilter filter) { return searchExecutionContext.newSourceLoader(filter, request.isForceSyntheticSource()); } From b69e817a210fdb8c6027a30860bf9e2f85b254ec Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Thu, 22 May 2025 13:21:42 +0100 Subject: [PATCH 5/5] source providers always force a sequential reader to load the fields --- .../search/lookup/ConcurrentSegmentSourceProvider.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/lookup/ConcurrentSegmentSourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/ConcurrentSegmentSourceProvider.java index 72ab4b79caeeb..a390168c12fc3 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/ConcurrentSegmentSourceProvider.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/ConcurrentSegmentSourceProvider.java @@ -32,7 +32,8 @@ class ConcurrentSegmentSourceProvider implements SourceProvider { ConcurrentSegmentSourceProvider(SourceLoader loader, boolean loadSource) { this.sourceLoader = loader; - this.storedFieldLoader = StoredFieldLoader.create(loadSource, sourceLoader.requiredStoredFields()); + // we force a sequential reader here since it is used during query execution where documents are scanned sequentially + this.storedFieldLoader = StoredFieldLoader.create(loadSource, sourceLoader.requiredStoredFields(), true); } @Override