diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java index e61171aeff027..b94af73a6b5f2 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java @@ -30,8 +30,10 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.index.mapper.SourceFieldMetrics; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.plugins.PluginsLoader; @@ -90,7 +92,7 @@ public class ScriptScoreBenchmark { private final SearchLookup lookup = new SearchLookup( fieldTypes::get, (mft, lookup, fdo) -> mft.fielddataBuilder(FieldDataContext.noRuntimeFields("benchmark")).build(fieldDataCache, breakerService), - SourceProvider.fromStoredFields() + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) ); @Param({ "expression", "metal", "painless_cast", "painless_def" }) diff --git a/docs/changelog/128213.yaml b/docs/changelog/128213.yaml new file mode 100644 index 0000000000000..a1f108cbbec8e --- /dev/null +++ b/docs/changelog/128213.yaml @@ -0,0 +1,5 @@ +pr: 128213 +summary: Refactor `SourceProvider` creation to consistently use `MappingLookup` +area: Mapping +type: enhancement +issues: [] diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index cd02c34a064b0..039f8c049e0c5 100644 --- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -107,7 +107,8 @@ protected void assertFetch(MapperService mapperService, String field, Object val ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); ParsedDocument doc = mapperService.documentMapper().parse(source); withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), ir -> { - Source s = SourceProvider.fromStoredFields().getSource(ir.leaves().get(0), 0); + Source s = SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) + .getSource(ir.leaves().get(0), 0); docValueFetcher.setNextReader(ir.leaves().get(0)); nativeFetcher.setNextReader(ir.leaves().get(0)); List fromDocValues = docValueFetcher.fetchValues(s, 0, new ArrayList<>()); diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java index 52e9830037832..a02a8da9e629e 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java @@ -66,6 +66,9 @@ public static StoredFieldLoader create(boolean loadSource, Set fields) { * otherwise, uses the heuristic defined in {@link StoredFieldLoader#reader(LeafReaderContext, int[])}. */ public static StoredFieldLoader create(boolean loadSource, Set fields, boolean forceSequentialReader) { + if (loadSource == false && fields.isEmpty()) { + return StoredFieldLoader.empty(); + } List fieldsToLoad = fieldsToLoad(loadSource, fields); return new StoredFieldLoader() { @Override diff --git a/server/src/main/java/org/elasticsearch/index/query/FilteredSearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/FilteredSearchExecutionContext.java index 8d53552a11bfe..5e26c6cbc99c8 100644 --- a/server/src/main/java/org/elasticsearch/index/query/FilteredSearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/FilteredSearchExecutionContext.java @@ -19,6 +19,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; @@ -39,6 +40,7 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.lookup.LeafFieldLookupProvider; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.lookup.SourceProvider; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -162,8 +164,8 @@ public boolean isSourceSynthetic() { } @Override - public SourceLoader newSourceLoader(boolean forceSyntheticSource) { - return in.newSourceLoader(forceSyntheticSource); + public SourceLoader newSourceLoader(@Nullable SourceFilter filter, boolean forceSyntheticSource) { + return in.newSourceLoader(filter, forceSyntheticSource); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index bf1ac90101d86..b2c0cdab8d16e 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSortConfig; @@ -57,6 +58,7 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; import org.elasticsearch.search.lookup.LeafFieldLookupProvider; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.lookup.SourceProvider; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xcontent.XContentParserConfiguration; @@ -439,15 +441,15 @@ public boolean isSourceSynthetic() { /** * Build something to load source {@code _source}. */ - public SourceLoader newSourceLoader(boolean forceSyntheticSource) { + public SourceLoader newSourceLoader(@Nullable SourceFilter filter, boolean forceSyntheticSource) { if (forceSyntheticSource) { return new SourceLoader.Synthetic( - null, + filter, () -> mappingLookup.getMapping().syntheticFieldLoader(null), mapperMetrics.sourceFieldMetrics() ); } - return mappingLookup.newSourceLoader(null, mapperMetrics.sourceFieldMetrics()); + return mappingLookup.newSourceLoader(filter, mapperMetrics.sourceFieldMetrics()); } /** @@ -506,9 +508,7 @@ public SearchLookup lookup() { } public SourceProvider createSourceProvider() { - return isSourceSynthetic() - ? SourceProvider.fromSyntheticSource(mappingLookup.getMapping(), null, mapperMetrics.sourceFieldMetrics()) - : SourceProvider.fromStoredFields(); + return SourceProvider.fromLookup(mappingLookup, null, mapperMetrics.sourceFieldMetrics()); } /** diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 983132ac9061a..c2b526128a9bc 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -70,6 +70,7 @@ import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.ShardSearchContextId; import org.elasticsearch.search.internal.ShardSearchRequest; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.profile.Profilers; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext; @@ -943,8 +944,8 @@ public ReaderContext readerContext() { } @Override - public SourceLoader newSourceLoader() { - return searchExecutionContext.newSourceLoader(request.isForceSyntheticSource()); + public SourceLoader newSourceLoader(@Nullable SourceFilter filter) { + return searchExecutionContext.newSourceLoader(filter, request.isForceSyntheticSource()); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java index 0bbbff3a5d5f4..4801c53ec0f1e 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchContext.java @@ -45,9 +45,9 @@ public class FetchContext { /** * Create a FetchContext based on a SearchContext */ - public FetchContext(SearchContext searchContext) { + public FetchContext(SearchContext searchContext, SourceLoader sourceLoader) { this.searchContext = searchContext; - this.sourceLoader = searchContext.newSourceLoader(); + this.sourceLoader = sourceLoader; this.storedFieldsContext = buildStoredFieldsContext(searchContext); this.fetchSourceContext = buildFetchSourceContext(searchContext); } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index e61450fb7019a..d9a8ee72b47c0 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -111,9 +111,8 @@ public Source getSource(LeafReaderContext ctx, int doc) { } private SearchHits buildSearchHits(SearchContext context, int[] docIdsToLoad, Profiler profiler, RankDocShardInfo rankDocs) { - - FetchContext fetchContext = new FetchContext(context); - SourceLoader sourceLoader = context.newSourceLoader(); + SourceLoader sourceLoader = context.newSourceLoader(null); + FetchContext fetchContext = new FetchContext(context, sourceLoader); PreloadedSourceProvider sourceProvider = new PreloadedSourceProvider(); PreloadedFieldLookupProvider fieldLookupProvider = new PreloadedFieldLookupProvider(); diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java index 3ed2bf0fff5c0..04368b7414aeb 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java @@ -14,6 +14,7 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.mapper.IdLoader; @@ -33,6 +34,7 @@ import org.elasticsearch.search.fetch.subphase.InnerHitsContext; import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext; import org.elasticsearch.search.fetch.subphase.highlight.SearchHighlightContext; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.profile.Profilers; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext; @@ -453,8 +455,8 @@ public ReaderContext readerContext() { } @Override - public SourceLoader newSourceLoader() { - return in.newSourceLoader(); + public SourceLoader newSourceLoader(@Nullable SourceFilter filter) { + return in.newSourceLoader(filter); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 9b1df285cd11c..580fb5efc7222 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -41,6 +41,7 @@ import org.elasticsearch.search.fetch.subphase.InnerHitsContext; import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext; import org.elasticsearch.search.fetch.subphase.highlight.SearchHighlightContext; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.profile.Profilers; import org.elasticsearch.search.query.QueryPhase; import org.elasticsearch.search.query.QuerySearchResult; @@ -441,7 +442,7 @@ public String toString() { /** * Build something to load source {@code _source}. */ - public abstract SourceLoader newSourceLoader(); + public abstract SourceLoader newSourceLoader(@Nullable SourceFilter sourceFilter); public abstract IdLoader newIdLoader(); } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/ConcurrentSegmentSourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/ConcurrentSegmentSourceProvider.java new file mode 100644 index 0000000000000..a390168c12fc3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/lookup/ConcurrentSegmentSourceProvider.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.search.lookup; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; +import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; +import org.elasticsearch.index.mapper.SourceLoader; + +import java.io.IOException; +import java.util.Map; + +/** + * A {@link SourceProvider} that loads _source from a concurrent search. + * + * NOTE: This is written under the assumption that individual segments are accessed by a single + * thread, even if separate segments may be searched concurrently. If we ever implement + * within-segment concurrency this will have to work entirely differently. + * **/ +class ConcurrentSegmentSourceProvider implements SourceProvider { + private final SourceLoader sourceLoader; + private final StoredFieldLoader storedFieldLoader; + private final Map leaves = ConcurrentCollections.newConcurrentMap(); + + ConcurrentSegmentSourceProvider(SourceLoader loader, boolean loadSource) { + this.sourceLoader = loader; + // we force a sequential reader here since it is used during query execution where documents are scanned sequentially + this.storedFieldLoader = StoredFieldLoader.create(loadSource, sourceLoader.requiredStoredFields(), true); + } + + @Override + public Source getSource(LeafReaderContext ctx, int doc) throws IOException { + final Object id = ctx.id(); + var leaf = leaves.get(id); + if (leaf == null) { + leaf = new Leaf(sourceLoader.leaf(ctx.reader(), null), storedFieldLoader.getLoader(ctx, null)); + var existing = leaves.put(id, leaf); + assert existing == null : "unexpected source provider [" + existing + "]"; + } + return leaf.getSource(ctx, doc); + } + + private static class Leaf implements SourceProvider { + private final SourceLoader.Leaf sourceLoader; + private final LeafStoredFieldLoader storedFieldLoader; + int doc = -1; + Source source = null; + + private Leaf(SourceLoader.Leaf sourceLoader, LeafStoredFieldLoader storedFieldLoader) { + this.sourceLoader = sourceLoader; + this.storedFieldLoader = storedFieldLoader; + } + + @Override + public Source getSource(LeafReaderContext ctx, int doc) throws IOException { + if (this.doc == doc) { + return source; + } + this.doc = doc; + storedFieldLoader.advanceTo(doc); + return source = sourceLoader.source(storedFieldLoader, doc); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java index 4696ef2299fd7..954930a9437c9 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceProvider.java @@ -10,10 +10,8 @@ package org.elasticsearch.search.lookup; import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; -import org.elasticsearch.index.mapper.Mapping; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.SourceFieldMetrics; -import org.elasticsearch.index.mapper.SourceLoader; import java.io.IOException; @@ -28,27 +26,14 @@ public interface SourceProvider { Source getSource(LeafReaderContext ctx, int doc) throws IOException; /** - * A SourceProvider that loads source from stored fields + * A SourceProvider that delegate loading source to the provided {@link MappingLookup}. * * The returned SourceProvider is thread-safe across segments, in that it may be * safely used by a searcher that searches different segments on different threads, * but it is not safe to use this to access documents from the same segment across * multiple threads. */ - static SourceProvider fromStoredFields() { - StoredFieldLoader storedFieldLoader = StoredFieldLoader.sequentialSource(); - return new StoredFieldSourceProvider(storedFieldLoader); - } - - /** - * A SourceProvider that loads source from synthetic source - * - * The returned SourceProvider is thread-safe across segments, in that it may be - * safely used by a searcher that searches different segments on different threads, - * but it is not safe to use this to access documents from the same segment across - * multiple threads. - */ - static SourceProvider fromSyntheticSource(Mapping mapping, SourceFilter filter, SourceFieldMetrics metrics) { - return new SyntheticSourceProvider(new SourceLoader.Synthetic(filter, () -> mapping.syntheticFieldLoader(filter), metrics)); + static SourceProvider fromLookup(MappingLookup lookup, SourceFilter filter, SourceFieldMetrics metrics) { + return new ConcurrentSegmentSourceProvider(lookup.newSourceLoader(filter, metrics), lookup.isSourceSynthetic() == false); } } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/StoredFieldSourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/StoredFieldSourceProvider.java deleted file mode 100644 index aa3caa6865bdb..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/lookup/StoredFieldSourceProvider.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.search.lookup; - -import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; -import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; -import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; - -import java.io.IOException; -import java.util.Map; - -// NB This is written under the assumption that individual segments are accessed by a single -// thread, even if separate segments may be searched concurrently. If we ever implement -// within-segment concurrency this will have to work entirely differently. -class StoredFieldSourceProvider implements SourceProvider { - - private final StoredFieldLoader storedFieldLoader; - private final Map leaves = ConcurrentCollections.newConcurrentMap(); - - StoredFieldSourceProvider(StoredFieldLoader storedFieldLoader) { - this.storedFieldLoader = storedFieldLoader; - } - - @Override - public Source getSource(LeafReaderContext ctx, int doc) throws IOException { - final Object id = ctx.id(); - var provider = leaves.get(id); - if (provider == null) { - provider = new LeafStoredFieldSourceProvider(storedFieldLoader.getLoader(ctx, null)); - var existing = leaves.put(id, provider); - assert existing == null : "unexpected source provider [" + existing + "]"; - } - return provider.getSource(doc); - } - - private static class LeafStoredFieldSourceProvider { - - final LeafStoredFieldLoader leafStoredFieldLoader; - int doc = -1; - Source source; - - private LeafStoredFieldSourceProvider(LeafStoredFieldLoader leafStoredFieldLoader) { - this.leafStoredFieldLoader = leafStoredFieldLoader; - } - - Source getSource(int doc) throws IOException { - if (this.doc == doc) { - return source; - } - this.doc = doc; - leafStoredFieldLoader.advanceTo(doc); - return source = Source.fromBytes(leafStoredFieldLoader.source()); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java b/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java deleted file mode 100644 index 8078f4cb9cb8e..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/lookup/SyntheticSourceProvider.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.search.lookup; - -import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; -import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; -import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; -import org.elasticsearch.index.mapper.SourceLoader; - -import java.io.IOException; -import java.util.Map; - -// NB This is written under the assumption that individual segments are accessed by a single -// thread, even if separate segments may be searched concurrently. If we ever implement -// within-segment concurrency this will have to work entirely differently. -class SyntheticSourceProvider implements SourceProvider { - - private final SourceLoader sourceLoader; - private final Map leaves = ConcurrentCollections.newConcurrentMap(); - - SyntheticSourceProvider(SourceLoader sourceLoader) { - this.sourceLoader = sourceLoader; - } - - @Override - public Source getSource(LeafReaderContext ctx, int doc) throws IOException { - final Object id = ctx.id(); - var provider = leaves.get(id); - if (provider == null) { - provider = new SyntheticSourceLeafLoader(ctx); - var existing = leaves.put(id, provider); - assert existing == null : "unexpected source provider [" + existing + "]"; - } - return provider.getSource(doc); - } - - private class SyntheticSourceLeafLoader { - private final LeafStoredFieldLoader leafLoader; - private final SourceLoader.Leaf leaf; - - SyntheticSourceLeafLoader(LeafReaderContext ctx) throws IOException { - this.leafLoader = (sourceLoader.requiredStoredFields().isEmpty()) - ? StoredFieldLoader.empty().getLoader(ctx, null) - : StoredFieldLoader.create(false, sourceLoader.requiredStoredFields()).getLoader(ctx, null); - this.leaf = sourceLoader.leaf(ctx.reader(), null); - } - - Source getSource(int doc) throws IOException { - leafLoader.advanceTo(doc); - return leaf.source(leafLoader, doc); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/search/rank/RankSearchContext.java b/server/src/main/java/org/elasticsearch/search/rank/RankSearchContext.java index 57a9b9282023f..c8c6dc942c148 100644 --- a/server/src/main/java/org/elasticsearch/search/rank/RankSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/rank/RankSearchContext.java @@ -14,6 +14,7 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.mapper.IdLoader; @@ -41,6 +42,7 @@ import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.ShardSearchContextId; import org.elasticsearch.search.internal.ShardSearchRequest; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.profile.Profilers; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext; @@ -536,7 +538,7 @@ public ReaderContext readerContext() { } @Override - public SourceLoader newSourceLoader() { + public SourceLoader newSourceLoader(@Nullable SourceFilter filter) { throw new UnsupportedOperationException(); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java index 723980721b76b..7b356b68abf4a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java @@ -346,7 +346,7 @@ public void testParseDocumentSubFieldAccess() throws IOException { (mft, lookupSupplier, fdo) -> mft.fielddataBuilder( new FieldDataContext("test", null, lookupSupplier, mapperService.mappingLookup()::sourcePaths, fdo) ).build(null, null), - SourceProvider.fromStoredFields() + SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) ); LeafSearchLookup leafSearchLookup = searchLookup.getLeafSearchLookup(reader.leaves().get(0)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java index d443fee5e9d15..b3b166690fc15 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java @@ -106,7 +106,11 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { DateFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup, fdt) -> null, SourceProvider.fromStoredFields()), + new SearchLookup( + field -> null, + (ft, lookup, fdt) -> null, + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) + ), DateFormatter.forPattern("epoch_millis"), OnScriptError.FAIL ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java index f23c5e608aba1..48f5db4fd7e25 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java @@ -103,7 +103,11 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { DoubleFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup, fdt) -> null, SourceProvider.fromStoredFields()), + new SearchLookup( + field -> null, + (ft, lookup, fdt) -> null, + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) + ), OnScriptError.FAIL ); DoubleFieldScript doubleFieldScript = leafFactory.newInstance(reader.leaves().get(0)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java index e6c78f506717e..0226aee272787 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java @@ -104,7 +104,11 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { IpFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup, fdt) -> null, SourceProvider.fromStoredFields()), + new SearchLookup( + field -> null, + (ft, lookup, fdt) -> null, + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) + ), OnScriptError.FAIL ); IpFieldScript ipFieldScript = leafFactory.newInstance(reader.leaves().get(0)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java index c6ad7c40e9a75..6d8363d97c734 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java @@ -103,7 +103,11 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { LongFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup, fdt) -> null, SourceProvider.fromStoredFields()), + new SearchLookup( + field -> null, + (ft, lookup, fdt) -> null, + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) + ), OnScriptError.FAIL ); LongFieldScript longFieldScript = leafFactory.newInstance(reader.leaves().get(0)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java index e9c75772637b4..800a428596b36 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java @@ -67,7 +67,7 @@ public void testFetchValue() throws Exception { SearchLookup lookup = new SearchLookup( mapperService::fieldType, fieldDataLookup(mapperService), - SourceProvider.fromStoredFields() + SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); FieldFetcher fieldFetcher = FieldFetcher.create( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java index 453ac0620e760..93846149b6daa 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java @@ -77,7 +77,7 @@ public void testFetchIdFieldValue() throws IOException { SearchLookup lookup = new SearchLookup( mapperService::fieldType, fieldDataLookup(mapperService), - SourceProvider.fromStoredFields() + SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(searchExecutionContext.lookup()).thenReturn(lookup); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index c36a126479e87..7555d1cd7fb48 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -408,7 +408,7 @@ protected Source getSourceFor(CheckedConsumer mapp iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { - SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), null, SourceFieldMetrics.NOOP); + SourceProvider provider = SourceProvider.fromLookup(mapper.mappers(), null, SourceFieldMetrics.NOOP); Source syntheticSource = provider.getSource(getOnlyLeafReader(reader).getContext(), 0); return syntheticSource; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java index ea9f8f6ae28a7..ac4842f0fc6a5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java @@ -45,11 +45,7 @@ public void testSyntheticSourceLoadLatency() throws IOException { iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { - SourceProvider provider = SourceProvider.fromSyntheticSource( - mapper.mapping(), - null, - createTestMapperMetrics().sourceFieldMetrics() - ); + SourceProvider provider = SourceProvider.fromLookup(mapper.mappers(), null, createTestMapperMetrics().sourceFieldMetrics()); Source synthetic = provider.getSource(getOnlyLeafReader(reader).getContext(), 0); assertEquals(synthetic.source().get("kwd"), "foo"); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java index 36f71c74838e6..7ad33b59122ae 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java @@ -134,7 +134,11 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { StringFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup, fdt) -> null, SourceProvider.fromStoredFields()), + new SearchLookup( + field -> null, + (ft, lookup, fdt) -> null, + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) + ), OnScriptError.FAIL ); StringFieldScript stringFieldScript = leafFactory.newInstance(reader.leaves().get(0)); @@ -164,7 +168,11 @@ public final void testFromSourceDoesNotEnforceCharsLimit() throws IOException { StringFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup, fdt) -> null, SourceProvider.fromStoredFields()), + new SearchLookup( + field -> null, + (ft, lookup, fdt) -> null, + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) + ), OnScriptError.FAIL ); StringFieldScript stringFieldScript = leafFactory.newInstance(reader.leaves().get(0)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index f74af767e9410..4670738c1d210 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1300,7 +1300,7 @@ public void testEmpty() throws Exception { MappedFieldType ft = mapperService.fieldType("field"); SourceProvider sourceProvider = mapperService.mappingLookup().isSourceSynthetic() ? (ctx, doc) -> { throw new IllegalArgumentException("Can't load source in scripts in synthetic mode"); - } : SourceProvider.fromStoredFields(); + } : SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()); SearchLookup searchLookup = new SearchLookup(null, null, sourceProvider); IndexFieldData sfd = ft.fielddataBuilder( new FieldDataContext("", null, () -> searchLookup, Set::of, MappedFieldType.FielddataOperation.SCRIPT) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index c1c21ccda580a..1ce916533a2c9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -1760,7 +1760,8 @@ protected void assertFetch(MapperService mapperService, String field, Object val ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); ParsedDocument doc = mapperService.documentMapper().parse(source); withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), ir -> { - Source s = SourceProvider.fromStoredFields().getSource(ir.leaves().get(0), 0); + Source s = SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) + .getSource(ir.leaves().get(0), 0); nativeFetcher.setNextReader(ir.leaves().get(0)); List fromNative = nativeFetcher.fetchValues(s, 0, new ArrayList<>()); DenseVectorFieldType denseVectorFieldType = (DenseVectorFieldType) ft; diff --git a/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java b/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java index 37fd33cf302b2..7cbef28bd9532 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/StoredFieldsSpecTests.java @@ -29,8 +29,9 @@ public class StoredFieldsSpecTests extends ESTestCase { public void testDefaults() { SearchSourceBuilder search = new SearchSourceBuilder(); + var context = searchContext(search); // defaults - return source and metadata fields - FetchContext fc = new FetchContext(searchContext(search)); + FetchContext fc = new FetchContext(context, context.newSourceLoader(null)); FetchSubPhaseProcessor sourceProcessor = new FetchSourcePhase().getProcessor(fc); assertNotNull(sourceProcessor); @@ -52,7 +53,8 @@ public void testDefaults() { public void testStoredFieldsDisabled() { SearchSourceBuilder search = new SearchSourceBuilder(); search.storedField("_none_"); - FetchContext fc = new FetchContext(searchContext(search)); + var context = searchContext(search); + FetchContext fc = new FetchContext(context, context.newSourceLoader(null)); assertNull(new StoredFieldsPhase().getProcessor(fc)); assertNull(new FetchSourcePhase().getProcessor(fc)); @@ -61,7 +63,8 @@ public void testStoredFieldsDisabled() { public void testScriptFieldsEnableMetadata() { SearchSourceBuilder search = new SearchSourceBuilder(); search.scriptField("field", new Script("script")); - FetchContext fc = new FetchContext(searchContext(search)); + var context = searchContext(search); + FetchContext fc = new FetchContext(context, null); FetchSubPhaseProcessor subPhaseProcessor = new ScriptFieldsPhase().getProcessor(fc); assertNotNull(subPhaseProcessor); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index c5f1efe561c22..5f26b046e0c8e 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -253,7 +253,8 @@ public void testMetadataFields() throws IOException { LeafReaderContext readerContext = searcher.getIndexReader().leaves().get(0); fieldFetcher.setNextReader(readerContext); - Source s = SourceProvider.fromStoredFields().getSource(readerContext, 0); + Source s = SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) + .getSource(readerContext, 0); Map fetchedFields = fieldFetcher.fetch(s, 0); assertThat(fetchedFields.size(), equalTo(5)); @@ -1539,7 +1540,11 @@ public void testFetchRuntimeFieldWithSourceDisabled() throws IOException { IndexSearcher searcher = newSearcher(iw); LeafReaderContext readerContext = searcher.getIndexReader().leaves().get(0); fieldFetcher.setNextReader(readerContext); - Source source = SourceProvider.fromStoredFields().getSource(readerContext, 0); + Source source = SourceProvider.fromLookup( + mapperService.mappingLookup(), + null, + mapperService.getMapperMetrics().sourceFieldMetrics() + ).getSource(readerContext, 0); Map fields = fieldFetcher.fetch(source, 0); assertEquals(1, fields.size()); DocumentField field = fields.get("runtime_field"); diff --git a/server/src/test/java/org/elasticsearch/search/lookup/SourceProviderTests.java b/server/src/test/java/org/elasticsearch/search/lookup/SourceProviderTests.java index b7ff79f7abd30..a899be8b070c2 100644 --- a/server/src/test/java/org/elasticsearch/search/lookup/SourceProviderTests.java +++ b/server/src/test/java/org/elasticsearch/search/lookup/SourceProviderTests.java @@ -27,6 +27,8 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.MappingLookup; +import org.elasticsearch.index.mapper.SourceFieldMetrics; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -46,7 +48,7 @@ public void testStoredFieldsSourceProvider() throws IOException { try (IndexReader reader = iw.getReader()) { LeafReaderContext readerContext = reader.leaves().get(0); - SourceProvider sourceProvider = SourceProvider.fromStoredFields(); + SourceProvider sourceProvider = SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP); Source source = sourceProvider.getSource(readerContext, 0); assertNotNull(source.internalSourceRef()); @@ -121,7 +123,7 @@ public ScoreMode scoreMode() { } private static CollectorManager assertingCollectorManager() { - SourceProvider sourceProvider = SourceProvider.fromStoredFields(); + SourceProvider sourceProvider = SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP); return new CollectorManager<>() { @Override public SourceAssertingCollector newCollector() { diff --git a/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java b/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java index d1fa8613ea393..cd8188689a760 100644 --- a/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java @@ -26,7 +26,9 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.index.mapper.GeoPointScriptFieldType; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.OnScriptError; +import org.elasticsearch.index.mapper.SourceFieldMetrics; import org.elasticsearch.script.AbstractLongFieldScript; import org.elasticsearch.script.GeoPointFieldScript; import org.elasticsearch.script.Script; @@ -93,7 +95,11 @@ public void testMatches() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - SearchLookup searchLookup = new SearchLookup(null, null, SourceProvider.fromStoredFields()); + SearchLookup searchLookup = new SearchLookup( + null, + null, + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) + ); Function leafFactory = ctx -> new GeoPointFieldScript( "test", Map.of(), diff --git a/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java b/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java index 0627a88284ac0..ac0a111cb0a52 100644 --- a/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java @@ -18,7 +18,9 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.OnScriptError; +import org.elasticsearch.index.mapper.SourceFieldMetrics; import org.elasticsearch.script.AbstractLongFieldScript; import org.elasticsearch.script.DateFieldScript; import org.elasticsearch.script.Script; @@ -71,7 +73,11 @@ public void testMatches() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181351]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - SearchLookup searchLookup = new SearchLookup(null, null, SourceProvider.fromStoredFields()); + SearchLookup searchLookup = new SearchLookup( + null, + null, + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) + ); Function leafFactory = ctx -> new DateFieldScript( "test", Map.of(), diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java index 47a227cebc956..1c785d58f9804 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java @@ -287,7 +287,11 @@ protected boolean supportsRangeQueries() { } protected static SearchExecutionContext mockContext(boolean allowExpensiveQueries, MappedFieldType mappedFieldType) { - return mockContext(allowExpensiveQueries, mappedFieldType, SourceProvider.fromStoredFields()); + return mockContext( + allowExpensiveQueries, + mappedFieldType, + SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP) + ); } protected static SearchExecutionContext mockContext( diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index ddcb3e6a42d80..4a1d33595eb79 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -858,7 +858,7 @@ protected static String syntheticSource(DocumentMapper mapper, SourceFilter filt final String synthetic1; final XContent xContent; { - SourceProvider provider = SourceProvider.fromSyntheticSource(mapper.mapping(), filter, SourceFieldMetrics.NOOP); + SourceProvider provider = SourceProvider.fromLookup(mapper.mappers(), filter, SourceFieldMetrics.NOOP); var source = provider.getSource(leafReader.getContext(), docId); synthetic1 = source.internalSourceRef().utf8ToString(); xContent = source.sourceContentType().xContent(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 52bf6484de80c..7e127ba307942 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -545,7 +545,7 @@ protected final List fetchFromDocValues(MapperService mapperService, MappedFi SearchLookup lookup = new SearchLookup( mapperService::fieldType, fieldDataLookup(mapperService), - SourceProvider.fromStoredFields() + SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) ); ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.getForField(ft, MappedFieldType.FielddataOperation.SEARCH)); IndexSearcher searcher = newSearcher(iw); @@ -566,7 +566,7 @@ protected static void assertScriptDocValues(MapperService mapperService, Object MappedFieldType ft = mapperService.fieldType("field"); SourceProvider sourceProvider = mapperService.mappingLookup().isSourceSynthetic() ? (ctx, doc) -> { throw new IllegalArgumentException("Can't load source in scripts in synthetic mode"); - } : SourceProvider.fromStoredFields(); + } : SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()); SearchLookup searchLookup = new SearchLookup(null, null, sourceProvider); IndexFieldData sfd = ft.fielddataBuilder( new FieldDataContext( @@ -865,7 +865,8 @@ protected void assertFetch(MapperService mapperService, String field, Object val ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); ParsedDocument doc = mapperService.documentMapper().parse(source); withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), ir -> { - Source s = SourceProvider.fromStoredFields().getSource(ir.leaves().get(0), 0); + Source s = SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) + .getSource(ir.leaves().get(0), 0); docValueFetcher.setNextReader(ir.leaves().get(0)); nativeFetcher.setNextReader(ir.leaves().get(0)); List fromDocValues = docValueFetcher.fetchValues(s, 0, new ArrayList<>()); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index e0c4be506969e..2559966ba5e7e 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -523,7 +523,7 @@ private SubSearchContext buildSubSearchContext( when(indexShard.shardId()).thenReturn(new ShardId("test", "test", 0)); when(indexShard.indexSettings()).thenReturn(indexSettings); when(ctx.indexShard()).thenReturn(indexShard); - when(ctx.newSourceLoader()).thenAnswer(inv -> searchExecutionContext.newSourceLoader(false)); + when(ctx.newSourceLoader(null)).thenAnswer(inv -> searchExecutionContext.newSourceLoader(null, false)); when(ctx.newIdLoader()).thenReturn(IdLoader.fromLeafStoredFieldLoader()); var res = new SubSearchContext(ctx); releasables.add(res); // TODO: nasty workaround for not getting the standard resource handling behavior of a real search context diff --git a/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java index 5c629075e6475..72417f5e61b27 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/fetch/HighlighterTestCase.java @@ -114,7 +114,7 @@ private static FetchContext fetchContext(SearchExecutionContext context, SearchS when(fetchContext.highlight()).thenReturn(search.highlighter().build(context)); when(fetchContext.parsedQuery()).thenReturn(new ParsedQuery(search.query().toQuery(context))); when(fetchContext.getSearchExecutionContext()).thenReturn(context); - when(fetchContext.sourceLoader()).thenReturn(context.newSourceLoader(false)); + when(fetchContext.sourceLoader()).thenReturn(context.newSourceLoader(null, false)); return fetchContext; } diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index bf612ebc70bc4..8ac52c3d48a0d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -13,6 +13,7 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; @@ -42,6 +43,7 @@ import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.ShardSearchContextId; import org.elasticsearch.search.internal.ShardSearchRequest; +import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.search.profile.Profilers; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.rank.context.QueryPhaseRankShardContext; @@ -552,8 +554,8 @@ public ReaderContext readerContext() { } @Override - public SourceLoader newSourceLoader() { - return searchExecutionContext.newSourceLoader(false); + public SourceLoader newSourceLoader(@Nullable SourceFilter filter) { + return searchExecutionContext.newSourceLoader(filter, false); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 31a859d0c1951..ef0b9aa931267 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -380,7 +380,7 @@ public String shardIdentifier() { @Override public SourceLoader newSourceLoader() { - return ctx.newSourceLoader(false); + return ctx.newSourceLoader(null, false); } @Override diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java index 47c4c8b23f42d..b3ee52c8dede5 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java @@ -167,7 +167,8 @@ protected void assertFetch(MapperService mapperService, String field, Object val ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); ParsedDocument doc = mapperService.documentMapper().parse(source); withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), ir -> { - Source s = SourceProvider.fromStoredFields().getSource(ir.leaves().get(0), 0); + Source s = SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) + .getSource(ir.leaves().get(0), 0); nativeFetcher.setNextReader(ir.leaves().get(0)); List fromNative = nativeFetcher.fetchValues(s, 0, new ArrayList<>()); assertThat(fromNative.size(), equalTo(1)); diff --git a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java index ac7ed8cf1c07f..69f95ae4bf52f 100644 --- a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java @@ -376,7 +376,8 @@ protected void assertFetch(MapperService mapperService, String field, Object val ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); ParsedDocument doc = mapperService.documentMapper().parse(source); withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), ir -> { - Source s = SourceProvider.fromStoredFields().getSource(ir.leaves().get(0), 0); + Source s = SourceProvider.fromLookup(mapperService.mappingLookup(), null, mapperService.getMapperMetrics().sourceFieldMetrics()) + .getSource(ir.leaves().get(0), 0); nativeFetcher.setNextReader(ir.leaves().get(0)); List fromNative = nativeFetcher.fetchValues(s, 0, new ArrayList<>()); RankVectorsFieldMapper.RankVectorsFieldType denseVectorFieldType = (RankVectorsFieldMapper.RankVectorsFieldType) ft;