From d468ef6c1032708eb8a2c85004de389946357943 Mon Sep 17 00:00:00 2001 From: Chris Parrinello Date: Tue, 30 Sep 2025 11:58:30 -0500 Subject: [PATCH 1/4] Add shard search subphase metrics for the fetch subphases. --- .../PercolatorHighlightSubFetchPhase.java | 10 ++ .../PercolatorMatchedSlotSubFetchPhase.java | 10 ++ .../action/search/TransportSearchIT.java | 37 ++++-- .../aggregations/metrics/TopHitsIT.java | 60 +++++---- .../search/fetch/FetchSubPhasePluginIT.java | 10 ++ .../stats/ShardSearchPhaseAPMMetrics.java | 28 +++- .../index/shard/SearchOperationListener.java | 19 +++ .../elasticsearch/node/NodeConstruction.java | 6 +- .../elasticsearch/search/SearchModule.java | 4 + .../search/fetch/FetchPhase.java | 8 ++ .../search/fetch/FetchProfiler.java | 5 + .../search/fetch/FetchSubPhase.java | 4 + .../search/fetch/FetchSubPhaseProcessor.java | 5 + .../search/fetch/subphase/ExplainPhase.java | 10 ++ .../fetch/subphase/FetchDocValuesPhase.java | 10 ++ .../fetch/subphase/FetchFieldsPhase.java | 10 ++ .../fetch/subphase/FetchScorePhase.java | 10 ++ .../fetch/subphase/FetchSourcePhase.java | 10 ++ .../fetch/subphase/FetchVersionPhase.java | 10 ++ .../search/fetch/subphase/InnerHitsPhase.java | 10 ++ .../fetch/subphase/MatchedQueriesPhase.java | 10 ++ .../fetch/subphase/ScriptFieldsPhase.java | 10 ++ .../fetch/subphase/SeqNoPrimaryTermPhase.java | 10 ++ .../fetch/subphase/StoredFieldsPhase.java | 9 ++ .../subphase/highlight/HighlightPhase.java | 10 ++ .../action/search/FetchSearchPhaseTests.java | 124 +++++++++++++----- .../ShardSearchPhaseAPMMetricsTests.java | 55 ++++++++ .../search/AsyncSearchSingleNodeTests.java | 37 ++++-- 28 files changed, 462 insertions(+), 79 deletions(-) diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java index 3efd968c598d1..c219c121a0523 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java @@ -68,6 +68,11 @@ public StoredFieldsSpec storedFieldsSpec() { return StoredFieldsSpec.NO_REQUIREMENTS; } + @Override + public String getName() { + return PercolatorHighlightSubFetchPhase.this.getName(); + } + @Override public void process(HitContext hit) throws IOException { boolean singlePercolateQuery = percolateQueries.size() == 1; @@ -138,6 +143,11 @@ public void process(HitContext hit) throws IOException { }; } + @Override + public String getName() { + return "percolator_highlight"; + } + static List locatePercolatorQuery(Query query) { if (query == null) { return Collections.emptyList(); diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java index b585b2d89303a..700119548fa71 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java @@ -77,6 +77,11 @@ public StoredFieldsSpec storedFieldsSpec() { return StoredFieldsSpec.NO_REQUIREMENTS; } + @Override + public String getName() { + return PercolatorMatchedSlotSubFetchPhase.this.getName(); + } + @Override public void process(HitContext hitContext) throws IOException { for (PercolateContext pc : percolateContexts) { @@ -128,6 +133,11 @@ public void process(HitContext hitContext) throws IOException { }; } + @Override + public String getName() { + return "percolator_matched_slot"; + } + static class PercolateContext { final PercolateQuery percolateQuery; final boolean singlePercolateQuery; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java index 89f46bee4b709..338a778d6b835 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/TransportSearchIT.java @@ -57,6 +57,7 @@ import org.elasticsearch.search.aggregations.support.AggregationContext; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.fetch.StoredFieldsSpec; @@ -95,22 +96,38 @@ public List getFetchSubPhases(FetchPhaseConstructionContext conte /** * Set up a fetch sub phase that throws an exception on indices whose name that start with "boom". */ - return Collections.singletonList(fetchContext -> new FetchSubPhaseProcessor() { + return Collections.singletonList(new FetchSubPhase() { @Override - public void setNextReader(LeafReaderContext readerContext) {} - - @Override - public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NO_REQUIREMENTS; + public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) throws IOException { + return new FetchSubPhaseProcessor() { + @Override + public void setNextReader(LeafReaderContext readerContext) {} + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + + @Override + public String getName() { + return "test"; + } + + @Override + public void process(FetchSubPhase.HitContext hitContext) { + if (fetchContext.getIndexName().startsWith("boom")) { + throw new RuntimeException("boom"); + } + } + }; } @Override - public void process(FetchSubPhase.HitContext hitContext) { - if (fetchContext.getIndexName().startsWith("boom")) { - throw new RuntimeException("boom"); - } + public String getName() { + return "test"; } }); + } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java index 58e639a260ab7..7cababbc53014 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java @@ -40,6 +40,7 @@ import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.ExecutionMode; +import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.fetch.StoredFieldsSpec; @@ -1377,31 +1378,44 @@ public void testScriptSorting() { public static class FetchPlugin extends Plugin implements SearchPlugin { @Override public List getFetchSubPhases(FetchPhaseConstructionContext context) { - return Collections.singletonList(fetchContext -> { - if (fetchContext.getIndexName().equals("idx")) { - return new FetchSubPhaseProcessor() { - - private LeafSearchLookup leafSearchLookup; - - @Override - public void setNextReader(LeafReaderContext ctx) { - leafSearchLookup = fetchContext.getSearchExecutionContext().lookup().getLeafSearchLookup(ctx); - } - - @Override - public void process(FetchSubPhase.HitContext hitContext) { - leafSearchLookup.setDocument(hitContext.docId()); - FieldLookup fieldLookup = leafSearchLookup.fields().get("text"); - hitContext.hit().setDocumentField(new DocumentField("text_stored_lookup", fieldLookup.getValues())); - } + return Collections.singletonList(new FetchSubPhase() { + @Override + public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) throws IOException { + if (fetchContext.getIndexName().equals("idx")) { + return new FetchSubPhaseProcessor() { + + private LeafSearchLookup leafSearchLookup; + + @Override + public void setNextReader(LeafReaderContext ctx) { + leafSearchLookup = fetchContext.getSearchExecutionContext().lookup().getLeafSearchLookup(ctx); + } + + @Override + public void process(FetchSubPhase.HitContext hitContext) { + leafSearchLookup.setDocument(hitContext.docId()); + FieldLookup fieldLookup = leafSearchLookup.fields().get("text"); + hitContext.hit().setDocumentField(new DocumentField("text_stored_lookup", fieldLookup.getValues())); + } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + + @Override + public String getName() { + return "test"; + } + }; + } + return null; + } - @Override - public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NO_REQUIREMENTS; - } - }; + @Override + public String getName() { + return "test"; } - return null; }); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java index 1320c3e23b16b..bff0988aadbcb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java @@ -121,6 +121,11 @@ public StoredFieldsSpec storedFieldsSpec() { return StoredFieldsSpec.NO_REQUIREMENTS; } + @Override + public String getName() { + return NAME; + } + @Override public void process(HitContext hitContext) throws IOException { hitExecute(searchContext, hitContext); @@ -128,6 +133,11 @@ public void process(HitContext hitContext) throws IOException { }; } + @Override + public String getName() { + return NAME; + } + private void hitExecute(FetchContext context, HitContext hitContext) throws IOException { TermVectorsFetchBuilder fetchSubPhaseBuilder = (TermVectorsFetchBuilder) context.getSearchExt(NAME); if (fetchSubPhaseBuilder == null) { diff --git a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java index ffd18d9712e3a..9365fbf6824bc 100644 --- a/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java +++ b/server/src/main/java/org/elasticsearch/index/search/stats/ShardSearchPhaseAPMMetrics.java @@ -17,18 +17,23 @@ import org.elasticsearch.telemetry.metric.LongHistogram; import org.elasticsearch.telemetry.metric.MeterRegistry; +import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; public final class ShardSearchPhaseAPMMetrics implements SearchOperationListener { public static final String QUERY_SEARCH_PHASE_METRIC = "es.search.shards.phases.query.duration.histogram"; public static final String FETCH_SEARCH_PHASE_METRIC = "es.search.shards.phases.fetch.duration.histogram"; + public static final String FETCH_SUBPHASE_METRIC_FORMAT = "es.search.shards.phases.fetch.subphase.%s.duration.histogram"; private final LongHistogram queryPhaseMetric; private final LongHistogram fetchPhaseMetric; + private final Map fetchSubPhaseMetrics; - public ShardSearchPhaseAPMMetrics(MeterRegistry meterRegistry) { + public ShardSearchPhaseAPMMetrics(MeterRegistry meterRegistry, List fetchSubPhaseNames) { this.queryPhaseMetric = meterRegistry.registerLongHistogram( QUERY_SEARCH_PHASE_METRIC, "Query search phase execution times at the shard level, expressed as a histogram", @@ -39,6 +44,17 @@ public ShardSearchPhaseAPMMetrics(MeterRegistry meterRegistry) { "Fetch search phase execution times at the shard level, expressed as a histogram", "ms" ); + this.fetchSubPhaseMetrics = fetchSubPhaseNames.stream() + .collect( + Collectors.toMap( + name -> name, + name -> meterRegistry.registerLongHistogram( + String.format(Locale.ROOT, FETCH_SUBPHASE_METRIC_FORMAT, name), + "Fetch sub-phase " + name + " execution times at the shard level, expressed as a histogram", + "ms" + ) + ) + ); } @Override @@ -55,6 +71,16 @@ public void onFetchPhase(SearchContext searchContext, long tookInNanos) { recordPhaseLatency(fetchPhaseMetric, tookInNanos, searchContext.request(), rangeTimestampFrom); } + @Override + public void onFetchSubPhase(SearchContext searchContext, String subPhaseName, long tookInNanos) { + SearchExecutionContext searchExecutionContext = searchContext.getSearchExecutionContext(); + Long rangeTimestampFrom = searchExecutionContext.getRangeTimestampFrom(); + LongHistogram histogramMetric = fetchSubPhaseMetrics.get(subPhaseName); + if (histogramMetric != null) { + recordPhaseLatency(histogramMetric, tookInNanos, searchContext.request(), rangeTimestampFrom); + } + } + private static void recordPhaseLatency( LongHistogram histogramMetric, long tookInNanos, diff --git a/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java b/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java index 9dc84090a610f..68b5a3158811e 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java +++ b/server/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java @@ -65,6 +65,14 @@ default void onFailedFetchPhase(SearchContext searchContext) {} */ default void onFetchPhase(SearchContext searchContext, long tookInNanos) {} + /** + * Executed after a fetch sub phase successfully finished for all docs in a shard. Used for APM metrics. + * @param searchContext the current search context + * @param subPhaseName the name of the fetch subphase + * @param tookInNanos the number of nanoseconds the fetch sub phase execution took + */ + default void onFetchSubPhase(SearchContext searchContext, String subPhaseName, long tookInNanos) {}; + /** * Executed before the DFS phase is executed * @param searchContext the current search context @@ -215,6 +223,17 @@ public void onPreDfsPhase(SearchContext searchContext) { } } + @Override + public void onFetchSubPhase(SearchContext searchContext, String subPhaseName, long tookInNanos) { + for (SearchOperationListener listener : listeners) { + try { + listener.onFetchSubPhase(searchContext, subPhaseName, tookInNanos); + } catch (Exception e) { + logger.warn(() -> "onFetchSubPhase listener [" + listener + "] failed", e); + } + } + } + @Override public void onFailedDfsPhase(SearchContext searchContext) { for (SearchOperationListener listener : listeners) { diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index 7a598475fc456..2013bf641f678 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -209,6 +209,7 @@ import org.elasticsearch.search.SearchService; import org.elasticsearch.search.SearchUtils; import org.elasticsearch.search.aggregations.support.AggregationUsageService; +import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.shutdown.PluginShutdownService; import org.elasticsearch.snapshots.IndexMetadataRestoreTransformer; import org.elasticsearch.snapshots.IndexMetadataRestoreTransformer.NoOpRestoreTransformer; @@ -855,7 +856,10 @@ private void construct( MergeMetrics mergeMetrics = new MergeMetrics(telemetryProvider.getMeterRegistry()); final List searchOperationListeners = List.of( - new ShardSearchPhaseAPMMetrics(telemetryProvider.getMeterRegistry()) + new ShardSearchPhaseAPMMetrics( + telemetryProvider.getMeterRegistry(), + searchModule.getFetchSubPhases().stream().map(FetchSubPhase::getName).toList() + ) ); List slowLogFieldProviders = pluginsService.loadServiceProviders(SlowLogFieldProvider.class); diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index f3aee46398432..14910da41fbd5 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -267,6 +267,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Consumer; @@ -1293,4 +1294,7 @@ public FetchPhase getFetchPhase() { return new FetchPhase(fetchSubPhases); } + public List getFetchSubPhases() { + return Collections.unmodifiableList(fetchSubPhases); + } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index 14c392b675a65..93a01a1b0fd3b 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -45,6 +45,7 @@ import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -172,6 +173,7 @@ && shouldExcludeInferenceFieldsFromSource(context.indexShard().indexSettings(), boolean requiresSource = storedFieldsSpec.requiresSource(); final int[] locallyAccumulatedBytes = new int[1]; NestedDocuments nestedDocuments = context.getSearchExecutionContext().getNestedDocuments(); + final Map subphaseAggregateDurations = new HashMap<>(); FetchPhaseDocsIterator docsIterator = new FetchPhaseDocsIterator() { @@ -229,7 +231,9 @@ protected SearchHit nextDoc(int doc) throws IOException { sourceProvider.source = hit.source(); fieldLookupProvider.setPreloadedStoredFieldValues(hit.hit().getId(), hit.loadedFields()); for (FetchSubPhaseProcessor processor : processors) { + long phaseStartTime = System.nanoTime(); processor.process(hit); + subphaseAggregateDurations.merge(processor.getName(), System.nanoTime() - phaseStartTime, Long::sum); } BytesReference sourceRef = hit.hit().getSourceRef(); @@ -254,6 +258,10 @@ protected SearchHit nextDoc(int doc) throws IOException { context.queryResult() ); + for (Map.Entry entry : subphaseAggregateDurations.entrySet()) { + context.indexShard().getSearchOperationListener().onFetchSubPhase(context, entry.getKey(), entry.getValue()); + } + if (context.isCancelled()) { for (SearchHit hit : hits) { // release all hits that would otherwise become owned and eventually released by SearchHits below diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchProfiler.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchProfiler.java index 5e7526fe9b13f..363a5e7ca8a3d 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchProfiler.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchProfiler.java @@ -129,6 +129,11 @@ public StoredFieldsSpec storedFieldsSpec() { return delegate.storedFieldsSpec(); } + @Override + public String getName() { + return delegate.getName(); // since this implementation wraps the delegate, return its name + } + @Override public void process(HitContext hitContext) throws IOException { Timer timer = breakdown.getNewTimer(FetchSubPhaseTiming.PROCESS); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java index 210058a25acd1..55f398a4671ee 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java @@ -102,4 +102,8 @@ public IndexReader topLevelReader() { */ FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) throws IOException; + /** + * The name of this fetch sub phase. Used for logging and stats. + */ + String getName(); } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseProcessor.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseProcessor.java index 5d0e626382a90..05c349b184915 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseProcessor.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseProcessor.java @@ -42,4 +42,9 @@ default Map getDebugInfo() { * The stored fields or source required by this sub phase */ StoredFieldsSpec storedFieldsSpec(); + + /** + * Name of the processor, for APM metrics purposes + */ + String getName(); } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java index 0e6172323277d..b98f11db086ec 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java @@ -66,6 +66,16 @@ public void process(HitContext hitContext) throws IOException { public StoredFieldsSpec storedFieldsSpec() { return StoredFieldsSpec.NO_REQUIREMENTS; } + + @Override + public String getName() { + return ExplainPhase.this.getName(); + } }; } + + @Override + public String getName() { + return "explain"; + } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java index a48536cc91365..1005ffbfaf4e3 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java @@ -68,6 +68,11 @@ public StoredFieldsSpec storedFieldsSpec() { return StoredFieldsSpec.NO_REQUIREMENTS; } + @Override + public String getName() { + return FetchDocValuesPhase.this.getName(); + } + @Override public void process(HitContext hit) throws IOException { for (DocValueField f : fields) { @@ -96,4 +101,9 @@ private static class DocValueField { this.fetcher = fetcher; } } + + @Override + public String getName() { + return "fetch_doc_values"; + } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java index e0cb5a668b4ab..d30c147793544 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java @@ -143,6 +143,11 @@ public StoredFieldsSpec storedFieldsSpec() { return metadataFieldFetcher.storedFieldsSpec(); } + @Override + public String getName() { + return FetchFieldsPhase.this.getName(); + } + @Override public void process(HitContext hitContext) throws IOException { final Map fields = fieldFetcher != null @@ -153,4 +158,9 @@ public void process(HitContext hitContext) throws IOException { } }; } + + @Override + public String getName() { + return "fetch_fields"; + } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java index d7ba4a0fc1a38..fa51602a802df 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchScorePhase.java @@ -48,6 +48,11 @@ public StoredFieldsSpec storedFieldsSpec() { return StoredFieldsSpec.NO_REQUIREMENTS; } + @Override + public String getName() { + return FetchScorePhase.this.getName(); + } + @Override public void process(HitContext hitContext) throws IOException { if (scorer == null || scorer.iterator().advance(hitContext.docId()) != hitContext.docId()) { @@ -57,4 +62,9 @@ public void process(HitContext hitContext) throws IOException { } }; } + + @Override + public String getName() { + return "fetch_score"; + } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java index df99a718887e1..84d35e92be52b 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java @@ -44,6 +44,11 @@ public StoredFieldsSpec storedFieldsSpec() { return StoredFieldsSpec.NEEDS_SOURCE; } + @Override + public String getName() { + return FetchSourcePhase.this.getName(); + } + @Override public void process(HitContext hitContext) { String index = fetchContext.getIndexName(); @@ -122,4 +127,9 @@ private static Source extractNested(Source in, SearchHit.NestedIdentity nestedId } return Source.fromMap(sourceMap, in.sourceContentType()); } + + @Override + public String getName() { + return "fetch_source"; + } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java index b2d2439f9961a..875cfe6cc6591 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchVersionPhase.java @@ -39,6 +39,11 @@ public StoredFieldsSpec storedFieldsSpec() { return StoredFieldsSpec.NO_REQUIREMENTS; } + @Override + public String getName() { + return FetchVersionPhase.this.getName(); + } + @Override public void process(HitContext hitContext) throws IOException { long version = Versions.NOT_FOUND; @@ -49,4 +54,9 @@ public void process(HitContext hitContext) throws IOException { } }; } + + @Override + public String getName() { + return "fetch_version"; + } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java index 374c96fdefe86..bb5a97419e147 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/InnerHitsPhase.java @@ -56,6 +56,11 @@ public StoredFieldsSpec storedFieldsSpec() { return storedFieldsSpec; } + @Override + public String getName() { + return InnerHitsPhase.this.getName(); + } + @Override public void process(HitContext hitContext) throws IOException { SearchHit hit = hitContext.hit(); @@ -110,4 +115,9 @@ private void hitExecute(Map innerHi h.mustIncRef(); } } + + @Override + public String getName() { + return "inner_hits"; + } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java index 53116ff8f6ebf..9c54af0b85878 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java @@ -99,6 +99,16 @@ public void process(HitContext hitContext) throws IOException { public StoredFieldsSpec storedFieldsSpec() { return StoredFieldsSpec.NO_REQUIREMENTS; } + + @Override + public String getName() { + return MatchedQueriesPhase.this.getName(); + } }; } + + @Override + public String getName() { + return "matched_queries"; + } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsPhase.java index 74fd4c7dc6b4c..ce987ffd88325 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/ScriptFieldsPhase.java @@ -49,6 +49,11 @@ public StoredFieldsSpec storedFieldsSpec() { return new StoredFieldsSpec(false, true, Set.of()); } + @Override + public String getName() { + return ScriptFieldsPhase.this.getName(); + } + @Override public void process(HitContext hitContext) { int docId = hitContext.docId(); @@ -93,4 +98,9 @@ private static FieldScript[] createLeafScripts(LeafReaderContext context, List highlightFields = new HashMap<>(); @@ -158,4 +163,9 @@ private FieldContext contextBuilders( } return new FieldContext(storedFieldsSpec, builders); } + + @Override + public String getName() { + return "highlight"; + } } diff --git a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java index b6ca12368f762..dcfce4ea2f840 100644 --- a/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/FetchSearchPhaseTests.java @@ -43,12 +43,15 @@ import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.SearchOperationListener; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; +import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchPhase; import org.elasticsearch.search.fetch.FetchPhaseExecutionException; import org.elasticsearch.search.fetch.FetchSearchResult; @@ -88,6 +91,8 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class FetchSearchPhaseTests extends ESTestCase { private static final long FETCH_PROFILE_TIME = 555; @@ -850,21 +855,36 @@ public void addEstimateBytesAndMaybeBreak(long bytes, String label) throws Circu } }; try (SearchContext searchContext = createSearchContext(contextIndexSearcher, true, breakingCircuitBreaker)) { - FetchPhase fetchPhase = new FetchPhase(List.of(fetchContext -> new FetchSubPhaseProcessor() { + FetchPhase fetchPhase = new FetchPhase(List.of(new FetchSubPhase() { @Override - public void setNextReader(LeafReaderContext readerContext) throws IOException { + public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) throws IOException { + return new FetchSubPhaseProcessor() { + @Override + public void setNextReader(LeafReaderContext readerContext) throws IOException { - } + } - @Override - public void process(FetchSubPhase.HitContext hitContext) throws IOException { - Source source = hitContext.source(); - hitContext.hit().sourceRef(source.internalSourceRef()); + @Override + public void process(FetchSubPhase.HitContext hitContext) throws IOException { + Source source = hitContext.source(); + hitContext.hit().sourceRef(source.internalSourceRef()); + } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NEEDS_SOURCE; + } + + @Override + public String getName() { + return "test"; + } + }; } @Override - public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + public String getName() { + return "test"; } })); fetchPhase.execute(searchContext, IntStream.range(0, 100).toArray(), null); @@ -904,23 +924,39 @@ public void testTimerStoppedAndSubPhasesExceptionsPropagate() throws IOException true ) ) { - FetchPhase fetchPhase = new FetchPhase(List.of(fetchContext -> new FetchSubPhaseProcessor() { + FetchPhase fetchPhase = new FetchPhase(List.of(new FetchSubPhase() { @Override - public void setNextReader(LeafReaderContext readerContext) throws IOException { - throw new IOException("bad things"); - } + public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) throws IOException { + return new FetchSubPhaseProcessor() { + @Override + public void setNextReader(LeafReaderContext readerContext) throws IOException { + throw new IOException("bad things"); + } - @Override - public void process(FetchSubPhase.HitContext hitContext) throws IOException { - Source source = hitContext.source(); - hitContext.hit().sourceRef(source.internalSourceRef()); + @Override + public void process(FetchSubPhase.HitContext hitContext) throws IOException { + Source source = hitContext.source(); + hitContext.hit().sourceRef(source.internalSourceRef()); + } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NEEDS_SOURCE; + } + + @Override + public String getName() { + return "test"; + } + }; } @Override - public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + public String getName() { + return "test"; } })); + FetchPhaseExecutionException fetchPhaseExecutionException = assertThrows( FetchPhaseExecutionException.class, () -> fetchPhase.execute(searchContext, IntStream.range(0, 100).toArray(), null) @@ -945,25 +981,40 @@ public boolean shouldCache(Query query) { } private static FetchPhase createFetchPhase(ContextIndexSearcher contextIndexSearcher) { - return new FetchPhase(Collections.singletonList(fetchContext -> new FetchSubPhaseProcessor() { - boolean processCalledOnce = false; - + return new FetchPhase(Collections.singletonList(new FetchSubPhase() { @Override - public void setNextReader(LeafReaderContext readerContext) {} + public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) throws IOException { + return new FetchSubPhaseProcessor() { + boolean processCalledOnce = false; - @Override - public void process(FetchSubPhase.HitContext hitContext) { - // we throw only once one doc has been fetched, so we can test partial results are returned - if (processCalledOnce) { - contextIndexSearcher.throwTimeExceededException(); - } else { - processCalledOnce = true; - } + @Override + public void setNextReader(LeafReaderContext readerContext) {} + + @Override + public void process(FetchSubPhase.HitContext hitContext) { + // we throw only once one doc has been fetched, so we can test partial results are returned + if (processCalledOnce) { + contextIndexSearcher.throwTimeExceededException(); + } else { + processCalledOnce = true; + } + } + + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + + @Override + public String getName() { + return "test"; + } + }; } @Override - public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NO_REQUIREMENTS; + public String getName() { + return "test"; } })); } @@ -1030,7 +1081,12 @@ public void onRemoval(ShardId shardId, Accountable accountable) { null, MapperMetrics.NOOP ); - TestSearchContext searchContext = new TestSearchContext(searchExecutionContext, null, contextIndexSearcher) { + + IndexShard mockShard = mock(IndexShard.class); + when(mockShard.getSearchOperationListener()).thenReturn(new SearchOperationListener() { + }); + + TestSearchContext searchContext = new TestSearchContext(searchExecutionContext, mockShard, contextIndexSearcher) { private final FetchSearchResult fetchSearchResult = new FetchSearchResult(); private final ShardSearchRequest request = new ShardSearchRequest( OriginalIndices.NONE, diff --git a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java index 25fa7d1f36b41..734d3546c83e5 100644 --- a/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java +++ b/server/src/test/java/org/elasticsearch/search/TelemetryMetrics/ShardSearchPhaseAPMMetricsTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.SystemIndexPlugin; +import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.telemetry.Measurement; import org.elasticsearch.telemetry.TestTelemetryPlugin; import org.elasticsearch.test.ESSingleNodeTestCase; @@ -30,11 +31,14 @@ import java.util.Collection; import java.util.List; +import java.util.Locale; import java.util.Map; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery; import static org.elasticsearch.index.search.stats.ShardSearchPhaseAPMMetrics.FETCH_SEARCH_PHASE_METRIC; +import static org.elasticsearch.index.search.stats.ShardSearchPhaseAPMMetrics.FETCH_SUBPHASE_METRIC_FORMAT; import static org.elasticsearch.index.search.stats.ShardSearchPhaseAPMMetrics.QUERY_SEARCH_PHASE_METRIC; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; @@ -124,6 +128,57 @@ public void testSearchTransportMetricsQueryThenFetch() { final List fetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement(FETCH_SEARCH_PHASE_METRIC); assertEquals(1, fetchMeasurements.size()); assertAttributes(fetchMeasurements, false, false); + final List storedFieldsFetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement( + String.format(Locale.ROOT, FETCH_SUBPHASE_METRIC_FORMAT, "stored_fields") + ); + assertEquals(1, storedFieldsFetchMeasurements.size()); + assertAttributes(storedFieldsFetchMeasurements, false, false); + final List fetchSourceFetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement( + String.format(Locale.ROOT, FETCH_SUBPHASE_METRIC_FORMAT, "fetch_source") + ); + assertEquals(1, fetchSourceFetchMeasurements.size()); + assertAttributes(fetchSourceFetchMeasurements, false, false); + final List fetchFieldsFetchMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement( + String.format(Locale.ROOT, FETCH_SUBPHASE_METRIC_FORMAT, "fetch_fields") + ); + assertEquals(1, fetchFieldsFetchMeasurements.size()); + assertAttributes(fetchFieldsFetchMeasurements, false, false); + } + + public void testSearchFetchSubPhaseMeasurements() { + assertSearchHitsWithoutFailures( + client().prepareSearch(indexName) + .setSearchType(SearchType.QUERY_THEN_FETCH) + .setPreFilterShardSize(1) // force a can match phase + .setExplain(true) + .setVersion(true) + .seqNoAndPrimaryTerm(true) + .highlighter(new HighlightBuilder().field("body")) + .setQuery(matchQuery("body", "doc1").queryName("foobar")) + .addDocValueField("docvalue_fields", "@timestamp"), + "1" + ); + + // assert that all fetch subphases are measured + // missing subphases like script fields are not measured here because they need an IT environment + List subphases = List.of( + "explain", + "stored_fields", + "fetch_doc_values", + "fetch_fields", + "fetch_source", + "fetch_version", + "seq_no_primary_term", + "matched_queries", + "highlight" + ); + for (String subphase : subphases) { + final List fetchSubPhaseMeasurements = getTestTelemetryPlugin().getLongHistogramMeasurement( + String.format(Locale.ROOT, FETCH_SUBPHASE_METRIC_FORMAT, subphase) + ); + assertEquals(1, fetchSubPhaseMeasurements.size()); + assertAttributes(fetchSubPhaseMeasurements, false, false); + } } public void testSearchTransportMetricsQueryThenFetchSystem() { diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java index fd4463df07a73..75ceba22f917c 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchSingleNodeTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; import org.elasticsearch.search.fetch.StoredFieldsSpec; @@ -30,6 +31,7 @@ import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchRequest; import org.hamcrest.CoreMatchers; +import java.io.IOException; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -121,21 +123,36 @@ public void testFetchFailuresOnlySomeShards() throws Exception { public static final class SubFetchPhasePlugin extends Plugin implements SearchPlugin { @Override public List getFetchSubPhases(FetchPhaseConstructionContext context) { - return Collections.singletonList(searchContext -> new FetchSubPhaseProcessor() { + return Collections.singletonList(new FetchSubPhase() { @Override - public void setNextReader(LeafReaderContext readerContext) {} + public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) throws IOException { + return new FetchSubPhaseProcessor() { + @Override + public void setNextReader(LeafReaderContext readerContext) {} - @Override - public StoredFieldsSpec storedFieldsSpec() { - return StoredFieldsSpec.NO_REQUIREMENTS; + @Override + public StoredFieldsSpec storedFieldsSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + + @Override + public String getName() { + return "test"; + } + + @Override + public void process(FetchSubPhase.HitContext hitContext) { + if (hitContext.hit().getId().startsWith("boom")) { + throw new RuntimeException("boom"); + } + + } + }; } @Override - public void process(FetchSubPhase.HitContext hitContext) { - if (hitContext.hit().getId().startsWith("boom")) { - throw new RuntimeException("boom"); - } - + public String getName() { + return "test"; } }); } From 495683ce32c6aa1b4c10b0a2d9f3dcfe6a1a6884 Mon Sep 17 00:00:00 2001 From: Chris Parrinello Date: Tue, 30 Sep 2025 12:46:55 -0500 Subject: [PATCH 2/4] Update docs/changelog/135713.yaml --- docs/changelog/135713.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/135713.yaml diff --git a/docs/changelog/135713.yaml b/docs/changelog/135713.yaml new file mode 100644 index 0000000000000..ab2e9c5c1b30f --- /dev/null +++ b/docs/changelog/135713.yaml @@ -0,0 +1,5 @@ +pr: 135713 +summary: Add shard search subphase metrics for the fetch subphases +area: Search +type: enhancement +issues: [] From 9d333139872e8de42467cbd295886550c3ef1cf6 Mon Sep 17 00:00:00 2001 From: Chris Parrinello Date: Tue, 30 Sep 2025 13:16:47 -0500 Subject: [PATCH 3/4] fix aggregator test cases --- .../elasticsearch/search/aggregations/AggregatorTestCase.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index f199fcaabd29b..5efef824621b0 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -114,6 +114,7 @@ import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.SearchOperationListener; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.CrankyCircuitBreakerService; import org.elasticsearch.indices.IndicesModule; @@ -522,6 +523,7 @@ private SubSearchContext buildSubSearchContext( IndexShard indexShard = mock(IndexShard.class); when(indexShard.shardId()).thenReturn(new ShardId("test", "test", 0)); when(indexShard.indexSettings()).thenReturn(indexSettings); + when(indexShard.getSearchOperationListener()).thenReturn(new SearchOperationListener() {}); when(ctx.indexShard()).thenReturn(indexShard); when(ctx.newSourceLoader(null)).thenAnswer(inv -> searchExecutionContext.newSourceLoader(null, false)); when(ctx.newIdLoader()).thenReturn(IdLoader.fromLeafStoredFieldLoader()); From c6c50e1026e0113c0f6df462cfa610bf3951544c Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 30 Sep 2025 18:24:26 +0000 Subject: [PATCH 4/4] [CI] Auto commit changes from spotless --- .../elasticsearch/search/aggregations/AggregatorTestCase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 5efef824621b0..5ea23fb3ea8d1 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -523,7 +523,8 @@ private SubSearchContext buildSubSearchContext( IndexShard indexShard = mock(IndexShard.class); when(indexShard.shardId()).thenReturn(new ShardId("test", "test", 0)); when(indexShard.indexSettings()).thenReturn(indexSettings); - when(indexShard.getSearchOperationListener()).thenReturn(new SearchOperationListener() {}); + when(indexShard.getSearchOperationListener()).thenReturn(new SearchOperationListener() { + }); when(ctx.indexShard()).thenReturn(indexShard); when(ctx.newSourceLoader(null)).thenAnswer(inv -> searchExecutionContext.newSourceLoader(null, false)); when(ctx.newIdLoader()).thenReturn(IdLoader.fromLeafStoredFieldLoader());