From 931bca54bd9175e25fd1e70ed36e37ffc9422d17 Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Tue, 28 Oct 2025 08:19:56 -0400 Subject: [PATCH 1/8] Add chunk_rescorer usage to output of explain for text_similarity_rank_retriever --- ...xt_similarity_rank_docs_explain_chunks.csv | 1 + .../resources/transport/upper_bounds/9.3.csv | 2 +- .../xpack/inference/InferenceFeatures.java | 4 +- .../textsimilarity/ChunkScorerConfig.java | 4 ++ .../textsimilarity/TextSimilarityRankDoc.java | 70 +++++++++++++++---- .../TextSimilarityRankRetrieverBuilder.java | 4 +- .../TextSimilarityRankDocTests.java | 21 +++++- .../70_text_similarity_rank_retriever.yml | 61 ++++++++++++++++ 8 files changed, 147 insertions(+), 20 deletions(-) create mode 100644 server/src/main/resources/transport/definitions/referable/text_similarity_rank_docs_explain_chunks.csv diff --git a/server/src/main/resources/transport/definitions/referable/text_similarity_rank_docs_explain_chunks.csv b/server/src/main/resources/transport/definitions/referable/text_similarity_rank_docs_explain_chunks.csv new file mode 100644 index 0000000000000..a79975c00dea2 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/text_similarity_rank_docs_explain_chunks.csv @@ -0,0 +1 @@ +9204000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index 47728be741b74..83b692cda34f1 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -esql_resolve_fields_response_removed_min_tv,9203000 +text_similarity_rank_docs_explain_chunks,9204000 diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java index bae88e6372fdb..a55a126976284 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java @@ -29,6 +29,7 @@ import static org.elasticsearch.xpack.inference.queries.LegacySemanticKnnVectorQueryRewriteInterceptor.SEMANTIC_KNN_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED; import static org.elasticsearch.xpack.inference.queries.LegacySemanticMatchQueryRewriteInterceptor.SEMANTIC_MATCH_QUERY_REWRITE_INTERCEPTION_SUPPORTED; import static org.elasticsearch.xpack.inference.queries.LegacySemanticSparseVectorQueryRewriteInterceptor.SEMANTIC_SPARSE_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED; +import static org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankDoc.TEXT_SIMILARITY_RANK_DOC_EXPLAIN_CHUNKS; import static org.elasticsearch.xpack.inference.rank.textsimilarity.TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_SNIPPETS; /** @@ -100,7 +101,8 @@ public Set getTestFeatures() { InterceptedInferenceQueryBuilder.NEW_SEMANTIC_QUERY_INTERCEPTORS, TEXT_SIMILARITY_RERANKER_SNIPPETS, ModelStats.SEMANTIC_TEXT_USAGE, - SEARCH_USAGE_EXTENDED_DATA + SEARCH_USAGE_EXTENDED_DATA, + TEXT_SIMILARITY_RANK_DOC_EXPLAIN_CHUNKS ) ); testFeatures.addAll(getFeatures()); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/ChunkScorerConfig.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/ChunkScorerConfig.java index 4a87dab559a91..94e30c6fcac6b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/ChunkScorerConfig.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/ChunkScorerConfig.java @@ -75,6 +75,10 @@ public Integer size() { return size; } + public int sizeOrDefault() { + return size != null ? size : DEFAULT_SIZE; + } + public String inferenceText() { return inferenceText; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDoc.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDoc.java index f118fd42bad4d..9916979b5e4bb 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDoc.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDoc.java @@ -8,9 +8,11 @@ package org.elasticsearch.xpack.inference.rank.textsimilarity; import org.apache.lucene.search.Explanation; +import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.search.rank.RankDoc; import org.elasticsearch.xcontent.XContentBuilder; @@ -19,19 +21,33 @@ public class TextSimilarityRankDoc extends RankDoc { + public static final NodeFeature TEXT_SIMILARITY_RANK_DOC_EXPLAIN_CHUNKS = new NodeFeature("text_similarity_rank_doc_explain_chunks"); + private static final TransportVersion TEXT_SIMILARITY_RANK_DOC_EXPLAIN_CHUNKS_VERSION = TransportVersion.fromName( + "text_similarity_rank_docs_explain_chunks" + ); + public static final String NAME = "text_similarity_rank_doc"; public final String inferenceId; public final String field; + public final ChunkScorerConfig chunkScorerConfig; public TextSimilarityRankDoc(int doc, float score, int shardIndex) { - this(doc, score, shardIndex, null, null); + this(doc, score, shardIndex, null, null, null); } - public TextSimilarityRankDoc(int doc, float score, int shardIndex, String inferenceId, String field) { + public TextSimilarityRankDoc( + int doc, + float score, + int shardIndex, + String inferenceId, + String field, + ChunkScorerConfig chunkScorerConfig + ) { super(doc, score, shardIndex); this.inferenceId = inferenceId; this.field = field; + this.chunkScorerConfig = chunkScorerConfig; } public TextSimilarityRankDoc(StreamInput in) throws IOException { @@ -43,22 +59,34 @@ public TextSimilarityRankDoc(StreamInput in) throws IOException { inferenceId = in.readString(); field = in.readString(); } + + if (in.getTransportVersion().supports(TEXT_SIMILARITY_RANK_DOC_EXPLAIN_CHUNKS_VERSION)) { + boolean hasChunkScorerConfig = in.readBoolean(); + chunkScorerConfig = hasChunkScorerConfig ? new ChunkScorerConfig(in) : null; + } else { + chunkScorerConfig = null; + } } @Override public Explanation explain(Explanation[] sources, String[] queryNames) { assert inferenceId != null && field != null; final String queryAlias = queryNames[0] == null ? "" : "[" + queryNames[0] + "]"; - return Explanation.match( - score, - "text_similarity_reranker match using inference endpoint: [" - + inferenceId - + "] on document field: [" - + field - + "] matching on source query " - + queryAlias, - sources - ); + + StringBuilder sb = new StringBuilder(); + + sb.append("text_similarity_reranker match using inference endpoint: [") + .append(inferenceId) + .append("] on document field: [") + .append(field) + .append("] matching on source query ") + .append(queryAlias); + + if (chunkScorerConfig != null) { + sb.append("and rescoring considering only top [").append(chunkScorerConfig.sizeOrDefault()).append("] best chunks"); + } + + return Explanation.match(score, sb.toString(), sources); } @Override @@ -70,17 +98,26 @@ public void doWriteTo(StreamOutput out) throws IOException { out.writeString(inferenceId == null ? "" : inferenceId); out.writeString(field == null ? "" : field); } + if (out.getTransportVersion().supports(TEXT_SIMILARITY_RANK_DOC_EXPLAIN_CHUNKS_VERSION)) { + boolean hasChunkScorerConfig = chunkScorerConfig != null; + out.writeBoolean(hasChunkScorerConfig); + if (hasChunkScorerConfig) { + chunkScorerConfig.writeTo(out); + } + } } @Override public boolean doEquals(RankDoc rd) { TextSimilarityRankDoc tsrd = (TextSimilarityRankDoc) rd; - return Objects.equals(inferenceId, tsrd.inferenceId) && Objects.equals(field, tsrd.field); + return Objects.equals(inferenceId, tsrd.inferenceId) + && Objects.equals(field, tsrd.field) + && Objects.equals(chunkScorerConfig, tsrd.chunkScorerConfig); } @Override public int doHashCode() { - return Objects.hash(inferenceId, field); + return Objects.hash(inferenceId, field, chunkScorerConfig); } @Override @@ -96,6 +133,8 @@ public String toString() { + inferenceId + ", field=" + field + + ", chunkScorerConfig=" + + chunkScorerConfig.toString() + '}'; } @@ -112,5 +151,8 @@ protected void doToXContent(XContentBuilder builder, Params params) throws IOExc if (field != null) { builder.field("field", field); } + if (chunkScorerConfig != null) { + builder.field("chunkScorerConfig", chunkScorerConfig); + } } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java index d45a93c1444e3..d28b76a23f07e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java @@ -198,7 +198,9 @@ protected RankDoc[] combineInnerRetrieverResults(List rankResults, b assert scoreDoc.score >= 0; if (minScore == null || scoreDoc.score >= minScore) { if (explain) { - filteredDocs.add(new TextSimilarityRankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex, inferenceId, field)); + filteredDocs.add( + new TextSimilarityRankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex, inferenceId, field, chunkScorerConfig) + ); } else { filteredDocs.add(new TextSimilarityRankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex)); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDocTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDocTests.java index fed4565c54bd4..e65b6f1bcab48 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDocTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDocTests.java @@ -18,6 +18,7 @@ import java.util.List; import static org.elasticsearch.search.rank.RankDoc.NO_RANK; +import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.generateRandomChunkingSettings; public class TextSimilarityRankDocTests extends AbstractRankDocWireSerializingTestCase { @@ -27,7 +28,8 @@ static TextSimilarityRankDoc createTestTextSimilarityRankDoc() { randomFloat(), randomBoolean() ? -1 : randomNonNegativeInt(), randomAlphaOfLength(randomIntBetween(2, 5)), - randomAlphaOfLength(randomIntBetween(2, 5)) + randomAlphaOfLength(randomIntBetween(2, 5)), + randomChunkScorerConfig() ); instance.rank = randomBoolean() ? NO_RANK : randomIntBetween(1, 10000); return instance; @@ -58,8 +60,9 @@ protected TextSimilarityRankDoc mutateInstance(TextSimilarityRankDoc instance) t int rank = instance.rank; String inferenceId = instance.inferenceId; String field = instance.field; + ChunkScorerConfig chunkScorerConfig = instance.chunkScorerConfig; - switch (randomInt(5)) { + switch (randomInt(6)) { case 0: doc = randomValueOtherThan(doc, ESTestCase::randomNonNegativeInt); break; @@ -78,11 +81,23 @@ protected TextSimilarityRankDoc mutateInstance(TextSimilarityRankDoc instance) t case 5: field = randomValueOtherThan(field, () -> randomAlphaOfLength(randomIntBetween(2, 5))); break; + case 6: + chunkScorerConfig = randomValueOtherThan(chunkScorerConfig, () -> randomChunkScorerConfig()); + break; default: throw new AssertionError(); } - TextSimilarityRankDoc mutated = new TextSimilarityRankDoc(doc, score, shardIndex, inferenceId, field); + TextSimilarityRankDoc mutated = new TextSimilarityRankDoc(doc, score, shardIndex, inferenceId, field, chunkScorerConfig); mutated.rank = rank; return mutated; } + + private static ChunkScorerConfig randomChunkScorerConfig() { + if (randomBoolean()) { + return null; + } + + Integer size = randomBoolean() ? randomIntBetween(1, 5) : null; + return new ChunkScorerConfig(size, randomAlphaOfLengthBetween(5, 15), generateRandomChunkingSettings(false)); + } } diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml index 61d586128675c..8e67141bcddb5 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml @@ -859,3 +859,64 @@ setup: - match: { hits.hits.0._id: "doc_1" } - match: { hits.hits.1._id: "doc_2" } + +--- +"Explain outputs information on chunk rescorer if selected": + + - requires: + cluster_features: "text_similarity_rank_doc_explain_chunks" + reason: rescore_chunks added explain support in 9.3.0 + + - do: + search: + index: test-index + body: + track_total_hits: true + explain: true + fields: [ "text", "semantic_text_field", "topic" ] + retriever: + text_similarity_reranker: + retriever: + standard: + query: + match: + topic: + query: "science" + rank_window_size: 10 + inference_id: my-rerank-model + inference_text: "how often does the moon hide the sun?" + field: semantic_text_field + size: 10 + + - match: { hits.total.value: 2 } + - length: { hits.hits: 2 } + + - match: { hits.hits.0._explanation.details.0.description: "text_similarity_reranker match using inference endpoint: [my-rerank-model] on document field: [semantic_text_field] matching on source query " } + + + - do: + search: + index: test-index + body: + track_total_hits: true + explain: true + fields: [ "text", "semantic_text_field", "topic" ] + retriever: + text_similarity_reranker: + retriever: + standard: + query: + match: + topic: + query: "science" + rank_window_size: 10 + inference_id: my-rerank-model + inference_text: "how often does the moon hide the sun?" + field: semantic_text_field + chunk_rescorer: { } + size: 10 + + - match: { hits.total.value: 2 } + - length: { hits.hits: 2 } + + - match: { hits.hits.0._explanation.details.0.description: "text_similarity_reranker match using inference endpoint: [my-rerank-model] on document field: [semantic_text_field] matching on source query and rescoring considering only top [1] best chunks" } From 414004d1c2c89857bf97ce4baf70cc9dc7d6e40b Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Tue, 28 Oct 2025 08:43:29 -0400 Subject: [PATCH 2/8] Update docs/changelog/137249.yaml --- docs/changelog/137249.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/137249.yaml diff --git a/docs/changelog/137249.yaml b/docs/changelog/137249.yaml new file mode 100644 index 0000000000000..35c18b5db1552 --- /dev/null +++ b/docs/changelog/137249.yaml @@ -0,0 +1,5 @@ +pr: 137249 +summary: Add `chunk_rescorer` usage to output of explain for `text_similarity_rank_retriever` +area: Relevance +type: enhancement +issues: [] From f40287e141c5d3b66deba8cc07f7a5ceae7d6e2d Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Tue, 28 Oct 2025 08:57:57 -0400 Subject: [PATCH 3/8] Update toString --- .../inference/rank/textsimilarity/TextSimilarityRankDoc.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDoc.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDoc.java index 9916979b5e4bb..2d4db4af7bc7f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDoc.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankDoc.java @@ -134,7 +134,7 @@ public String toString() { + ", field=" + field + ", chunkScorerConfig=" - + chunkScorerConfig.toString() + + chunkScorerConfig + '}'; } From 6ef38816f2c4be22130b69d6fcb0847d64c3618b Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Tue, 28 Oct 2025 11:06:15 -0400 Subject: [PATCH 4/8] Add support for profile --- .../action/search/TransportSearchAction.java | 4 +- .../org/elasticsearch/index/IndexService.java | 1 + .../query/CoordinatorRewriteContext.java | 1 + .../index/query/QueryRewriteContext.java | 17 ++++- .../index/query/SearchExecutionContext.java | 1 + .../elasticsearch/indices/IndicesService.java | 6 +- .../elasticsearch/search/SearchService.java | 8 ++- .../retriever/CompoundRetrieverBuilder.java | 3 +- .../search/TransportSearchActionTests.java | 2 +- .../index/query/QueryRewriteContextTests.java | 2 + .../test/AbstractBuilderTestCase.java | 1 + .../textsimilarity/ChunkScorerConfig.java | 10 +++ .../TextSimilarityRankRetrieverBuilder.java | 4 +- .../70_text_similarity_rank_retriever.yml | 64 +++++++++++++++++++ 14 files changed, 111 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index d5085701efb88..a7e265f0f3eea 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -598,6 +598,7 @@ public void onFailure(Exception e) { }); final boolean isExplain = source != null && source.explain() != null && source.explain(); + final boolean isProfile = source != null && source.profile(); Rewriteable.rewriteAndFetch( original, searchService.getRewriteContext( @@ -607,7 +608,8 @@ public void onFailure(Exception e) { resolvedIndices, original.pointInTimeBuilder(), shouldMinimizeRoundtrips(original), - isExplain + isExplain, + isProfile ), rewriteListener ); diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 9af701fa81642..e4de5ba388da1 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -839,6 +839,7 @@ public QueryRewriteContext newQueryRewriteContext( null, null, null, + false, false ); } diff --git a/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java index 918179307438a..b470c2a32af19 100644 --- a/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java @@ -123,6 +123,7 @@ public CoordinatorRewriteContext( null, null, null, + false, false ); this.dateFieldRangeInfo = dateFieldRangeInfo; diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java index 5ca77374dee59..d15140be0c8a0 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java @@ -80,6 +80,7 @@ public class QueryRewriteContext { private QueryRewriteInterceptor queryRewriteInterceptor; private final Boolean ccsMinimizeRoundTrips; private final boolean isExplain; + private final boolean isProfile; private Long timeRangeFilterFromMillis; private boolean trackTimeRangeFilterFrom = true; @@ -103,7 +104,8 @@ public QueryRewriteContext( final PointInTimeBuilder pit, final QueryRewriteInterceptor queryRewriteInterceptor, final Boolean ccsMinimizeRoundTrips, - final boolean isExplain + final boolean isExplain, + final boolean isProfile ) { this.parserConfiguration = parserConfiguration; @@ -127,6 +129,7 @@ public QueryRewriteContext( this.queryRewriteInterceptor = queryRewriteInterceptor; this.ccsMinimizeRoundTrips = ccsMinimizeRoundTrips; this.isExplain = isExplain; + this.isProfile = isProfile; } public QueryRewriteContext(final XContentParserConfiguration parserConfiguration, final Client client, final LongSupplier nowInMillis) { @@ -150,6 +153,7 @@ public QueryRewriteContext(final XContentParserConfiguration parserConfiguration null, null, null, + false, false ); } @@ -175,6 +179,7 @@ public QueryRewriteContext( pit, queryRewriteInterceptor, ccsMinimizeRoundTrips, + false, false ); } @@ -189,7 +194,8 @@ public QueryRewriteContext( final PointInTimeBuilder pit, final QueryRewriteInterceptor queryRewriteInterceptor, final Boolean ccsMinimizeRoundTrips, - final boolean isExplain + final boolean isExplain, + final boolean isProfile ) { this( parserConfiguration, @@ -211,7 +217,8 @@ public QueryRewriteContext( pit, queryRewriteInterceptor, ccsMinimizeRoundTrips, - isExplain + isExplain, + isProfile ); } @@ -324,6 +331,10 @@ public boolean isExplain() { return this.isExplain; } + public boolean isProfile() { + return this.isProfile; + } + public NamedWriteableRegistry getWriteableRegistry() { return writeableRegistry; } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 6919bd725133c..d50c4f0f618f5 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -282,6 +282,7 @@ private SearchExecutionContext( null, null, null, + false, false ); this.shardId = shardId; diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 551217aeb7bea..8997741ed6250 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -1853,7 +1853,8 @@ public QueryRewriteContext getRewriteContext( ResolvedIndices resolvedIndices, PointInTimeBuilder pit, final Boolean ccsMinimizeRoundTrips, - final boolean isExplain + final boolean isExplain, + final boolean isProfile ) { return new QueryRewriteContext( parserConfig, @@ -1865,7 +1866,8 @@ public QueryRewriteContext getRewriteContext( pit, queryRewriteInterceptor, ccsMinimizeRoundTrips, - isExplain + isExplain, + isProfile ); } diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 4305e5cf15035..b6598cd112e42 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -2158,7 +2158,7 @@ public QueryRewriteContext getRewriteContext( PointInTimeBuilder pit, final Boolean ccsMinimizeRoundTrips ) { - return getRewriteContext(nowInMillis, minTransportVersion, clusterAlias, resolvedIndices, pit, ccsMinimizeRoundTrips, false); + return getRewriteContext(nowInMillis, minTransportVersion, clusterAlias, resolvedIndices, pit, ccsMinimizeRoundTrips, false, false); } /** @@ -2171,7 +2171,8 @@ public QueryRewriteContext getRewriteContext( ResolvedIndices resolvedIndices, PointInTimeBuilder pit, final Boolean ccsMinimizeRoundTrips, - final boolean isExplain + final boolean isExplain, + final boolean isProfile ) { return indicesService.getRewriteContext( nowInMillis, @@ -2180,7 +2181,8 @@ public QueryRewriteContext getRewriteContext( resolvedIndices, pit, ccsMinimizeRoundTrips, - isExplain + isExplain, + isProfile ); } diff --git a/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java index 8042be444292d..e17ca2870a72d 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java @@ -191,7 +191,8 @@ public void onResponse(MultiSearchResponse items) { failures.forEach(ex::addSuppressed); listener.onFailure(ex); } else { - results.set(combineInnerRetrieverResults(topDocs, ctx.isExplain())); + boolean enrichResults = ctx.isExplain() || ctx.isProfile(); + results.set(combineInnerRetrieverResults(topDocs, enrichResults)); listener.onResponse(null); } } diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java index a99b724e06936..2a5945296d166 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java @@ -1784,7 +1784,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { NodeClient client = new NodeClient(settings, threadPool, TestProjectResolvers.alwaysThrow()); SearchService searchService = mock(SearchService.class); - when(searchService.getRewriteContext(any(), any(), any(), any(), any(), anyBoolean(), anyBoolean())).thenReturn( + when(searchService.getRewriteContext(any(), any(), any(), any(), any(), anyBoolean(), anyBoolean(), anyBoolean())).thenReturn( new QueryRewriteContext(null, null, null, null, null, null, null, null, null) ); ClusterService clusterService = new ClusterService( diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java index 0cc92ae4f16e6..b27552075f4d2 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java @@ -57,6 +57,7 @@ public void testGetTierPreference() { null, null, null, + false, false ); @@ -89,6 +90,7 @@ public void testGetTierPreference() { null, null, null, + false, false ); diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 644e0e711abb4..fe60bbe6253ea 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -641,6 +641,7 @@ QueryRewriteContext createQueryRewriteContext() { null, createMockQueryRewriteInterceptor(), null, + false, false ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/ChunkScorerConfig.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/ChunkScorerConfig.java index 94e30c6fcac6b..ef3f595e14e33 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/ChunkScorerConfig.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/ChunkScorerConfig.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.inference.rank.textsimilarity; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -101,4 +102,13 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(size, inferenceText, chunkingSettings); } + + @Override + public String toString() { + return "ChunkScorerConfig{" + + "size=" + sizeOrDefault() + + ", inferenceText=[" + inferenceText + ']' + + ", chunkingSettings=" + chunkingSettings + + "}"; + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java index d28b76a23f07e..ac9b2b16e455e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java @@ -187,7 +187,7 @@ protected TextSimilarityRankRetrieverBuilder clone( } @Override - protected RankDoc[] combineInnerRetrieverResults(List rankResults, boolean explain) { + protected RankDoc[] combineInnerRetrieverResults(List rankResults, boolean enrichResults) { assert rankResults.size() == 1; ScoreDoc[] scoreDocs = rankResults.getFirst(); List filteredDocs = new ArrayList<>(); @@ -197,7 +197,7 @@ protected RankDoc[] combineInnerRetrieverResults(List rankResults, b ScoreDoc scoreDoc = scoreDocs[i]; assert scoreDoc.score >= 0; if (minScore == null || scoreDoc.score >= minScore) { - if (explain) { + if (enrichResults) { filteredDocs.add( new TextSimilarityRankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex, inferenceId, field, chunkScorerConfig) ); diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml index 8e67141bcddb5..91ed1ebf92e72 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml @@ -920,3 +920,67 @@ setup: - length: { hits.hits: 2 } - match: { hits.hits.0._explanation.details.0.description: "text_similarity_reranker match using inference endpoint: [my-rerank-model] on document field: [semantic_text_field] matching on source query and rescoring considering only top [1] best chunks" } + +--- +"Profile outputs information on chunk rescorer if selected": + + - requires: + test_runner_features: [contains] + cluster_features: "text_similarity_rank_doc_explain_chunks" + reason: rescore_chunks added explain support in 9.3.0 + + - do: + search: + index: test-index + body: + track_total_hits: true + profile: true + fields: [ "text", "semantic_text_field", "topic" ] + retriever: + text_similarity_reranker: + retriever: + standard: + query: + match: + topic: + query: "science" + rank_window_size: 10 + inference_id: my-rerank-model + inference_text: "how often does the moon hide the sun?" + field: semantic_text_field + size: 10 + + - match: { hits.total.value: 2 } + - length: { hits.hits: 2 } + + - contains: { profile.shards.0.searches.0.query.0.description: "+RankDocsQuery{rank_docs:[TextSimilarityRankDoc" } + - contains: { profile.shards.0.searches.0.query.0.description: "inferenceId=my-rerank-model, field=semantic_text_field, chunkScorerConfig=null}" } + + - do: + search: + index: test-index + body: + track_total_hits: true + profile: true + fields: [ "text", "semantic_text_field", "topic" ] + retriever: + text_similarity_reranker: + retriever: + standard: + query: + match: + topic: + query: "science" + rank_window_size: 10 + inference_id: my-rerank-model + inference_text: "how often does the moon hide the sun?" + field: semantic_text_field + chunk_rescorer: { } + size: 10 + + - match: { hits.total.value: 2 } + - length: { hits.hits: 2 } + + - contains: { profile.shards.0.searches.0.query.0.description: "+RankDocsQuery{rank_docs:[TextSimilarityRankDoc" } + - contains: { profile.shards.0.searches.0.query.0.description: "inferenceId=my-rerank-model, field=semantic_text_field, chunkScorerConfig=ChunkScorerConfig{size=1, inferenceText=[null], chunkingSettings=" } + From 8ca23e108912fc3271b3f2be05921255195bd890 Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Tue, 28 Oct 2025 11:08:03 -0400 Subject: [PATCH 5/8] Update 137249.yaml --- docs/changelog/137249.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/137249.yaml b/docs/changelog/137249.yaml index 35c18b5db1552..e153281de4fa9 100644 --- a/docs/changelog/137249.yaml +++ b/docs/changelog/137249.yaml @@ -1,5 +1,5 @@ pr: 137249 -summary: Add `chunk_rescorer` usage to output of explain for `text_similarity_rank_retriever` +summary: Add `chunk_rescorer` usage to output of explain and profile for `text_similarity_rank_retriever` area: Relevance type: enhancement issues: [] From 683f08f191a7588a94e09ca221be93dc6e8515d0 Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Tue, 28 Oct 2025 11:15:47 -0400 Subject: [PATCH 6/8] That's what you get for resolving conflicts in the UI, fixed compile failure --- .../rank/textsimilarity/ChunkScorerConfig.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/ChunkScorerConfig.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/ChunkScorerConfig.java index f85418aaad0c0..cc0ecb72f7ae3 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/ChunkScorerConfig.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/ChunkScorerConfig.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.rank.textsimilarity; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -107,12 +106,10 @@ public int hashCode() { @Override public String toString() { - return "ChunkScorerConfig{" - + "size=" + sizeOrDefault() - + ", inferenceText=[" + inferenceText + ']' - + ", chunkingSettings=" + chunkingSettings - + "}"; - + return "ChunkScorerConfig{" + "size=" + sizeOrDefault() + ", inferenceText=[" + inferenceText + ']' + ", chunkingSettings=" + + chunkingSettings + "}"; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); From 6501e6951d8d9eb1e3ccdd974edbe7fde004e59b Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 28 Oct 2025 15:22:59 +0000 Subject: [PATCH 7/8] [CI] Auto commit changes from spotless --- .../rank/textsimilarity/ChunkScorerConfig.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/ChunkScorerConfig.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/ChunkScorerConfig.java index cc0ecb72f7ae3..c7d57bfae25d2 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/ChunkScorerConfig.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/ChunkScorerConfig.java @@ -106,8 +106,15 @@ public int hashCode() { @Override public String toString() { - return "ChunkScorerConfig{" + "size=" + sizeOrDefault() + ", inferenceText=[" + inferenceText + ']' + ", chunkingSettings=" - + chunkingSettings + "}"; + return "ChunkScorerConfig{" + + "size=" + + sizeOrDefault() + + ", inferenceText=[" + + inferenceText + + ']' + + ", chunkingSettings=" + + chunkingSettings + + "}"; } @Override From 789429c0a872df2c3418359141557d633ccd02a7 Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Tue, 28 Oct 2025 12:22:42 -0400 Subject: [PATCH 8/8] Fix test compilation --- .../AbstractInterceptedInferenceQueryBuilderTestCase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/AbstractInterceptedInferenceQueryBuilderTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/AbstractInterceptedInferenceQueryBuilderTestCase.java index 169ae6767303d..9ae5e3dcc591d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/AbstractInterceptedInferenceQueryBuilderTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/AbstractInterceptedInferenceQueryBuilderTestCase.java @@ -514,6 +514,7 @@ protected QueryRewriteContext createIndexMetadataContext( null, null, null, + false, false ); }