Skip to content

Commit 7d33c5c

Browse files
pmpailispmpailis
authored
[8.x] Backporting propagating nested inner_hits to the parent compound retriever (#116707)
1 parent 7668eee commit 7d33c5c

File tree

19 files changed

+248
-42
lines changed

19 files changed

+248
-42
lines changed

‎docs/changelog/116408.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 116408
2+
summary: Propagating nested `inner_hits` to the parent compound retriever
3+
area: Ranking
4+
type: bug
5+
issues:
6+
- 116397

‎server/src/internalClusterTest/java/org/elasticsearch/search/nested/SimpleNestedIT.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@
2121
import org.elasticsearch.cluster.health.ClusterHealthStatus;
2222
import org.elasticsearch.common.settings.Settings;
2323
import org.elasticsearch.core.TimeValue;
24+
import org.elasticsearch.index.query.InnerHitBuilder;
2425
import org.elasticsearch.index.query.QueryBuilders;
26+
import org.elasticsearch.search.builder.SearchSourceBuilder;
2527
import org.elasticsearch.search.sort.NestedSortBuilder;
2628
import org.elasticsearch.search.sort.SortBuilders;
2729
import org.elasticsearch.search.sort.SortMode;
@@ -1581,6 +1583,64 @@ public void testCheckFixedBitSetCache() throws Exception {
15811583
assertThat(clusterStatsResponse.getIndicesStats().getSegments().getBitsetMemoryInBytes(), equalTo(0L));
15821584
}
15831585

1586+
public void testSkipNestedInnerHits() throws Exception {
1587+
assertAcked(prepareCreate("test").setMapping("nested1", "type=nested"));
1588+
ensureGreen();
1589+
1590+
prepareIndex("test").setId("1")
1591+
.setSource(
1592+
jsonBuilder().startObject()
1593+
.field("field1", "value1")
1594+
.startArray("nested1")
1595+
.startObject()
1596+
.field("n_field1", "foo")
1597+
.field("n_field2", "bar")
1598+
.endObject()
1599+
.endArray()
1600+
.endObject()
1601+
)
1602+
.get();
1603+
1604+
waitForRelocation(ClusterHealthStatus.GREEN);
1605+
GetResponse getResponse = client().prepareGet("test", "1").get();
1606+
assertThat(getResponse.isExists(), equalTo(true));
1607+
assertThat(getResponse.getSourceAsBytesRef(), notNullValue());
1608+
refresh();
1609+
1610+
assertNoFailuresAndResponse(
1611+
prepareSearch("test").setSource(
1612+
new SearchSourceBuilder().query(
1613+
QueryBuilders.nestedQuery("nested1", QueryBuilders.termQuery("nested1.n_field1", "foo"), ScoreMode.Avg)
1614+
.innerHit(new InnerHitBuilder())
1615+
)
1616+
),
1617+
res -> {
1618+
assertNotNull(res.getHits());
1619+
assertHitCount(res, 1);
1620+
assertThat(res.getHits().getHits().length, equalTo(1));
1621+
// by default we should get inner hits
1622+
assertNotNull(res.getHits().getHits()[0].getInnerHits());
1623+
assertNotNull(res.getHits().getHits()[0].getInnerHits().get("nested1"));
1624+
}
1625+
);
1626+
1627+
assertNoFailuresAndResponse(
1628+
prepareSearch("test").setSource(
1629+
new SearchSourceBuilder().query(
1630+
QueryBuilders.nestedQuery("nested1", QueryBuilders.termQuery("nested1.n_field1", "foo"), ScoreMode.Avg)
1631+
.innerHit(new InnerHitBuilder())
1632+
).skipInnerHits(true)
1633+
),
1634+
res -> {
1635+
assertNotNull(res.getHits());
1636+
assertHitCount(res, 1);
1637+
assertThat(res.getHits().getHits().length, equalTo(1));
1638+
// if we explicitly say to ignore inner hits, then this should now be null
1639+
assertNull(res.getHits().getHits()[0].getInnerHits());
1640+
}
1641+
);
1642+
}
1643+
15841644
private void assertDocumentCount(String index, long numdocs) {
15851645
IndicesStatsResponse stats = indicesAdmin().prepareStats(index).clear().setDocs(true).get();
15861646
assertNoFailures(stats);

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ static TransportVersion def(int id) {
194194
public static final TransportVersion DATA_STREAM_INDEX_VERSION_DEPRECATION_CHECK = def(8_788_00_0);
195195
public static final TransportVersion ADD_COMPATIBILITY_VERSIONS_TO_NODE_INFO = def(8_789_00_0);
196196
public static final TransportVersion VERTEX_AI_INPUT_TYPE_ADDED = def(8_790_00_0);
197-
197+
public static final TransportVersion SKIP_INNER_HITS_SEARCH_SOURCE = def(8_791_00_0);
198198
/*
199199
* STOP! READ THIS FIRST! No, really,
200200
* ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _

‎server/src/main/java/org/elasticsearch/search/retriever/rankdoc/RankDocsQueryBuilder.java‎ renamed to ‎server/src/main/java/org/elasticsearch/index/query/RankDocsQueryBuilder.java‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
* License v3.0 only", or the "Server Side Public License, v 1".
88
*/
99

10-
package org.elasticsearch.search.retriever.rankdoc;
10+
package org.elasticsearch.index.query;
1111

1212
import org.apache.lucene.index.IndexReader;
1313
import org.apache.lucene.search.Query;
@@ -16,15 +16,13 @@
1616
import org.elasticsearch.common.Strings;
1717
import org.elasticsearch.common.io.stream.StreamInput;
1818
import org.elasticsearch.common.io.stream.StreamOutput;
19-
import org.elasticsearch.index.query.AbstractQueryBuilder;
20-
import org.elasticsearch.index.query.QueryBuilder;
21-
import org.elasticsearch.index.query.QueryRewriteContext;
22-
import org.elasticsearch.index.query.SearchExecutionContext;
2319
import org.elasticsearch.search.rank.RankDoc;
20+
import org.elasticsearch.search.retriever.rankdoc.RankDocsQuery;
2421
import org.elasticsearch.xcontent.XContentBuilder;
2522

2623
import java.io.IOException;
2724
import java.util.Arrays;
25+
import java.util.Map;
2826
import java.util.Objects;
2927

3028
import static org.elasticsearch.TransportVersions.RRF_QUERY_REWRITE;
@@ -55,6 +53,15 @@ public RankDocsQueryBuilder(StreamInput in) throws IOException {
5553
}
5654
}
5755

56+
@Override
57+
protected void extractInnerHitBuilders(Map<String, InnerHitContextBuilder> innerHits) {
58+
if (queryBuilders != null) {
59+
for (QueryBuilder query : queryBuilders) {
60+
InnerHitContextBuilder.extractInnerHits(query, innerHits);
61+
}
62+
}
63+
}
64+
5865
@Override
5966
protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
6067
if (queryBuilders != null) {
@@ -71,7 +78,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws
7178
return super.doRewrite(queryRewriteContext);
7279
}
7380

74-
RankDoc[] rankDocs() {
81+
public RankDoc[] rankDocs() {
7582
return rankDocs;
7683
}
7784

‎server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ private SearchCapabilities() {}
3434
private static final String KQL_QUERY_SUPPORTED = "kql_query";
3535
/** Support multi-dense-vector field mapper. */
3636
private static final String MULTI_DENSE_VECTOR_FIELD_MAPPER = "multi_dense_vector_field_mapper";
37+
/** Support propagating nested retrievers' inner_hits to top-level compound retrievers . */
38+
private static final String NESTED_RETRIEVER_INNER_HITS_SUPPORT = "nested_retriever_inner_hits_support";
3739

3840
public static final Set<String> CAPABILITIES;
3941
static {
@@ -42,6 +44,7 @@ private SearchCapabilities() {}
4244
capabilities.add(BIT_DENSE_VECTOR_SYNTHETIC_SOURCE_CAPABILITY);
4345
capabilities.add(BYTE_FLOAT_BIT_DOT_PRODUCT_CAPABILITY);
4446
capabilities.add(DENSE_VECTOR_DOCVALUE_FIELDS);
47+
capabilities.add(NESTED_RETRIEVER_INNER_HITS_SUPPORT);
4548
if (MultiDenseVectorFieldMapper.FEATURE_FLAG.isEnabled()) {
4649
capabilities.add(MULTI_DENSE_VECTOR_FIELD_MAPPER);
4750
}

‎server/src/main/java/org/elasticsearch/search/SearchModule.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
import org.elasticsearch.index.query.QueryBuilder;
5353
import org.elasticsearch.index.query.QueryStringQueryBuilder;
5454
import org.elasticsearch.index.query.RangeQueryBuilder;
55+
import org.elasticsearch.index.query.RankDocsQueryBuilder;
5556
import org.elasticsearch.index.query.RegexpQueryBuilder;
5657
import org.elasticsearch.index.query.ScriptQueryBuilder;
5758
import org.elasticsearch.index.query.SimpleQueryStringBuilder;
@@ -238,7 +239,6 @@
238239
import org.elasticsearch.search.retriever.RetrieverBuilder;
239240
import org.elasticsearch.search.retriever.RetrieverParserContext;
240241
import org.elasticsearch.search.retriever.StandardRetrieverBuilder;
241-
import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder;
242242
import org.elasticsearch.search.sort.FieldSortBuilder;
243243
import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
244244
import org.elasticsearch.search.sort.ScoreSortBuilder;

‎server/src/main/java/org/elasticsearch/search/SearchService.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1300,13 +1300,17 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc
13001300
);
13011301
if (query != null) {
13021302
QueryBuilder rewrittenForInnerHits = Rewriteable.rewrite(query, innerHitsRewriteContext, true);
1303-
InnerHitContextBuilder.extractInnerHits(rewrittenForInnerHits, innerHitBuilders);
1303+
if (false == source.skipInnerHits()) {
1304+
InnerHitContextBuilder.extractInnerHits(rewrittenForInnerHits, innerHitBuilders);
1305+
}
13041306
searchExecutionContext.setAliasFilter(context.request().getAliasFilter().getQueryBuilder());
13051307
context.parsedQuery(searchExecutionContext.toQuery(query));
13061308
}
13071309
if (source.postFilter() != null) {
13081310
QueryBuilder rewrittenForInnerHits = Rewriteable.rewrite(source.postFilter(), innerHitsRewriteContext, true);
1309-
InnerHitContextBuilder.extractInnerHits(rewrittenForInnerHits, innerHitBuilders);
1311+
if (false == source.skipInnerHits()) {
1312+
InnerHitContextBuilder.extractInnerHits(rewrittenForInnerHits, innerHitBuilders);
1313+
}
13101314
context.parsedPostFilter(searchExecutionContext.toQuery(source.postFilter()));
13111315
}
13121316
if (innerHitBuilders.size() > 0) {

‎server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,8 @@ public static HighlightBuilder highlight() {
213213

214214
private Map<String, Object> runtimeMappings = emptyMap();
215215

216+
private boolean skipInnerHits = false;
217+
216218
/**
217219
* Constructs a new search source builder.
218220
*/
@@ -289,6 +291,11 @@ public SearchSourceBuilder(StreamInput in) throws IOException {
289291
if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) {
290292
rankBuilder = in.readOptionalNamedWriteable(RankBuilder.class);
291293
}
294+
if (in.getTransportVersion().onOrAfter(TransportVersions.SKIP_INNER_HITS_SEARCH_SOURCE)) {
295+
skipInnerHits = in.readBoolean();
296+
} else {
297+
skipInnerHits = false;
298+
}
292299
}
293300

294301
@Override
@@ -378,6 +385,9 @@ public void writeTo(StreamOutput out) throws IOException {
378385
} else if (rankBuilder != null) {
379386
throw new IllegalArgumentException("cannot serialize [rank] to version [" + out.getTransportVersion().toReleaseVersion() + "]");
380387
}
388+
if (out.getTransportVersion().onOrAfter(TransportVersions.SKIP_INNER_HITS_SEARCH_SOURCE)) {
389+
out.writeBoolean(skipInnerHits);
390+
}
381391
}
382392

383393
/**
@@ -1279,6 +1289,7 @@ private SearchSourceBuilder shallowCopy(
12791289
rewrittenBuilder.collapse = collapse;
12801290
rewrittenBuilder.pointInTimeBuilder = pointInTimeBuilder;
12811291
rewrittenBuilder.runtimeMappings = runtimeMappings;
1292+
rewrittenBuilder.skipInnerHits = skipInnerHits;
12821293
return rewrittenBuilder;
12831294
}
12841295

@@ -1855,6 +1866,9 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
18551866
if (false == runtimeMappings.isEmpty()) {
18561867
builder.field(RUNTIME_MAPPINGS_FIELD.getPreferredName(), runtimeMappings);
18571868
}
1869+
if (skipInnerHits) {
1870+
builder.field("skipInnerHits", true);
1871+
}
18581872

18591873
return builder;
18601874
}
@@ -1867,6 +1881,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
18671881
return builder;
18681882
}
18691883

1884+
public SearchSourceBuilder skipInnerHits(boolean skipInnerHits) {
1885+
this.skipInnerHits = skipInnerHits;
1886+
return this;
1887+
}
1888+
1889+
public boolean skipInnerHits() {
1890+
return this.skipInnerHits;
1891+
}
1892+
18701893
public static class IndexBoost implements Writeable, ToXContentObject {
18711894
private final String index;
18721895
private final float boost;
@@ -2121,7 +2144,8 @@ public int hashCode() {
21212144
collapse,
21222145
trackTotalHitsUpTo,
21232146
pointInTimeBuilder,
2124-
runtimeMappings
2147+
runtimeMappings,
2148+
skipInnerHits
21252149
);
21262150
}
21272151

@@ -2166,7 +2190,8 @@ public boolean equals(Object obj) {
21662190
&& Objects.equals(collapse, other.collapse)
21672191
&& Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo)
21682192
&& Objects.equals(pointInTimeBuilder, other.pointInTimeBuilder)
2169-
&& Objects.equals(runtimeMappings, other.runtimeMappings);
2193+
&& Objects.equals(runtimeMappings, other.runtimeMappings)
2194+
&& Objects.equals(skipInnerHits, other.skipInnerHits);
21702195
}
21712196

21722197
@Override

‎server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ public int doHashCode() {
236236
return Objects.hash(innerRetrievers);
237237
}
238238

239-
protected SearchSourceBuilder createSearchSourceBuilder(PointInTimeBuilder pit, RetrieverBuilder retrieverBuilder) {
239+
protected final SearchSourceBuilder createSearchSourceBuilder(PointInTimeBuilder pit, RetrieverBuilder retrieverBuilder) {
240240
var sourceBuilder = new SearchSourceBuilder().pointInTimeBuilder(pit)
241241
.trackTotalHits(false)
242242
.storedFields(new StoredFieldsContext(false))
@@ -254,6 +254,11 @@ protected SearchSourceBuilder createSearchSourceBuilder(PointInTimeBuilder pit,
254254
}
255255
sortBuilders.add(new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME));
256256
sourceBuilder.sort(sortBuilders);
257+
sourceBuilder.skipInnerHits(true);
258+
return finalizeSourceBuilder(sourceBuilder);
259+
}
260+
261+
protected SearchSourceBuilder finalizeSourceBuilder(SearchSourceBuilder sourceBuilder) {
257262
return sourceBuilder;
258263
}
259264

‎server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
import org.elasticsearch.index.query.BoolQueryBuilder;
1616
import org.elasticsearch.index.query.QueryBuilder;
1717
import org.elasticsearch.index.query.QueryRewriteContext;
18+
import org.elasticsearch.index.query.RankDocsQueryBuilder;
1819
import org.elasticsearch.search.builder.SearchSourceBuilder;
19-
import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder;
2020
import org.elasticsearch.search.vectors.ExactKnnQueryBuilder;
2121
import org.elasticsearch.search.vectors.KnnSearchBuilder;
2222
import org.elasticsearch.search.vectors.QueryVectorBuilder;

0 commit comments

Comments
 (0)