Rework RRF to be evaluated during rewrite phase

docs/reference/search/rrf.asciidoc

@@ -300,13 +300,12 @@ We have both the ranker's `score` and the `_rank` option to show our top-ranked
             "value" : 5,
             "relation" : "eq"
         },
-        "max_score" : null,
+        "max_score" : ...,
         "hits" : [
             {
                 "_index" : "example-index",
                 "_id" : "3",
                 "_score" : 0.8333334,
-                "_rank" : 1,
                 "_source" : {
                     "integer" : 1,
                     "vector" : [
@@ -319,7 +318,6 @@ We have both the ranker's `score` and the `_rank` option to show our top-ranked
                 "_index" : "example-index",
                 "_id" : "2",
                 "_score" : 0.5833334,
-                "_rank" : 2,
                 "_source" : {
                     "integer" : 2,
                     "vector" : [
@@ -332,7 +330,6 @@ We have both the ranker's `score` and the `_rank` option to show our top-ranked
                 "_index" : "example-index",
                 "_id" : "4",
                 "_score" : 0.5,
-                "_rank" : 3,
                 "_source" : {
                     "integer" : 2,
                     "text" : "rrf rrf rrf rrf"
@@ -499,7 +496,6 @@ Working with the example above, and by adding `explain=true` to the search reque
             "_index": "example-index",
             "_id": "3",
             "_score": 0.8333334,
-            "_rank": 1,
             "_explanation":
             {
                 "value": 0.8333334,                                                                                                                                               <1>
@@ -608,7 +604,6 @@ The response would now include the named query in the explanation:
             "_index": "example-index",
             "_id": "3",
             "_score": 0.8333334,
-            "_rank": 1,
             "_explanation":
             {
                 "value": 0.8333334,

server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java

@@ -36,6 +36,7 @@
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.NestedSortBuilder;
 import org.elasticsearch.search.sort.ScoreSortBuilder;
+import org.elasticsearch.search.sort.ShardDocSortField;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -205,8 +206,8 @@ public void testRankDocsRetrieverBasicWithPagination() {
             null
         );
         // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and
-        // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results)
-        // so ideal rank would be: 6, 2, 1, 4, 7, 3 and with pagination, we'd just omit the first result
+        // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results)
+        // so ideal rank would be: 6, 2, 1, 3, 4, 7 and with pagination, we'd just omit the first result
         source.retriever(
             new CompoundRetrieverWithRankDocs(
                 rankWindowSize,
@@ -227,9 +228,9 @@ public void testRankDocsRetrieverBasicWithPagination() {
             assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2"));
             assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1"));
-            assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_4"));
-            assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_7"));
-            assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_3"));
+            assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_3"));
+            assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_4"));
+            assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_7"));
         });
     }
 
@@ -267,13 +268,15 @@ public void testRankDocsRetrieverWithAggs() {
                 )
             )
         );
+        source.size(1);
         source.aggregation(new TermsAggregationBuilder("topic").field(TOPIC_FIELD));
         SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source);
         ElasticsearchAssertions.assertResponse(req, resp -> {
             assertNull(resp.pointInTimeId());
             assertNotNull(resp.getHits().getTotalHits());
-            assertThat(resp.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(resp.getHits().getTotalHits().value, equalTo(5L));
             assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getHits().length, equalTo(1));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2"));
             assertNotNull(resp.getAggregations());
             assertNotNull(resp.getAggregations().get("topic"));
@@ -307,8 +310,8 @@ public void testRankDocsRetrieverWithCollapse() {
             null
         );
         // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and
-        // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results)
-        // so ideal rank would be: 6, 2, 1, 4, 7, 3
+        // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results)
+        // so ideal rank would be: 6, 2, 1, 3, 4, 7
         // with collapsing on topic field we would have 6, 2, 1, 7
         source.retriever(
             new CompoundRetrieverWithRankDocs(
@@ -338,7 +341,6 @@ public void testRankDocsRetrieverWithCollapse() {
             assertThat(resp.getHits().getAt(1).field(TOPIC_FIELD).getValue().toString(), equalTo("astronomy"));
             assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_1"));
             assertThat(resp.getHits().getAt(2).field(TOPIC_FIELD).getValue().toString(), equalTo("technology"));
-            assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getHits().length, equalTo(3));
             assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(0).getId(), equalTo("doc_4"));
             assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(1).getId(), equalTo("doc_3"));
             assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(2).getId(), equalTo("doc_1"));
@@ -347,11 +349,7 @@ public void testRankDocsRetrieverWithCollapse() {
         });
     }
 
-    public void testRankDocsRetrieverWithCollapseAndAggs() {
-        // same as above, but we only want to bring back the top result from each subsearch
-        // so that would be 1, 2, and 7
-        // and final rank would be (based on score): 2, 1, 7
-        // aggs should still account for the same docs as the testRankDocsRetriever test, i.e. all but doc_5
+    public void testRankDocsRetrieverWithNestedCollapseAndAggs() {
         final int rankWindowSize = 10;
         SearchSourceBuilder source = new SearchSourceBuilder();
         StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder();
@@ -375,8 +373,8 @@ public void testRankDocsRetrieverWithCollapseAndAggs() {
             null
         );
         // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and
-        // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results)
-        // so ideal rank would be: 6, 2, 1, 4, 7, 3
+        // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results)
+        // so ideal rank would be: 6, 2, 1, 3, 4, 7
         source.retriever(
             new CompoundRetrieverWithRankDocs(
                 rankWindowSize,
@@ -392,7 +390,7 @@ public void testRankDocsRetrieverWithCollapseAndAggs() {
         ElasticsearchAssertions.assertResponse(req, resp -> {
             assertNull(resp.pointInTimeId());
             assertNotNull(resp.getHits().getTotalHits());
-            assertThat(resp.getHits().getTotalHits().value, equalTo(5L));
+            assertThat(resp.getHits().getTotalHits().value, equalTo(6L));
             assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6"));
             assertNotNull(resp.getAggregations());
@@ -427,8 +425,8 @@ public void testRankDocsRetrieverWithNestedQuery() {
             null
         );
         // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and
-        // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results)
-        // so ideal rank would be: 6, 2, 1, 4, 3, 7
+        // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results)
+        // so ideal rank would be: 6, 2, 1, 3, 4, 7
         source.retriever(
             new CompoundRetrieverWithRankDocs(
                 rankWindowSize,
@@ -506,11 +504,11 @@ public void testRankDocsRetrieverMultipleCompoundRetrievers() {
             assertThat(resp.getHits().getTotalHits().value, equalTo(6L));
             assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_4"));
-            assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_6"));
+            assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1"));
             assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_2"));
-            assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_1"));
-            assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_7"));
-            assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_3"));
+            assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_3"));
+            assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_6"));
+            assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_7"));
         });
     }
 
@@ -545,9 +543,9 @@ public void testRankDocsRetrieverDifferentNestedSorting() {
             assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_4"));
             assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1"));
-            assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_7"));
+            assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_2"));
             assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_6"));
-            assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_2"));
+            assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_7"));
         });
     }
 
@@ -673,22 +671,14 @@ private RankDoc[] getRankDocs(SearchResponse searchResponse) {
             for (int i = 0; i < size; i++) {
                 var hit = searchResponse.getHits().getAt(i);
                 long sortValue = (long) hit.getRawSortValues()[hit.getRawSortValues().length - 1];
-                int doc = decodeDoc(sortValue);
-                int shardRequestIndex = decodeShardRequestIndex(sortValue);
+                int doc = ShardDocSortField.decodeDoc(sortValue);
+                int shardRequestIndex = ShardDocSortField.decodeShardRequestIndex(sortValue);
                 docs[i] = new RankDoc(doc, hit.getScore(), shardRequestIndex);
                 docs[i].rank = i + 1;
             }
             return docs;
         }
 
-        public static int decodeDoc(long value) {
-            return (int) value;
-        }
-
-        public static int decodeShardRequestIndex(long value) {
-            return (int) (value >> 32);
-        }
-
         record RankDocAndHitRatio(RankDoc rankDoc, float hitRatio) {}
 
         /**

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -225,6 +225,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ILM_ADD_SEARCHABLE_SNAPSHOT_TOTAL_SHARDS_PER_NODE = def(8_749_00_0);
     public static final TransportVersion SEMANTIC_TEXT_SEARCH_INFERENCE_ID = def(8_750_00_0);
     public static final TransportVersion ML_INFERENCE_CHUNKING_SETTINGS = def(8_751_00_0);
+    public static final TransportVersion RRF_QUERY_REWRITE = def(8_752_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java

@@ -502,6 +502,8 @@ void executeRequest(
         });
         final SearchSourceBuilder source = original.source();
         if (shouldOpenPIT(source)) {
+            // disabling shard reordering for request
+            original.setPreFilterShardSize(Integer.MAX_VALUE);
             openPIT(client, original, searchService.getDefaultKeepAliveInMillis(), listener.delegateFailureAndWrap((delegate, resp) -> {
                 // We set the keep alive to -1 to indicate that we don't need the pit id in the response.
                 // This is needed since we delete the pit prior to sending the response so the id doesn't exist anymore.

server/src/main/java/org/elasticsearch/common/lucene/Lucene.java

@@ -74,7 +74,6 @@
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.lucene.grouping.TopFieldGroups;
-import org.elasticsearch.search.retriever.rankdoc.RankDocsSortField;
 import org.elasticsearch.search.sort.ShardDocSortField;
 
 import java.io.IOException;
@@ -553,8 +552,6 @@ private static SortField rewriteMergeSortField(SortField sortField) {
             return newSortField;
         } else if (sortField.getClass() == ShardDocSortField.class) {
             return new SortField(sortField.getField(), SortField.Type.LONG, sortField.getReverse());
-        } else if (sortField.getClass() == RankDocsSortField.class) {
-            return new SortField(sortField.getField(), SortField.Type.INT, sortField.getReverse());
         } else {
             return sortField;
         }

server/src/main/java/org/elasticsearch/search/SearchModule.java

@@ -239,7 +239,6 @@
 import org.elasticsearch.search.retriever.RetrieverParserContext;
 import org.elasticsearch.search.retriever.StandardRetrieverBuilder;
 import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder;
-import org.elasticsearch.search.retriever.rankdoc.RankDocsSortBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
 import org.elasticsearch.search.sort.ScoreSortBuilder;
@@ -868,7 +867,6 @@ private void registerSorts() {
         namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, ScoreSortBuilder.NAME, ScoreSortBuilder::new));
         namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, ScriptSortBuilder.NAME, ScriptSortBuilder::new));
         namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, FieldSortBuilder.NAME, FieldSortBuilder::new));
-        namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, RankDocsSortBuilder.NAME, RankDocsSortBuilder::new));
     }
 
     private static <T> void registerFromPlugin(List<SearchPlugin> plugins, Function<SearchPlugin, List<T>> producer, Consumer<T> consumer) {

server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java

@@ -2207,12 +2207,7 @@ public ActionRequestValidationException validate(
         boolean allowPartialSearchResults
     ) {
         if (retriever() != null) {
-            if (allowPartialSearchResults && retriever().isCompound()) {
-                validationException = addValidationError(
-                    "cannot specify a compound retriever and [allow_partial_search_results]",
-                    validationException
-                );
-            }
+            validationException = retriever().validate(this, validationException, allowPartialSearchResults);
             List<String> specified = new ArrayList<>();
             if (subSearches().isEmpty() == false) {
                 specified.add(QUERY_FIELD.getPreferredName());
@@ -2229,9 +2224,6 @@ public ActionRequestValidationException validate(
             if (sorts() != null) {
                 specified.add(SORT_FIELD.getPreferredName());
             }
-            if (minScore() != null) {
-                specified.add(MIN_SCORE_FIELD.getPreferredName());
-            }
             if (rankBuilder() != null) {
                 specified.add(RANK_FIELD.getPreferredName());
             }
@@ -2331,21 +2323,10 @@ public ActionRequestValidationException validate(
             if (rescores() != null && rescores().isEmpty() == false) {
                 validationException = addValidationError("[rank] cannot be used with [rescore]", validationException);
             }
-            if (sorts() != null && sorts().isEmpty() == false) {
-                validationException = addValidationError("[rank] cannot be used with [sort]", validationException);
-            }
-            if (collapse() != null) {
-                validationException = addValidationError("[rank] cannot be used with [collapse]", validationException);
-            }
+
             if (suggest() != null && suggest().getSuggestions().isEmpty() == false) {
                 validationException = addValidationError("[rank] cannot be used with [suggest]", validationException);
             }
-            if (highlighter() != null) {
-                validationException = addValidationError("[rank] cannot be used with [highlighter]", validationException);
-            }
-            if (pointInTimeBuilder() != null) {
-                validationException = addValidationError("[rank] cannot be used with [point in time]", validationException);
-            }
         }
 
         if (rescores() != null) {

server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsContext.java

@@ -34,7 +34,7 @@ public class StoredFieldsContext implements Writeable {
     private final List<String> fieldNames;
     private final boolean fetchFields;
 
-    private StoredFieldsContext(boolean fetchFields) {
+    public StoredFieldsContext(boolean fetchFields) {
         this.fetchFields = fetchFields;
         this.fieldNames = null;
     }

server/src/main/java/org/elasticsearch/search/fetch/subphase/ExplainPhase.java

@@ -10,6 +10,7 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Explanation;
+import org.elasticsearch.index.mapper.NestedLookup;
 import org.elasticsearch.search.fetch.FetchContext;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.fetch.FetchSubPhaseProcessor;
@@ -45,8 +46,17 @@ public void process(HitContext hitContext) throws IOException {
                 for (RescoreContext rescore : context.rescore()) {
                     explanation = rescore.rescorer().explain(topLevelDocId, context.searcher(), rescore, explanation);
                 }
+
                 if (context.rankBuilder() != null) {
-                    explanation = context.rankBuilder().explainHit(explanation, hitContext.rankDoc(), queryNames);
+                    // if we have nested fields, then the query is wrapped using an additional filter on the _primary_term field
+                    // through the DefaultSearchContext#buildFilteredQuery so we have to extract the actual query
+                    if (context.getSearchExecutionContext().nestedLookup() != NestedLookup.EMPTY) {
+                        explanation = explanation.getDetails()[0];
+                    }
+
+                    if (context.rankBuilder() != null) {
+                        explanation = context.rankBuilder().explainHit(explanation, hitContext.rankDoc(), queryNames);
+                    }
                 }
                 // we use the top level doc id, since we work with the top level searcher
                 hitContext.hit().explanation(explanation);