Rework RRF to be evaluated during rewrite phase

docs/reference/search/rrf.asciidoc

@@ -300,13 +300,12 @@ We have both the ranker's `score` and the `_rank` option to show our top-ranked
             "value" : 5,
             "relation" : "eq"
         },
-        "max_score" : null,
+        "max_score" : ...,
         "hits" : [
             {
                 "_index" : "example-index",
                 "_id" : "3",
                 "_score" : 0.8333334,
-                "_rank" : 1,
                 "_source" : {
                     "integer" : 1,
                     "vector" : [
@@ -319,7 +318,6 @@ We have both the ranker's `score` and the `_rank` option to show our top-ranked
                 "_index" : "example-index",
                 "_id" : "2",
                 "_score" : 0.5833334,
-                "_rank" : 2,
                 "_source" : {
                     "integer" : 2,
                     "vector" : [
@@ -332,7 +330,6 @@ We have both the ranker's `score` and the `_rank` option to show our top-ranked
                 "_index" : "example-index",
                 "_id" : "4",
                 "_score" : 0.5,
-                "_rank" : 3,
                 "_source" : {
                     "integer" : 2,
                     "text" : "rrf rrf rrf rrf"
@@ -499,7 +496,6 @@ Working with the example above, and by adding `explain=true` to the search reque
             "_index": "example-index",
             "_id": "3",
             "_score": 0.8333334,
-            "_rank": 1,
             "_explanation":
             {
                 "value": 0.8333334,                                                                                                                                               <1>
@@ -608,7 +604,6 @@ The response would now include the named query in the explanation:
             "_index": "example-index",
             "_id": "3",
             "_score": 0.8333334,
-            "_rank": 1,
             "_explanation":
             {
                 "value": 0.8333334,

server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java

@@ -36,6 +36,7 @@
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.NestedSortBuilder;
 import org.elasticsearch.search.sort.ScoreSortBuilder;
+import org.elasticsearch.search.sort.ShardDocSortField;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -205,8 +206,8 @@ public void testRankDocsRetrieverBasicWithPagination() {
             null
         );
         // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and
-        // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results)
-        // so ideal rank would be: 6, 2, 1, 4, 7, 3 and with pagination, we'd just omit the first result
+        // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results)
+        // so ideal rank would be: 6, 2, 1, 3, 4, 7 and with pagination, we'd just omit the first result
         source.retriever(
             new CompoundRetrieverWithRankDocs(
                 rankWindowSize,
@@ -227,9 +228,9 @@ public void testRankDocsRetrieverBasicWithPagination() {
             assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2"));
             assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1"));
-            assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_4"));
-            assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_7"));
-            assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_3"));
+            assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_3"));
+            assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_4"));
+            assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_7"));
         });
     }
 
@@ -267,13 +268,15 @@ public void testRankDocsRetrieverWithAggs() {
                 )
             )
         );
+        source.size(1);
         source.aggregation(new TermsAggregationBuilder("topic").field(TOPIC_FIELD));
         SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source);
         ElasticsearchAssertions.assertResponse(req, resp -> {
             assertNull(resp.pointInTimeId());
             assertNotNull(resp.getHits().getTotalHits());
-            assertThat(resp.getHits().getTotalHits().value, equalTo(1L));
+            assertThat(resp.getHits().getTotalHits().value, equalTo(5L));
             assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
+            assertThat(resp.getHits().getHits().length, equalTo(1));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2"));
             assertNotNull(resp.getAggregations());
             assertNotNull(resp.getAggregations().get("topic"));
@@ -307,8 +310,8 @@ public void testRankDocsRetrieverWithCollapse() {
             null
         );
         // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and
-        // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results)
-        // so ideal rank would be: 6, 2, 1, 4, 7, 3
+        // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results)
+        // so ideal rank would be: 6, 2, 1, 3, 4, 7
         // with collapsing on topic field we would have 6, 2, 1, 7
         source.retriever(
             new CompoundRetrieverWithRankDocs(
@@ -347,11 +350,7 @@ public void testRankDocsRetrieverWithCollapse() {
         });
     }
 
-    public void testRankDocsRetrieverWithCollapseAndAggs() {
-        // same as above, but we only want to bring back the top result from each subsearch
-        // so that would be 1, 2, and 7
-        // and final rank would be (based on score): 2, 1, 7
-        // aggs should still account for the same docs as the testRankDocsRetriever test, i.e. all but doc_5
+    public void testRankDocsRetrieverWithNestedCollapseAndAggs() {
         final int rankWindowSize = 10;
         SearchSourceBuilder source = new SearchSourceBuilder();
         StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder();
@@ -375,8 +374,8 @@ public void testRankDocsRetrieverWithCollapseAndAggs() {
             null
         );
         // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and
-        // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results)
-        // so ideal rank would be: 6, 2, 1, 4, 7, 3
+        // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results)
+        // so ideal rank would be: 6, 2, 1, 3, 4, 7
         source.retriever(
             new CompoundRetrieverWithRankDocs(
                 rankWindowSize,
@@ -392,7 +391,7 @@ public void testRankDocsRetrieverWithCollapseAndAggs() {
         ElasticsearchAssertions.assertResponse(req, resp -> {
             assertNull(resp.pointInTimeId());
             assertNotNull(resp.getHits().getTotalHits());
-            assertThat(resp.getHits().getTotalHits().value, equalTo(5L));
+            assertThat(resp.getHits().getTotalHits().value, equalTo(6L));
             assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6"));
             assertNotNull(resp.getAggregations());
@@ -427,8 +426,8 @@ public void testRankDocsRetrieverWithNestedQuery() {
             null
         );
         // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and
-        // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results)
-        // so ideal rank would be: 6, 2, 1, 4, 3, 7
+        // resolves ties based on actual score, and then the doc (we're forcing 1 shard for consistent results)
+        // so ideal rank would be: 6, 2, 1, 3, 4, 7
         source.retriever(
             new CompoundRetrieverWithRankDocs(
                 rankWindowSize,
@@ -506,11 +505,11 @@ public void testRankDocsRetrieverMultipleCompoundRetrievers() {
             assertThat(resp.getHits().getTotalHits().value, equalTo(6L));
             assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_4"));
-            assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_6"));
+            assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1"));
             assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_2"));
-            assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_1"));
-            assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_7"));
-            assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_3"));
+            assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_3"));
+            assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_6"));
+            assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_7"));
         });
     }
 
@@ -545,9 +544,9 @@ public void testRankDocsRetrieverDifferentNestedSorting() {
             assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO));
             assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_4"));
             assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1"));
-            assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_7"));
+            assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_2"));
             assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_6"));
-            assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_2"));
+            assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_7"));
         });
     }
 
@@ -673,22 +672,14 @@ private RankDoc[] getRankDocs(SearchResponse searchResponse) {
             for (int i = 0; i < size; i++) {
                 var hit = searchResponse.getHits().getAt(i);
                 long sortValue = (long) hit.getRawSortValues()[hit.getRawSortValues().length - 1];
-                int doc = decodeDoc(sortValue);
-                int shardRequestIndex = decodeShardRequestIndex(sortValue);
+                int doc = ShardDocSortField.decodeDoc(sortValue);
+                int shardRequestIndex = ShardDocSortField.decodeShardRequestIndex(sortValue);
                 docs[i] = new RankDoc(doc, hit.getScore(), shardRequestIndex);
                 docs[i].rank = i + 1;
             }
             return docs;
         }
 
-        public static int decodeDoc(long value) {
-            return (int) value;
-        }
-
-        public static int decodeShardRequestIndex(long value) {
-            return (int) (value >> 32);
-        }
-
         record RankDocAndHitRatio(RankDoc rankDoc, float hitRatio) {}
 
         /**

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -224,7 +224,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_TELEMETRY_MEMORY_ADDED = def(8_748_00_0);
     public static final TransportVersion ILM_ADD_SEARCHABLE_SNAPSHOT_TOTAL_SHARDS_PER_NODE = def(8_749_00_0);
     public static final TransportVersion SEMANTIC_TEXT_SEARCH_INFERENCE_ID = def(8_750_00_0);
-
+    public static final TransportVersion RRF_QUERY_REWRITE = def(8_751_00_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java

@@ -526,7 +526,7 @@ private GroupShardsIterator<SearchShardIterator> getIterator(
                 iter.skip(true);
             }
         }
-        if (shouldSortShards(results.minAndMaxes) == false) {
+        if (false == shouldSortShards(results.minAndMaxes)) {
             return shardsIts;
         }
         FieldSortBuilder fieldSort = FieldSortBuilder.getPrimaryFieldSortOrNull(request.source());

server/src/main/java/org/elasticsearch/action/search/FetchSearchPhase.java

@@ -116,7 +116,8 @@ private void innerRun() throws Exception {
         // still use DFS_QUERY_THEN_FETCH, which does not perform the "query and fetch" optimization during the query phase.
         final boolean queryAndFetchOptimization = searchPhaseShardResults.length() == 1
             && context.getRequest().hasKnnSearch() == false
-            && reducedQueryPhase.queryPhaseRankCoordinatorContext() == null;
+            && reducedQueryPhase.queryPhaseRankCoordinatorContext() == null
+            && (context.getRequest().source() == null || context.getRequest().source().rankBuilder() == null);
         if (queryAndFetchOptimization) {
             assert assertConsistentWithQueryAndFetchOptimization();
             // query AND fetch optimization

server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java

@@ -465,10 +465,13 @@ private static SearchHits getHits(
                     searchHit.setRank(((RankDoc) shardDoc).rank);
                     searchHit.score(shardDoc.score);
                 } else if (sortedTopDocs.isSortedByField) {
-                    FieldDoc fieldDoc = (FieldDoc) shardDoc;
-                    searchHit.sortValues(fieldDoc.fields, reducedQueryPhase.sortValueFormats);
-                    if (sortScoreIndex != -1) {
-                        searchHit.score(((Number) fieldDoc.fields[sortScoreIndex]).floatValue());
+                    if (shardDoc instanceof FieldDoc fieldDoc) {
+                        searchHit.sortValues(fieldDoc.fields, reducedQueryPhase.sortValueFormats);
+                        if (sortScoreIndex != -1) {
+                            searchHit.score(((Number) fieldDoc.fields[sortScoreIndex]).floatValue());
+                        }
+                    } else {
+                        throw new IllegalArgumentException("Unsupported ScoreDoc type: " + shardDoc.getClass());
                     }
                 } else {
                     searchHit.score(shardDoc.score);
@@ -638,7 +641,7 @@ static ReducedQueryPhase reducedQueryPhase(
         final SearchProfileResultsBuilder profileBuilder = profileShardResults.isEmpty()
             ? null
             : new SearchProfileResultsBuilder(profileShardResults);
-        final SortedTopDocs sortedTopDocs;
+        SortedTopDocs sortedTopDocs;
         if (queryPhaseRankCoordinatorContext == null) {
             sortedTopDocs = sortDocs(isScrollRequest, bufferedTopDocs, from, size, reducedCompletionSuggestions);
         } else {
@@ -811,7 +814,7 @@ SearchPhaseResults<SearchPhaseResult> newSearchPhaseResults(
         int numShards,
         Consumer<Exception> onPartialMergeFailure
     ) {
-        final int size = request.source() == null || request.source().size() == -1 ? SearchService.DEFAULT_SIZE : request.source().size();
+        final int size = request.source() == null || request.source().size() == -1 ? DEFAULT_SIZE : request.source().size();
         // Use CountOnlyQueryPhaseResultConsumer for requests without aggs, suggest, etc. things only wanting a total count and
         // returning no hits
         if (size == 0
@@ -839,7 +842,7 @@ SearchPhaseResults<SearchPhaseResult> newSearchPhaseResults(
     public static final class TopDocsStats {
         final int trackTotalHitsUpTo;
         long totalHits;
-        private TotalHits.Relation totalHitsRelation;
+        private Relation totalHitsRelation;
         public long fetchHits;
         private float maxScore = Float.NEGATIVE_INFINITY;
         public boolean timedOut;
@@ -880,7 +883,7 @@ void add(TopDocsAndMaxScore topDocs, boolean timedOut, Boolean terminatedEarly)
             if (trackTotalHitsUpTo != SearchContext.TRACK_TOTAL_HITS_DISABLED) {
                 totalHits += topDocs.topDocs.totalHits.value;
                 if (topDocs.topDocs.totalHits.relation == Relation.GREATER_THAN_OR_EQUAL_TO) {
-                    totalHitsRelation = TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO;
+                    totalHitsRelation = Relation.GREATER_THAN_OR_EQUAL_TO;
                 }
             }
             fetchHits += topDocs.topDocs.scoreDocs.length;

server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java

@@ -502,6 +502,8 @@ void executeRequest(
         });
         final SearchSourceBuilder source = original.source();
         if (shouldOpenPIT(source)) {
+            // disabling shard reordering for request
+            original.setPreFilterShardSize(Integer.MAX_VALUE);
             openPIT(client, original, searchService.getDefaultKeepAliveInMillis(), listener.delegateFailureAndWrap((delegate, resp) -> {
                 // We set the keep alive to -1 to indicate that we don't need the pit id in the response.
                 // This is needed since we delete the pit prior to sending the response so the id doesn't exist anymore.

server/src/main/java/org/elasticsearch/common/lucene/Lucene.java

@@ -74,7 +74,6 @@
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.lucene.grouping.TopFieldGroups;
-import org.elasticsearch.search.retriever.rankdoc.RankDocsSortField;
 import org.elasticsearch.search.sort.ShardDocSortField;
 
 import java.io.IOException;
@@ -553,8 +552,6 @@ private static SortField rewriteMergeSortField(SortField sortField) {
             return newSortField;
         } else if (sortField.getClass() == ShardDocSortField.class) {
             return new SortField(sortField.getField(), SortField.Type.LONG, sortField.getReverse());
-        } else if (sortField.getClass() == RankDocsSortField.class) {
-            return new SortField(sortField.getField(), SortField.Type.INT, sortField.getReverse());
         } else {
             return sortField;
         }

server/src/main/java/org/elasticsearch/search/SearchModule.java

@@ -239,7 +239,6 @@
 import org.elasticsearch.search.retriever.RetrieverParserContext;
 import org.elasticsearch.search.retriever.StandardRetrieverBuilder;
 import org.elasticsearch.search.retriever.rankdoc.RankDocsQueryBuilder;
-import org.elasticsearch.search.retriever.rankdoc.RankDocsSortBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
 import org.elasticsearch.search.sort.ScoreSortBuilder;
@@ -868,7 +867,6 @@ private void registerSorts() {
         namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, ScoreSortBuilder.NAME, ScoreSortBuilder::new));
         namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, ScriptSortBuilder.NAME, ScriptSortBuilder::new));
         namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, FieldSortBuilder.NAME, FieldSortBuilder::new));
-        namedWriteables.add(new NamedWriteableRegistry.Entry(SortBuilder.class, RankDocsSortBuilder.NAME, RankDocsSortBuilder::new));
     }
 
     private static <T> void registerFromPlugin(List<SearchPlugin> plugins, Function<SearchPlugin, List<T>> producer, Consumer<T> consumer) {