Merge branch '9.0' into backport-90-consider-aborting-merges-while-enqueued

Author: albertzaharovits

docs/changelog/129223.yaml

@@ -0,0 +1,5 @@
+pr: 129223
+summary: Fix text similarity reranker does not propagate min score correctly
+area: Search
+type: bug
+issues: []

muted-tests.yml

@@ -288,6 +288,9 @@ tests:
 - class: org.elasticsearch.index.engine.ThreadPoolMergeExecutorServiceTests
   method: testIORateIsAdjustedForAllRunningMergeTasks
   issue: https://github.com/elastic/elasticsearch/issues/129531
+- class: org.elasticsearch.packaging.test.DockerTests
+  method: test600Interrupt
+  issue: https://github.com/elastic/elasticsearch/issues/128144
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java

@@ -195,7 +195,8 @@ public void onFailure(Exception e) {
         RankDocsRetrieverBuilder rankDocsRetrieverBuilder = new RankDocsRetrieverBuilder(
             rankWindowSize,
             newRetrievers.stream().map(s -> s.retriever).toList(),
-            results::get
+            results::get,
+            this.minScore
         );
         rankDocsRetrieverBuilder.retrieverName(retrieverName());
         return rankDocsRetrieverBuilder;

server/src/main/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilder.java

@@ -33,13 +33,14 @@ public class RankDocsRetrieverBuilder extends RetrieverBuilder {
     final List<RetrieverBuilder> sources;
     final Supplier<RankDoc[]> rankDocs;
 
-    public RankDocsRetrieverBuilder(int rankWindowSize, List<RetrieverBuilder> sources, Supplier<RankDoc[]> rankDocs) {
+    public RankDocsRetrieverBuilder(int rankWindowSize, List<RetrieverBuilder> sources, Supplier<RankDoc[]> rankDocs, Float minScore) {
         this.rankWindowSize = rankWindowSize;
         this.rankDocs = rankDocs;
         if (sources == null || sources.isEmpty()) {
             throw new IllegalArgumentException("sources must not be null or empty");
         }
         this.sources = sources;
+        this.minScore = minScore;
     }
 
     @Override
@@ -48,7 +49,7 @@ public String getName() {
     }
 
     private boolean sourceHasMinScore() {
-        return minScore != null || sources.stream().anyMatch(x -> x.minScore() != null);
+        return this.minScore != null || sources.stream().anyMatch(x -> x.minScore() != null);
     }
 
     private boolean sourceShouldRewrite(QueryRewriteContext ctx) throws IOException {
@@ -132,7 +133,7 @@ public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder
             searchSourceBuilder.size(rankWindowSize);
         }
         if (sourceHasMinScore()) {
-            searchSourceBuilder.minScore(this.minScore() == null ? Float.MIN_VALUE : this.minScore());
+            searchSourceBuilder.minScore(this.minScore == null ? Float.MIN_VALUE : this.minScore);
         }
         if (searchSourceBuilder.size() + searchSourceBuilder.from() > rankDocResults.length) {
             searchSourceBuilder.size(Math.max(0, rankDocResults.length - searchSourceBuilder.from()));

server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java

@@ -97,7 +97,7 @@ private List<QueryBuilder> preFilters(QueryRewriteContext queryRewriteContext) t
     }
 
     private RankDocsRetrieverBuilder createRandomRankDocsRetrieverBuilder(QueryRewriteContext queryRewriteContext) throws IOException {
-        return new RankDocsRetrieverBuilder(randomIntBetween(1, 100), innerRetrievers(queryRewriteContext), rankDocsSupplier());
+        return new RankDocsRetrieverBuilder(randomIntBetween(1, 100), innerRetrievers(queryRewriteContext), rankDocsSupplier(), null);
     }
 
     public void testExtractToSearchSourceBuilder() throws IOException {

x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestRerankingServiceExtension.java

@@ -36,6 +36,7 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
@@ -148,16 +149,28 @@ public void chunkedInfer(
         }
 
         private RankedDocsResults makeResults(List<String> input) {
-            List<RankedDocsResults.RankedDoc> results = new ArrayList<>();
             int totalResults = input.size();
-            float minScore = random.nextFloat(-1f, 1f);
-            float resultDiff = 0.2f;
-            for (int i = 0; i < input.size(); i++) {
-                results.add(
-                    new RankedDocsResults.RankedDoc(totalResults - 1 - i, minScore + resultDiff * (totalResults - i), input.get(i))
-                );
+            try {
+                List<RankedDocsResults.RankedDoc> results = new ArrayList<>();
+                for (int i = 0; i < totalResults; i++) {
+                    results.add(new RankedDocsResults.RankedDoc(i, Float.parseFloat(input.get(i)), input.get(i)));
+                }
+                return new RankedDocsResults(results.stream().sorted(Comparator.reverseOrder()).toList());
+            } catch (NumberFormatException ex) {
+                List<RankedDocsResults.RankedDoc> results = new ArrayList<>();
+                float minScore = random.nextFloat(-1f, 1f);
+                float resultDiff = 0.2f;
+                for (int i = 0; i < input.size(); i++) {
+                    results.add(
+                        new RankedDocsResults.RankedDoc(
+                            totalResults - 1 - i,
+                            minScore + resultDiff * (totalResults - i),
+                            input.get(totalResults - 1 - i)
+                        )
+                    );
+                }
+                return new RankedDocsResults(results);
             }
-            return new RankedDocsResults(results);
         }
 
         protected ServiceSettings getServiceSettingsFromMap(Map<String, Object> serviceSettingsMap) {

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java

@@ -28,6 +28,9 @@ public class InferenceFeatures implements FeatureSpecification {
     private static final NodeFeature SEMANTIC_TEXT_HIGHLIGHTER = new NodeFeature("semantic_text.highlighter");
     private static final NodeFeature SEMANTIC_TEXT_HIGHLIGHTER_DEFAULT = new NodeFeature("semantic_text.highlighter.default");
     private static final NodeFeature SEMANTIC_TEXT_MATCH_ALL_HIGHLIGHTER = new NodeFeature("semantic_text.match_all_highlighter");
+    private static final NodeFeature TEST_RERANKING_SERVICE_PARSE_TEXT_AS_SCORE = new NodeFeature(
+        "test_reranking_service.parse_text_as_score"
+    );
 
     @Override
     public Set<NodeFeature> getTestFeatures() {
@@ -44,10 +47,12 @@ public Set<NodeFeature> getTestFeatures() {
             SemanticInferenceMetadataFieldsMapper.EXPLICIT_NULL_FIXES,
             SEMANTIC_KNN_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED,
             TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_ALIAS_HANDLING_FIX,
+            TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_MINSCORE_FIX,
             SemanticInferenceMetadataFieldsMapper.INFERENCE_METADATA_FIELDS_ENABLED_BY_DEFAULT,
             SEMANTIC_TEXT_HIGHLIGHTER_DEFAULT,
             SEMANTIC_KNN_FILTER_FIX,
-            SEMANTIC_TEXT_MATCH_ALL_HIGHLIGHTER
+            SEMANTIC_TEXT_MATCH_ALL_HIGHLIGHTER,
+            TEST_RERANKING_SERVICE_PARSE_TEXT_AS_SCORE
         );
     }
 }

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 
@@ -39,6 +40,7 @@ public class TextSimilarityRankRetrieverBuilder extends CompoundRetrieverBuilder
     public static final NodeFeature TEXT_SIMILARITY_RERANKER_ALIAS_HANDLING_FIX = new NodeFeature(
         "text_similarity_reranker_alias_handling_fix"
     );
+    public static final NodeFeature TEXT_SIMILARITY_RERANKER_MINSCORE_FIX = new NodeFeature("text_similarity_reranker_minscore_fix");
 
     public static final ParseField RETRIEVER_FIELD = new ParseField("retriever");
     public static final ParseField INFERENCE_ID_FIELD = new ParseField("inference_id");
@@ -141,23 +143,21 @@ protected TextSimilarityRankRetrieverBuilder clone(
     protected RankDoc[] combineInnerRetrieverResults(List<ScoreDoc[]> rankResults, boolean explain) {
         assert rankResults.size() == 1;
         ScoreDoc[] scoreDocs = rankResults.getFirst();
-        TextSimilarityRankDoc[] textSimilarityRankDocs = new TextSimilarityRankDoc[scoreDocs.length];
+        List<TextSimilarityRankDoc> filteredDocs = new ArrayList<>();
+        // Filtering by min_score must be done here, after reranking.
+        // Applying min_score in the child retriever could prematurely exclude documents that would receive high scores from the reranker.
         for (int i = 0; i < scoreDocs.length; i++) {
             ScoreDoc scoreDoc = scoreDocs[i];
             assert scoreDoc.score >= 0;
-            if (explain) {
-                textSimilarityRankDocs[i] = new TextSimilarityRankDoc(
-                    scoreDoc.doc,
-                    scoreDoc.score,
-                    scoreDoc.shardIndex,
-                    inferenceId,
-                    field
-                );
-            } else {
-                textSimilarityRankDocs[i] = new TextSimilarityRankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex);
+            if (minScore == null || scoreDoc.score >= minScore) {
+                if (explain) {
+                    filteredDocs.add(new TextSimilarityRankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex, inferenceId, field));
+                } else {
+                    filteredDocs.add(new TextSimilarityRankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex));
+                }
             }
         }
-        return textSimilarityRankDocs;
+        return filteredDocs.toArray(new TextSimilarityRankDoc[0]);
     }
 
     @Override