highlighter, fastvector: improve/fix merging of boosts to improve snippet ranking (#15434)

Fix QueryPhraseMap.markTerminal() boost override due to conflicting query expansion.

When a query includes overlapping phrases, the expansion process may generate duplicate phrases—one with the original (possibly high) user-defined boost, and another one with the boost of 1. As a result, the final boost value assigned to the QueryPhraseMap may be incorrect, since it is determined by whichever duplicate is processed last during the creation of the QueryPhraseMap in the markTerminal method.

Author: fragosoluana

lucene/CHANGES.txt

@@ -112,6 +112,8 @@ Bug Fixes
 
 * GITHUB#15125: Handle inconsistent schema on flush with index sorts (Nhat Nguyen)
 
+* GITHUB#15434: Don't override boost of conflicting expanded phrases by taking the max boost in markTerminal for FVHighlighter (Luana Fragoso)
+
 Changes in Runtime Behavior
 ---------------------
 * GITHUB#14187: The query cache is now disabled by default. (Adrien Grand)

lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java

@@ -429,7 +429,7 @@ private void markTerminal(float boost) {
     private void markTerminal(int slop, float boost) {
       this.terminal = true;
       this.slop = slop;
-      this.boost = boost;
+      this.boost = Math.max(this.boost, boost);
       this.termOrPhraseNumber = fieldQuery.nextTermOrPhraseNumber();
     }
 

lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/TestFieldQuery.java

@@ -19,6 +19,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -814,6 +815,48 @@ public void testQueryPhraseMapOverlap2gram() throws Exception {
     assertEquals(0, qpm2.subMap.size());
   }
 
+  public void testQueryPhraseMapDuplicate() throws IOException {
+    BooleanQuery.Builder query = new BooleanQuery.Builder();
+    Query bq = toPhraseQuery(analyze("a b c", F, analyzerB), F);
+    bq = new BoostQuery(bq, 100);
+    query.add(bq, Occur.SHOULD);
+
+    bq = toPhraseQuery(analyze("a b", F, analyzerB), F);
+    bq = new BoostQuery(bq, 20);
+    query.add(bq, Occur.SHOULD);
+
+    bq = toPhraseQuery(analyze("b c", F, analyzerB), F);
+    bq = new BoostQuery(bq, 50);
+    query.add(bq, Occur.SHOULD);
+
+    bq = query.build();
+    FieldQuery fq = new FieldQuery(bq, true, true);
+    Set<Query> flatQueries = new LinkedHashSet<>();
+    fq.flatten(bq, searcher, flatQueries, 1f);
+
+    assertCollectionQueries(
+        fq.expand(flatQueries),
+        pqF(100, "a", "b", "c"),
+        pqF(20, "a", "b"),
+        //  "a b c": 1 -> expanded from "a b" + "b c"
+        new BoostQuery(pqF(1f, "a", "b", "c"), 1f),
+        pqF(50, "b", "c"));
+
+    Map<String, QueryPhraseMap> map = fq.rootMaps;
+    QueryPhraseMap a_qpm = map.get("f").subMap.get("a");
+    assertEquals(0, a_qpm.boost, 0.0);
+    QueryPhraseMap b_qpm = a_qpm.subMap.get("b");
+    assertEquals(20, b_qpm.boost, 0.0);
+    QueryPhraseMap c_qpm = b_qpm.subMap.get("c");
+    // make sure final boost is from the query and not the expanded boost 1
+    assertEquals(100, c_qpm.boost, 0.0);
+
+    b_qpm = map.get("f").subMap.get("b");
+    assertEquals(0, b_qpm.boost, 0.0);
+    c_qpm = b_qpm.subMap.get("c");
+    assertEquals(50, c_qpm.boost, 0.0);
+  }
+
   public void testSearchPhrase() throws Exception {
     Query query = pqF("a", "b", "c");