Utilize iterator and not approximation in MaxScoreBulkScorer (#15380)

* Utilize iterator and not approximation in MaxScoreBulkScorer

* CHANGES

* add assert on verifiedMatches

* adjust assertion

* adding some random approx query wrapping

* iter

Author: benwtrent

lucene/CHANGES.txt

@@ -224,6 +224,9 @@ Bug Fixes
 * GITHUB#15334: Fix NPE for MemoryIndex when using nearest neighbor search. MemoryIndex does not support
   nearest neighbor search, but it should not throw an NPE (Ben Trent)
 
+* GITHUB#15380: Fix potential EOF introduced by optimized filter iterations in MaxScoreBulkScorer. The
+  approximation and the match verification could get out of sync, resulting in an EOFException. (Ben Trent)
+
 Other
 ---------------------
 * GITHUB#15237: Fix SmartChinese to only deserialize dictionary data from classpath with a native array

lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java

@@ -101,9 +101,20 @@ DisiWrapper getSubMatches() throws IOException {
         }
       }
       unverifiedMatches.clear();
+      assert assertVerifiedMatchesApproximationDocID();
       return verifiedMatches;
     }
 
+    private boolean assertVerifiedMatchesApproximationDocID() {
+      int docID = docID();
+      for (DisiWrapper w = verifiedMatches; w != null; w = w.next) {
+        if (w.doc != docID) {
+          return false;
+        }
+      }
+      return true;
+    }
+
     @Override
     public boolean matches() throws IOException {
       verifiedMatches = null;

lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java

@@ -171,7 +171,8 @@ private void scoreInnerWindowWithFilter(
     DisiWrapper top = essentialQueue.top();
     assert top.doc < max;
     while (top.doc < filter.doc) {
-      top.doc = top.approximation.advance(filter.doc);
+      // Must use the iterator as `top` might be a two-phase iterator
+      top.doc = top.iterator.advance(filter.doc);
       top = essentialQueue.updateTop();
     }
 

lucene/core/src/test/org/apache/lucene/search/TestBooleanRewrites.java

@@ -34,6 +34,7 @@
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.tests.search.RandomApproximationQuery;
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.lucene.tests.util.TestUtil;
 
@@ -518,11 +519,13 @@ private Query randomBooleanQuery(Random random) {
   }
 
   private Query randomWrapper(Random random, Query query) {
-    switch (random.nextInt(2)) {
+    switch (random.nextInt(3)) {
       case 0:
         return new BoostQuery(query, TestUtil.nextInt(random, 0, 4));
       case 1:
         return new ConstantScoreQuery(query);
+      case 2:
+        return new RandomApproximationQuery(query, random);
       default:
         throw new AssertionError();
     }

lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java

@@ -32,6 +32,7 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.search.RandomApproximationQuery;
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.lucene.util.Bits;
 
@@ -146,6 +147,10 @@ public void testFilteredDisjunction() throws Exception {
             new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("foo", "A"))), 2);
         Query clause2 = new ConstantScoreQuery(new TermQuery(new Term("foo", "C")));
         Query filter = new TermQuery(new Term("foo", "B"));
+        if (random().nextBoolean()) {
+          clause1 = new RandomApproximationQuery(clause1, random());
+          clause2 = new RandomApproximationQuery(clause2, random());
+        }
         LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
         Scorer scorer1 =
             searcher
@@ -214,6 +219,10 @@ public void testFilteredDisjunctionWithSkipping() throws Exception {
             new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("foo", "A"))), 2);
         Query clause2 = new ConstantScoreQuery(new TermQuery(new Term("foo", "C")));
         Query filter = new TermQuery(new Term("foo", "B"));
+        if (random().nextBoolean()) {
+          clause1 = new RandomApproximationQuery(clause1, random());
+          clause2 = new RandomApproximationQuery(clause2, random());
+        }
         LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
         Scorer scorer1 =
             searcher