Rewrite SortedNumericDocValuesRangeQuery using DocValuesSkippers (#15083)

If DocValuesSkippers are enabled on a field, we can check at rewrite time
if a range query will match all or none of the documents in all segments,
and rewrite to a corresponding MatchAllDocsQuery or MatchNoDocsQuery.

Author: romseygeek

lucene/CHANGES.txt

@@ -253,6 +253,8 @@ Optimizations
 * GITHUB#15039: Score computations are now more reliably vectorized.
   (Adrien Grand, Guo Feng)
 
+* GITHUB#15083: Use DocValuesSkippers in SortedNumericDocValuesRangeQuery#rewrite(). (Alan Woodward)
+
 Changes in Runtime Behavior
 ---------------------
 * GITHUB#14823: Decrease TieredMergePolicy's default number of segments per

lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java

@@ -31,6 +31,7 @@
 import org.apache.lucene.search.DocValuesRangeIterator;
 import org.apache.lucene.search.FieldExistsQuery;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryVisitor;
@@ -97,6 +98,17 @@ public Query rewrite(IndexSearcher indexSearcher) throws IOException {
     if (lowerValue > upperValue) {
       return new MatchNoDocsQuery();
     }
+    long globalMin = DocValuesSkipper.globalMinValue(indexSearcher, field);
+    long globalMax = DocValuesSkipper.globalMaxValue(indexSearcher, field);
+    if (lowerValue > globalMax || upperValue < globalMin) {
+      return new MatchNoDocsQuery();
+    }
+    if (lowerValue <= globalMin
+        && upperValue >= globalMax
+        && DocValuesSkipper.globalDocCount(indexSearcher, field)
+            == indexSearcher.getIndexReader().maxDoc()) {
+      return new MatchAllDocsQuery();
+    }
     return super.rewrite(indexSearcher);
   }
 

lucene/core/src/java/org/apache/lucene/index/DocValuesSkipper.java

@@ -18,6 +18,7 @@
 
 import java.io.IOException;
 import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
 
 /**
  * Skipper for {@link DocValues}.
@@ -122,4 +123,67 @@ public final void advance(long minValue, long maxValue) throws IOException {
       advance(maxDocID + 1);
     }
   }
+
+  /**
+   * Returns the minimum value for a field across all segments, or {@link Long#MIN_VALUE} if not
+   * available
+   *
+   * @param searcher a searcher over the index
+   * @param field the field to retrieve values for
+   */
+  public static long globalMinValue(IndexSearcher searcher, String field) throws IOException {
+    long minValue = Long.MAX_VALUE;
+    for (LeafReaderContext ctx : searcher.getLeafContexts()) {
+      if (ctx.reader().getFieldInfos().fieldInfo(field) == null) {
+        continue; // no field values in this segment, so we can ignore it
+      }
+      DocValuesSkipper skipper = ctx.reader().getDocValuesSkipper(field);
+      if (skipper == null) {
+        minValue = Long.MIN_VALUE;
+      } else {
+        minValue = Math.min(minValue, skipper.minValue());
+      }
+    }
+    return minValue;
+  }
+
+  /**
+   * Returns the maximum value for a field across all segments, or {@link Long#MIN_VALUE} if not
+   * available
+   *
+   * @param searcher a searcher over the index
+   * @param field the field to retrieve values for
+   */
+  public static long globalMaxValue(IndexSearcher searcher, String field) throws IOException {
+    long maxValue = Long.MIN_VALUE;
+    for (LeafReaderContext ctx : searcher.getLeafContexts()) {
+      if (ctx.reader().getFieldInfos().fieldInfo(field) == null) {
+        continue; // no field values in this segment, so we can ignore it
+      }
+      DocValuesSkipper skipper = ctx.reader().getDocValuesSkipper(field);
+      if (skipper == null) {
+        maxValue = Long.MAX_VALUE;
+      } else {
+        maxValue = Math.max(maxValue, skipper.maxValue());
+      }
+    }
+    return maxValue;
+  }
+
+  /**
+   * Returns the total skipper document count for a field across all segments
+   *
+   * @param searcher a searcher over the index
+   * @param field the field to retrieve values for
+   */
+  public static int globalDocCount(IndexSearcher searcher, String field) throws IOException {
+    int docCount = 0;
+    for (LeafReaderContext ctx : searcher.getLeafContexts()) {
+      DocValuesSkipper skipper = ctx.reader().getDocValuesSkipper(field);
+      if (skipper != null) {
+        docCount += skipper.docCount();
+      }
+    }
+    return docCount;
+  }
 }

lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java

@@ -16,6 +16,9 @@
  */
 package org.apache.lucene.search;
 
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.instanceOf;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -648,4 +651,76 @@ private void assertCount(IndexSearcher searcher, Query query, int expectedCount)
     Weight w = searcher.createWeight(query, ScoreMode.COMPLETE, 1.0f);
     assertEquals(expectedCount, w.count(searcher.reader.leaves().getFirst()));
   }
+
+  public void testSortedNumericDocValuesRangeQueryRewrites() throws Exception {
+    try (Directory dir = newDirectory();
+        RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {
+      for (int i = 0; i < 100; i++) {
+        Document doc = new Document();
+        doc.add(SortedNumericDocValuesField.indexedField("with_index", 100 + i));
+        doc.add(new SortedNumericDocValuesField("without_index", 100 + i));
+        if (i % 17 == 0) {
+          iw.commit();
+        }
+        if (i != 55) {
+          doc.add(SortedNumericDocValuesField.indexedField("sparse", 100 + i));
+        }
+        if (i == 74) {
+          doc.add(SortedNumericDocValuesField.indexedField("super_sparse", 174));
+        }
+        iw.addDocument(doc);
+      }
+      iw.commit();
+
+      try (IndexReader reader = iw.getReader()) {
+        IndexSearcher searcher = new IndexSearcher(reader);
+        assertThat(
+            searcher.rewrite(SortedNumericDocValuesField.newSlowRangeQuery("with_index", 0, 50)),
+            instanceOf(MatchNoDocsQuery.class));
+        assertThat(
+            searcher.rewrite(SortedNumericDocValuesField.newSlowRangeQuery("with_index", 0, 250)),
+            instanceOf(MatchAllDocsQuery.class));
+        assertThat(
+            searcher.rewrite(SortedNumericDocValuesField.newSlowRangeQuery("sparse", 0, 50)),
+            instanceOf(MatchNoDocsQuery.class));
+        assertThat(
+            searcher.rewrite(SortedNumericDocValuesField.newSlowRangeQuery("super_sparse", 0, 50)),
+            instanceOf(MatchNoDocsQuery.class));
+        assertThat(
+            searcher.rewrite(
+                SortedNumericDocValuesField.newSlowRangeQuery("super_sparse", 250, 350)),
+            instanceOf(MatchNoDocsQuery.class));
+        assertThat(
+            searcher
+                .rewrite(SortedNumericDocValuesField.newSlowRangeQuery("super_sparse", 174, 174))
+                .getClass()
+                .toString(),
+            containsString("SortedNumericDocValuesRangeQuery"));
+        assertThat(
+            searcher
+                .rewrite(SortedNumericDocValuesField.newSlowRangeQuery("with_index", 0, 150))
+                .getClass()
+                .toString(),
+            containsString("SortedNumericDocValuesRangeQuery"));
+        assertThat(
+            searcher
+                .rewrite(SortedNumericDocValuesField.newSlowRangeQuery("with_index", 150, 250))
+                .getClass()
+                .toString(),
+            containsString("SortedNumericDocValuesRangeQuery"));
+        assertThat(
+            searcher
+                .rewrite(SortedNumericDocValuesField.newSlowRangeQuery("with_index", 120, 150))
+                .getClass()
+                .toString(),
+            containsString("SortedNumericDocValuesRangeQuery"));
+        assertThat(
+            searcher
+                .rewrite(SortedNumericDocValuesField.newSlowRangeQuery("sparse", 0, 250))
+                .getClass()
+                .toString(),
+            containsString("SortedNumericDocValuesRangeQuery"));
+      }
+    }
+  }
 }