Optimize FieldExistsQuery to leverage index statistic in DocValuesSkipper (#14830)

Author: bugmakerrrrrr

lucene/CHANGES.txt

@@ -177,6 +177,8 @@ Optimizations
 * GITHUB#14836: Sometimes use FixedBitSet when doing HNSW searches. This slightly improves HNSW
   search performance on smaller graphs. (Ben Trent)
 
+* GITHUB#14830: Optimize FieldExistsQuery to leverage index statistic in DocValuesSkipper. (Pan Guixin)
+
 Bug Fixes
 ---------------------
 * GITHUB#14654: ValueSource.fromDoubleValuesSource(dvs).getSortField() would throw errors when

lucene/core/src/java/org/apache/lucene/search/FieldExistsQuery.java

@@ -21,6 +21,8 @@
 import org.apache.lucene.document.KnnFloatVectorField;
 import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValuesSkipIndexType;
+import org.apache.lucene.index.DocValuesSkipper;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
@@ -138,17 +140,14 @@ public Query rewrite(IndexSearcher indexSearcher) throws IOException {
           != DocValuesType.NONE) { // the field indexes doc values or points
 
         // This optimization is possible due to LUCENE-9334 enforcing a field to always use the
-        // same data structures (all or nothing). Since there's no index statistic to detect when
-        // all documents have doc values for a specific field, FieldExistsQuery can only be
-        // rewritten to MatchAllDocsQuery for doc values field, when that same field also indexes
-        // terms or point values which do have index statistics, and those statistics confirm that
-        // all documents in this segment have values terms or point values.
-
-        Terms terms = leaf.terms(field);
-        PointValues pointValues = leaf.getPointValues(field);
+        // same data structures (all or nothing).
+        final Terms terms = leaf.terms(field);
+        final PointValues pointValues = leaf.getPointValues(field);
+        final DocValuesSkipper docValuesSkipper = leaf.getDocValuesSkipper(field);
 
         if ((terms == null || terms.getDocCount() != leaf.maxDoc())
-            && (pointValues == null || pointValues.getDocCount() != leaf.maxDoc())) {
+            && (pointValues == null || pointValues.getDocCount() != leaf.maxDoc())
+            && (docValuesSkipper == null || docValuesSkipper.docCount() != leaf.maxDoc())) {
           allReadersRewritable = false;
           break;
         }
@@ -248,6 +247,9 @@ public int count(LeafReaderContext context) throws IOException {
             } else if (fieldInfo.getIndexOptions() != IndexOptions.NONE) {
               Terms terms = reader.terms(field);
               return terms == null ? 0 : terms.getDocCount();
+            } else if (fieldInfo.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) {
+              DocValuesSkipper docValuesSkipper = reader.getDocValuesSkipper(field);
+              return docValuesSkipper == null ? 0 : docValuesSkipper.docCount();
             }
           }
 

lucene/core/src/test/org/apache/lucene/search/TestFieldExistsQuery.java

@@ -88,6 +88,24 @@ public void testDocValuesRewriteWithPointValuesPresent() throws IOException {
     dir.close();
   }
 
+  public void testDocValuesRewriteWithDocValuesSkipperPresent() throws IOException {
+    Directory dir = newDirectory();
+    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+    final int numDocs = atLeast(100);
+    for (int i = 0; i < numDocs; ++i) {
+      Document doc = new Document();
+      doc.add(NumericDocValuesField.indexedField("dim", 2));
+      iw.addDocument(doc);
+    }
+    iw.commit();
+    final IndexReader reader = iw.getReader();
+    iw.close();
+
+    assertEquals(new MatchAllDocsQuery(), new FieldExistsQuery("dim").rewrite(newSearcher(reader)));
+    reader.close();
+    dir.close();
+  }
+
   public void testDocValuesNoRewrite() throws IOException {
     Directory dir = newDirectory();
     RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
@@ -741,6 +759,41 @@ private float[] randomVector(int dim) {
     return v;
   }
 
+  public void testDeleteDocValues() throws IOException {
+    try (Directory dir = newDirectory();
+        RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {
+      final int numDocs = atLeast(100);
+
+      boolean allDocsHaveValue = random().nextBoolean();
+      BitSet docWithValue = new FixedBitSet(numDocs);
+      for (int i = 0; i < numDocs; ++i) {
+        Document doc = new Document();
+        if (allDocsHaveValue || random().nextBoolean()) {
+          doc.add(NumericDocValuesField.indexedField("num", i));
+          docWithValue.set(i);
+        }
+        doc.add(new StringField("id", Integer.toString(i), Store.NO));
+        iw.addDocument(doc);
+      }
+
+      if (random().nextBoolean()) {
+        final int numDeleted = random().nextInt(numDocs) + 1;
+        for (int i = 0; i < numDeleted; ++i) {
+          int id = random().nextInt(numDocs);
+          iw.deleteDocuments(new Term("id", Integer.toString(id)));
+          docWithValue.clear(id);
+        }
+      }
+
+      try (IndexReader reader = iw.getReader()) {
+        final IndexSearcher searcher = newSearcher(reader);
+
+        final int count = searcher.count(new FieldExistsQuery("num"));
+        assertEquals(docWithValue.cardinality(), count);
+      }
+    }
+  }
+
   public void testDeleteAllPointDocs() throws Exception {
     try (Directory dir = newDirectory();
         RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {