Fix timestamp range query optimization for indices with doc values skipper (#123930)

When running a timestamp range query, as an optimization we check if the
query range overlaps with the total range of values within a shard before
executing the query on that shard. That way, if the range is disjoint, we
can skip execution for that shard. To get the range of values within a
shard, we usually use the PointValues index on the shard. However, when the
doc values skipper is enabled, the point values are not (as the reason for
the skipper is to reduce storage overhead by removing the point values
index). In this case, we need to instead get the range of values within the
shard by using the skipper. This patch implements that logic.

Follow-up to #123191

Author: jordan-powers

server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java

@@ -235,6 +235,15 @@ public void testSearchIdleStats() throws InterruptedException {
     }
 
     public void testSearchIdleBoolQueryMatchOneIndex() throws InterruptedException {
+        checkSearchIdleBoolQueryMatchOneIndex(IndexSettings.DOC_VALUES_SKIPPER.isEnabled());
+    }
+
+    public void testSearchIdleBoolQueryMatchOneIndexWithDocValuesSkipper() throws InterruptedException {
+        assumeTrue("doc values skipper feature should be enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled());
+        checkSearchIdleBoolQueryMatchOneIndex(false);
+    }
+
+    private void checkSearchIdleBoolQueryMatchOneIndex(boolean disableDocValuesSkippers) throws InterruptedException {
         // GIVEN
         final String idleIndex = "test1";
         final String activeIndex = "test2";
@@ -259,7 +268,7 @@ public void testSearchIdleBoolQueryMatchOneIndex() throws InterruptedException {
             .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-05-12T00:00:00.000Z")
             .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-05-13T23:59:59.999Z");
 
-        if (IndexSettings.DOC_VALUES_SKIPPER.isEnabled()) {
+        if (disableDocValuesSkippers) {
             idleIndexSettingsBuilder.put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), false);
             activeIndexSettingsBuilder.put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), false);
         }

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -16,6 +16,7 @@
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
+import org.apache.lucene.index.DocValuesSkipper;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.PointValues;
@@ -69,6 +70,7 @@
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.util.Collections;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
@@ -827,8 +829,24 @@ public Relation isFieldWithinQuery(
             QueryRewriteContext context
         ) throws IOException {
             if (isIndexed() == false && pointsMetadataAvailable == false && hasDocValues()) {
-                // we don't have a quick way to run this check on doc values, so fall back to default assuming we are within bounds
-                return Relation.INTERSECTS;
+                if (hasDocValuesSkipper() == false) {
+                    // we don't have a quick way to run this check on doc values, so fall back to default assuming we are within bounds
+                    return Relation.INTERSECTS;
+                }
+                long minValue = Long.MAX_VALUE;
+                long maxValue = Long.MIN_VALUE;
+                List<LeafReaderContext> leaves = reader.leaves();
+                if (leaves.size() == 0) {
+                    // no data, so nothing matches
+                    return Relation.DISJOINT;
+                }
+                for (LeafReaderContext ctx : leaves) {
+                    DocValuesSkipper skipper = ctx.reader().getDocValuesSkipper(name());
+                    assert skipper != null : "no skipper for field:" + name() + " and reader:" + reader;
+                    minValue = Long.min(minValue, skipper.minValue());
+                    maxValue = Long.max(maxValue, skipper.maxValue());
+                }
+                return isFieldWithinQuery(minValue, maxValue, from, to, includeLower, includeUpper, timeZone, dateParser, context);
             }
             byte[] minPackedValue = PointValues.getMinPackedValue(reader, name());
             if (minPackedValue == null) {

server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java

@@ -8,6 +8,7 @@
  */
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.document.Field;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.SortedNumericDocValuesField;
@@ -85,12 +86,51 @@ public void testIsFieldWithinQueryDateNanos() throws IOException {
         isFieldWithinRangeTestCase(ft);
     }
 
+    public void testIsFieldWithinQueryDateMillisDocValueSkipper() throws IOException {
+        DateFieldType ft = new DateFieldType(
+            "my_date",
+            false,
+            false,
+            false,
+            true,
+            true,
+            DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
+            Resolution.MILLISECONDS,
+            null,
+            null,
+            Collections.emptyMap()
+        );
+        isFieldWithinRangeTestCase(ft);
+    }
+
+    public void testIsFieldWithinQueryDateNanosDocValueSkipper() throws IOException {
+        DateFieldType ft = new DateFieldType(
+            "my_date",
+            false,
+            false,
+            false,
+            true,
+            true,
+            DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER,
+            Resolution.NANOSECONDS,
+            null,
+            null,
+            Collections.emptyMap()
+        );
+        isFieldWithinRangeTestCase(ft);
+    }
+
     public void isFieldWithinRangeTestCase(DateFieldType ft) throws IOException {
 
         Directory dir = newDirectory();
         IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
         LuceneDocument doc = new LuceneDocument();
-        LongPoint field = new LongPoint("my_date", ft.parse("2015-10-12"));
+        Field field;
+        if (ft.hasDocValuesSkipper()) {
+            field = SortedNumericDocValuesField.indexedField("my_date", ft.parse("2015-10-12"));
+        } else {
+            field = new LongPoint("my_date", ft.parse("2015-10-12"));
+        }
         doc.add(field);
         w.addDocument(doc);
         field.setLongValue(ft.parse("2016-04-03"));