Allow re-enabling of sparse indexes on @timestamp field in LogsDB

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -190,6 +190,8 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion KEYWORD_MULTI_FIELDS_NOT_STORED_WHEN_IGNORED = def(9_040_0_00, Version.LUCENE_10_3_0);
     public static final IndexVersion UPGRADE_TO_LUCENE_10_3_1 = def(9_041_0_00, Version.LUCENE_10_3_1);
 
+    public static final IndexVersion REENABLED_TIMESTAMP_DOC_VALUES_SPARSE_INDEX = def(9_042_0_00, Version.LUCENE_10_3_1);
+
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -1164,7 +1164,7 @@ private DateFieldMapper(
      * Determines whether the doc values skipper (sparse index) should be used for the {@code @timestamp} field.
      * <p>
      * The doc values skipper is enabled only if {@code index.mapping.use_doc_values_skipper} is set to {@code true},
-     * the index was created on or after {@link IndexVersions#TIMESTAMP_DOC_VALUES_SPARSE_INDEX}, and the
+     * the index was created on or after {@link IndexVersions#REENABLED_TIMESTAMP_DOC_VALUES_SPARSE_INDEX}, and the
      * field has doc values enabled. Additionally, the index mode must be {@link IndexMode#LOGSDB} or {@link IndexMode#TIME_SERIES}, and
      * the index sorting configuration must include the {@code @timestamp} field.
      *
@@ -1186,7 +1186,7 @@ private static boolean shouldUseDocValuesSkipper(
         final IndexSortConfig indexSortConfig,
         final String fullFieldName
     ) {
-        return indexCreatedVersion.onOrAfter(IndexVersions.TIMESTAMP_DOC_VALUES_SPARSE_INDEX)
+        return indexCreatedVersion.onOrAfter(IndexVersions.REENABLED_TIMESTAMP_DOC_VALUES_SPARSE_INDEX)
             && useDocValuesSkipper
             && hasDocValues
             && (IndexMode.LOGSDB.equals(indexMode) || IndexMode.TIME_SERIES.equals(indexMode))

server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedDocValuesRangeQuery.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.search.aggregations.bucket.filter;
+
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.Query;
+import org.elasticsearch.core.SuppressForbidden;
+
+import java.lang.reflect.Field;
+import java.util.Objects;
+
+class MergedDocValuesRangeQuery {
+
+    @SuppressForbidden(reason = "Uses reflection to access package-private lucene class")
+    public static Query merge(Query query, Query extraQuery) {
+        Class<? extends Query> queryClass = query.getClass();
+        Class<? extends Query> extraQueryClass = extraQuery.getClass();
+
+        if (queryClass.equals(extraQueryClass) == false
+            || queryClass.getCanonicalName().equals("org.apache.lucene.document.SortedNumericDocValuesRangeQuery") == false) {
+            return null;
+        }
+
+        try {
+            Field fieldName = queryClass.getDeclaredField("field");
+            fieldName.setAccessible(true);
+
+            String field = fieldName.get(query).toString();
+            if (Objects.equals(field, fieldName.get(extraQuery)) == false) {
+                return null;
+            }
+
+            Field lowerValue = queryClass.getDeclaredField("lowerValue");
+            Field upperValue = queryClass.getDeclaredField("upperValue");
+            lowerValue.setAccessible(true);
+            upperValue.setAccessible(true);
+
+            long q1LowerValue = lowerValue.getLong(query);
+            long q1UpperValue = upperValue.getLong(query);
+            long q2LowerValue = lowerValue.getLong(extraQuery);
+            long q2UpperValue = upperValue.getLong(extraQuery);
+
+            if (q1UpperValue < q2LowerValue || q2UpperValue < q1LowerValue) {
+                return new MatchNoDocsQuery("Non-overlapping range queries");
+            }
+
+            return NumericDocValuesField.newSlowRangeQuery(
+                field,
+                Math.max(q1LowerValue, q2LowerValue),
+                Math.min(q1UpperValue, q2UpperValue)
+            );
+
+        } catch (NoSuchFieldException | IllegalAccessException e) {
+            return null;
+        }
+    }
+}

server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java

@@ -132,13 +132,12 @@ QueryToFilterAdapter union(Query extraQuery) throws IOException {
         extraQuery = searcher().rewrite(new ConstantScoreQuery(extraQuery));
         Query unwrappedExtraQuery = unwrap(extraQuery);
         Query unwrappedQuery = unwrap(query);
-        if (unwrappedQuery instanceof PointRangeQuery q1 && unwrappedExtraQuery instanceof PointRangeQuery q2) {
-            Query merged = MergedPointRangeQuery.merge(q1, q2);
-            if (merged != null) {
-                // Should we rewrap here?
-                return new QueryToFilterAdapter(searcher(), key(), merged);
-            }
+
+        Query merged = maybeMergeRangeQueries(unwrappedQuery, unwrappedExtraQuery);
+        if (merged != null) {
+            return new QueryToFilterAdapter(searcher(), key(), merged);
         }
+
         BooleanQuery.Builder builder = new BooleanQuery.Builder();
         builder.add(query, BooleanClause.Occur.FILTER);
         builder.add(extraQuery, BooleanClause.Occur.FILTER);
@@ -155,6 +154,13 @@ public boolean isInefficientUnion() {
         };
     }
 
+    private static Query maybeMergeRangeQueries(Query query, Query extraQuery) throws IOException {
+        if (query instanceof PointRangeQuery q1 && extraQuery instanceof PointRangeQuery q2) {
+            return MergedPointRangeQuery.merge(q1, q2);
+        }
+        return MergedDocValuesRangeQuery.merge(query, extraQuery);
+    }
+
     private static Query unwrap(Query query) {
         while (true) {
             switch (query) {

server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java

@@ -12,6 +12,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValuesSkipper;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.PointValues;
 import org.apache.lucene.search.BooleanClause;
@@ -314,6 +315,12 @@ public Function<Rounding, Rounding.Prepared> roundingPreparer(AggregationContext
                             range[0] = dft.resolution().parsePointAsMillis(min);
                             range[1] = dft.resolution().parsePointAsMillis(max);
                         }
+                    } else if (dft.hasDocValuesSkipper()) {
+                        log.trace("Attempting to apply skipper-based data rounding");
+                        range[0] = dft.resolution()
+                            .roundDownToMillis(DocValuesSkipper.globalMinValue(context.searcher(), fieldContext.field()));
+                        range[1] = dft.resolution()
+                            .roundDownToMillis(DocValuesSkipper.globalMaxValue(context.searcher(), fieldContext.field()));
                     }
                     log.trace("Bounds after index bound date rounding: {}, {}", range[0], range[1]);
 

server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.mapper.RangeFieldMapper;
@@ -429,7 +430,11 @@ public Function<byte[], Number> getPointReaderOrNull() {
      * the ordering.
      */
     public boolean alignsWithSearchIndex() {
-        return script() == null && missing() == null && fieldType() != null && fieldType().indexType().supportsSortShortcuts();
+        boolean hasDocValuesSkipper = fieldType() instanceof DateFieldMapper.DateFieldType dft && dft.hasDocValuesSkipper();
+        return script() == null
+            && missing() == null
+            && fieldType() != null
+            && (fieldType().indexType().supportsSortShortcuts() || hasDocValuesSkipper);
     }
 
     /**

server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/MergedDocValuesRangeQueryTests.java

@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.search.aggregations.bucket.filter;
+
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.apache.lucene.search.TermQuery;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.nullValue;
+
+public class MergedDocValuesRangeQueryTests extends ESTestCase {
+
+    public void testNotRangeQueries() {
+        assertThat(
+            MergedDocValuesRangeQuery.merge(LongPoint.newRangeQuery("field", 1, 4), new TermQuery(new Term("field", "foo"))),
+            nullValue()
+        );
+
+        assertThat(
+            MergedDocValuesRangeQuery.merge(NumericDocValuesField.newSlowRangeQuery("field", 1, 4), LongPoint.newRangeQuery("field", 2, 4)),
+            nullValue()
+        );
+
+        assertThat(
+            MergedDocValuesRangeQuery.merge(LongPoint.newRangeQuery("field", 2, 4), NumericDocValuesField.newSlowRangeQuery("field", 1, 4)),
+            nullValue()
+        );
+    }
+
+    public void testDifferentFields() {
+        assertThat(
+            MergedDocValuesRangeQuery.merge(
+                NumericDocValuesField.newSlowRangeQuery("field1", 1, 4),
+                NumericDocValuesField.newSlowRangeQuery("field2", 2, 4)
+            ),
+            nullValue()
+        );
+    }
+
+    public void testNoOverlap() {
+        assertThat(
+            MergedDocValuesRangeQuery.merge(
+                NumericDocValuesField.newSlowRangeQuery("field", 1, 4),
+                NumericDocValuesField.newSlowRangeQuery("field", 6, 8)
+            ),
+            instanceOf(MatchNoDocsQuery.class)
+        );
+    }
+
+    public void testOverlap() {
+        assertThat(
+            MergedDocValuesRangeQuery.merge(
+                NumericDocValuesField.newSlowRangeQuery("field", 1, 6),
+                NumericDocValuesField.newSlowRangeQuery("field", 4, 8)
+            ),
+            equalTo(NumericDocValuesField.newSlowRangeQuery("field", 4, 6))
+        );
+
+        assertThat(
+            MergedDocValuesRangeQuery.merge(
+                NumericDocValuesField.newSlowRangeQuery("field", 4, 8),
+                NumericDocValuesField.newSlowRangeQuery("field", 1, 6)
+            ),
+            equalTo(NumericDocValuesField.newSlowRangeQuery("field", 4, 6))
+        );
+
+        assertThat(
+            MergedDocValuesRangeQuery.merge(
+                NumericDocValuesField.newSlowRangeQuery("field", 1, 8),
+                NumericDocValuesField.newSlowRangeQuery("field", 4, 6)
+            ),
+            equalTo(NumericDocValuesField.newSlowRangeQuery("field", 4, 6))
+        );
+
+        assertThat(
+            MergedDocValuesRangeQuery.merge(
+                NumericDocValuesField.newSlowRangeQuery("field", 4, 6),
+                NumericDocValuesField.newSlowRangeQuery("field", 1, 8)
+            ),
+            equalTo(NumericDocValuesField.newSlowRangeQuery("field", 4, 6))
+        );
+    }
+
+}

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java

@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.esql.stats;
 
+import org.apache.lucene.index.DocValuesSkipper;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
@@ -17,6 +18,7 @@
 import org.apache.lucene.index.PointValues;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.index.mapper.ConstantFieldType;
@@ -201,21 +203,29 @@ public Object min(FieldName field) {
         var stat = cache.computeIfAbsent(field.string(), this::makeFieldStats);
         // Consolidate min for indexed date fields only, skip the others and mixed-typed fields.
         MappedFieldType fieldType = stat.config.fieldType;
-        if (fieldType == null || stat.config.indexed == false || fieldType instanceof DateFieldType == false) {
+        boolean hasDocValueSkipper = fieldType instanceof DateFieldType dft && dft.hasDocValuesSkipper();
+        if (fieldType == null
+            || (hasDocValueSkipper == false && stat.config.indexed == false)
+            || fieldType instanceof DateFieldType == false) {
             return null;
         }
         if (stat.min == null) {
             var min = new long[] { Long.MAX_VALUE };
             Holder<Boolean> foundMinValue = new Holder<>(false);
             doWithContexts(r -> {
-                byte[] minPackedValue = PointValues.getMinPackedValue(r, field.string());
-                if (minPackedValue != null && minPackedValue.length == 8) {
-                    long minValue = NumericUtils.sortableBytesToLong(minPackedValue, 0);
-                    if (minValue <= min[0]) {
-                        min[0] = minValue;
-                        foundMinValue.set(true);
+                long minValue = Long.MAX_VALUE;
+                if (hasDocValueSkipper) {
+                    minValue = DocValuesSkipper.globalMinValue(new IndexSearcher(r), field.string());
+                } else {
+                    byte[] minPackedValue = PointValues.getMinPackedValue(r, field.string());
+                    if (minPackedValue != null && minPackedValue.length == 8) {
+                        minValue = NumericUtils.sortableBytesToLong(minPackedValue, 0);
                     }
                 }
+                if (minValue <= min[0]) {
+                    min[0] = minValue;
+                    foundMinValue.set(true);
+                }
                 return true;
             }, true);
             stat.min = foundMinValue.get() ? min[0] : null;
@@ -228,21 +238,29 @@ public Object max(FieldName field) {
         var stat = cache.computeIfAbsent(field.string(), this::makeFieldStats);
         // Consolidate max for indexed date fields only, skip the others and mixed-typed fields.
         MappedFieldType fieldType = stat.config.fieldType;
-        if (fieldType == null || stat.config.indexed == false || fieldType instanceof DateFieldType == false) {
+        boolean hasDocValueSkipper = fieldType instanceof DateFieldType dft && dft.hasDocValuesSkipper();
+        if (fieldType == null
+            || (hasDocValueSkipper == false && stat.config.indexed == false)
+            || fieldType instanceof DateFieldType == false) {
             return null;
         }
         if (stat.max == null) {
             var max = new long[] { Long.MIN_VALUE };
             Holder<Boolean> foundMaxValue = new Holder<>(false);
             doWithContexts(r -> {
-                byte[] maxPackedValue = PointValues.getMaxPackedValue(r, field.string());
-                if (maxPackedValue != null && maxPackedValue.length == 8) {
-                    long maxValue = NumericUtils.sortableBytesToLong(maxPackedValue, 0);
-                    if (maxValue >= max[0]) {
-                        max[0] = maxValue;
-                        foundMaxValue.set(true);
+                long maxValue = Long.MIN_VALUE;
+                if (hasDocValueSkipper) {
+                    maxValue = DocValuesSkipper.globalMaxValue(new IndexSearcher(r), field.string());
+                } else {
+                    byte[] maxPackedValue = PointValues.getMaxPackedValue(r, field.string());
+                    if (maxPackedValue != null && maxPackedValue.length == 8) {
+                        maxValue = NumericUtils.sortableBytesToLong(maxPackedValue, 0);
                     }
                 }
+                if (maxValue >= max[0]) {
+                    max[0] = maxValue;
+                    foundMaxValue.set(true);
+                }
                 return true;
             }, true);
             stat.max = foundMaxValue.get() ? max[0] : null;