Re-enable use_doc_values_skipper and add specialized lucene query for @timestamp field filtering.

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -678,7 +678,7 @@ public boolean isES87TSDBCodecEnabled() {
     public static final boolean DOC_VALUES_SKIPPER = new FeatureFlag("doc_values_skipper").isEnabled();
     public static final Setting<Boolean> USE_DOC_VALUES_SKIPPER = Setting.boolSetting(
         "index.mapping.use_doc_values_skipper",
-        false,
+        DOC_VALUES_SKIPPER,
         Property.IndexScope,
         Property.Final
     );
@@ -1098,7 +1098,9 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         recoverySourceEnabled = RecoverySettings.INDICES_RECOVERY_SOURCE_ENABLED_SETTING.get(nodeSettings);
         recoverySourceSyntheticEnabled = DiscoveryNode.isStateless(nodeSettings) == false
             && scopedSettings.get(RECOVERY_USE_SYNTHETIC_SOURCE_SETTING);
-        useDocValuesSkipper = DOC_VALUES_SKIPPER && scopedSettings.get(USE_DOC_VALUES_SKIPPER);
+        useDocValuesSkipper = DOC_VALUES_SKIPPER
+            && scopedSettings.get(USE_DOC_VALUES_SKIPPER)
+            && version.onOrAfter(IndexVersions.ENABLE_TSID_HOSTNAME_TIMESTAMP_DOC_VALUES_SKIPPERS_WITH_FF);
         if (recoverySourceSyntheticEnabled) {
             if (DiscoveryNode.isStateless(settings)) {
                 throw new IllegalArgumentException("synthetic recovery source is only allowed in stateful");

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -168,6 +168,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion DEFAULT_OVERSAMPLE_VALUE_FOR_BBQ = def(9_024_0_00, Version.LUCENE_10_2_1);
     public static final IndexVersion SEMANTIC_TEXT_DEFAULTS_TO_BBQ = def(9_025_0_00, Version.LUCENE_10_2_1);
     public static final IndexVersion DEFAULT_TO_ACORN_HNSW_FILTER_HEURISTIC = def(9_026_0_00, Version.LUCENE_10_2_1);
+    public static final IndexVersion ENABLE_TSID_HOSTNAME_TIMESTAMP_DOC_VALUES_SKIPPERS_WITH_FF = def(9_027_0_00, Version.LUCENE_10_2_1);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -25,6 +25,7 @@
 import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
@@ -38,7 +39,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.index.IndexMode;
-import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexSortConfig;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
@@ -50,6 +50,7 @@
 import org.elasticsearch.index.query.DateRangeIncludingNowQuery;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.lucene.queries.TimestampQuery;
 import org.elasticsearch.script.DateFieldScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptCompiler;
@@ -463,7 +464,7 @@ public DateFieldMapper build(MapperBuilderContext context) {
             c.getIndexSettings().getMode(),
             c.getIndexSettings().getIndexSortConfig(),
             c.indexVersionCreated(),
-            IndexSettings.USE_DOC_VALUES_SKIPPER.get(c.getSettings())
+            c.getIndexSettings().useDocValuesSkipper()
         );
     });
 
@@ -478,7 +479,7 @@ public DateFieldMapper build(MapperBuilderContext context) {
             c.getIndexSettings().getMode(),
             c.getIndexSettings().getIndexSortConfig(),
             c.indexVersionCreated(),
-            IndexSettings.USE_DOC_VALUES_SKIPPER.get(c.getSettings())
+            c.getIndexSettings().useDocValuesSkipper()
         );
     });
 
@@ -739,6 +740,16 @@ public Query rangeQuery(
                 parser = forcedDateParser;
             }
             return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> {
+                var indexSettings = context.getIndexSettings();
+                var indexMode = indexSettings.getMode();
+                boolean sortOnTimestamp = indexSettings.getIndexSortConfig().hasSortOnField(DataStream.TIMESTAMP_FIELD_NAME);
+                if ((indexMode == IndexMode.TIME_SERIES || indexMode == IndexMode.LOGSDB)
+                    && sortOnTimestamp
+                    && indexSettings.useDocValuesSkipper()
+                    && name().equals(DataStream.TIMESTAMP_FIELD_NAME)) {
+                    return new TimestampQuery(l, u);
+                }
+
                 Query query;
                 if (isIndexed()) {
                     query = LongPoint.newRangeQuery(name(), l, u);

server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java

@@ -14,7 +14,6 @@
 import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
-import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.ObjectMapper.Dynamic;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.xcontent.XContentParser;
@@ -409,7 +408,7 @@ public boolean newDynamicDateField(DocumentParserContext context, String name, D
                     context.indexSettings().getMode(),
                     context.indexSettings().getIndexSortConfig(),
                     context.indexSettings().getIndexVersionCreated(),
-                    IndexSettings.USE_DOC_VALUES_SKIPPER.get(context.indexSettings().getSettings())
+                    context.indexSettings().useDocValuesSkipper()
                 ),
                 context
             );

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -84,7 +84,6 @@
 import static org.apache.lucene.index.IndexWriter.MAX_TERM_LENGTH;
 import static org.elasticsearch.core.Strings.format;
 import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING;
-import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER;
 import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
 
 /**
@@ -215,7 +214,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex
                 mappingParserContext.getIndexSettings().getIndexVersionCreated(),
                 mappingParserContext.getIndexSettings().getMode(),
                 mappingParserContext.getIndexSettings().getIndexSortConfig(),
-                USE_DOC_VALUES_SKIPPER.get(mappingParserContext.getSettings()),
+                mappingParserContext.getIndexSettings().useDocValuesSkipper(),
                 mappingParserContext.getIndexSettings().sourceKeepMode()
             );
         }

server/src/main/java/org/elasticsearch/lucene/queries/TimestampComparator.java

@@ -0,0 +1,237 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package org.elasticsearch.lucene.queries;
+
+import org.apache.lucene.index.DocValuesSkipper;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.LeafFieldComparator;
+import org.apache.lucene.search.Scorable;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.util.FixedBitSet;
+import org.elasticsearch.cluster.metadata.DataStream;
+
+import java.io.IOException;
+
+/**
+ * Comparator when sorting using @timestamp field when logsdb or time series index modes are enabled.
+ */
+public class TimestampComparator extends FieldComparator<Long> {
+
+    private static final String FIELD_NAME = DataStream.TIMESTAMP_FIELD_NAME;
+
+    private final long[] values;
+    private final boolean reverse;
+
+    protected long topValue;
+    protected long bottom;
+
+    protected boolean topValueSet;
+    protected boolean singleSort; // singleSort is true, if sort is based on a single sort field.
+    protected boolean hitsThresholdReached;
+    protected boolean queueFull;
+
+    public TimestampComparator(int numHits, boolean reverse) {
+        this.reverse = reverse;
+        this.values = new long[numHits];
+    }
+
+    @Override
+    public void setTopValue(Long value) {
+        topValueSet = true;
+        topValue = value;
+    }
+
+    @Override
+    public void setSingleSort() {
+        singleSort = true;
+    }
+
+    @Override
+    public void disableSkipping() {}
+
+    @Override
+    public int compare(int slot1, int slot2) {
+        return Long.compare(values[slot1], values[slot2]);
+    }
+
+    @Override
+    public Long value(int slot) {
+        return Long.valueOf(values[slot]);
+    }
+
+    @Override
+    public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException {
+        return new TimestampLeafComparator(context.reader());
+    }
+
+    class TimestampLeafComparator implements LeafFieldComparator {
+
+        private final int maxDoc;
+        private final Sort indexSort;
+        private final boolean useTimestampFieldOnly;
+        private final NumericDocValues timestamps;
+        private final DocValuesSkipper timestampSkipper;
+        private final DocValuesSkipper primaryFieldSkipper;
+
+        private final boolean leafTopSet = topValueSet;
+
+        private DocIdSetIterator competitiveIterator;
+        private long iteratorCost = -1;
+
+        TimestampLeafComparator(LeafReader reader) throws IOException {
+            this.maxDoc = reader.maxDoc();
+            this.indexSort = reader.getMetaData().sort();
+            assert indexSort != null;
+            assert indexSort.getSort().length == 1 || indexSort.getSort().length == 2;
+
+            this.timestamps = TimestampQuery.getNumericDocValues(reader);
+            this.timestampSkipper = reader.getDocValuesSkipper(FIELD_NAME);
+
+            String primarySortField = indexSort.getSort()[0].getField();
+            boolean timestampIsPrimarySort = primarySortField.equals(FIELD_NAME);
+            var primaryFieldValues = timestampIsPrimarySort ? null : reader.getSortedDocValues(primarySortField);
+            useTimestampFieldOnly = timestampIsPrimarySort || (primaryFieldValues == null || primaryFieldValues.getValueCount() <= 1);
+            if (useTimestampFieldOnly == false) {
+                primaryFieldSkipper = reader.getDocValuesSkipper(primarySortField);
+            } else {
+                primaryFieldSkipper = null;
+            }
+
+            if (timestampSkipper == null || timestampSkipper.maxValue() < bottom) {
+                this.competitiveIterator = DocIdSetIterator.empty();
+            } else {
+                this.competitiveIterator = DocIdSetIterator.all(maxDoc);
+            }
+            this.iteratorCost = competitiveIterator.cost();
+        }
+
+        @Override
+        public void setBottom(int slot) throws IOException {
+            bottom = values[slot];
+            queueFull = true; // if we are setting bottom, it means that we have collected enough hits
+            updateCompetitiveIterator(); // update an iterator if we set a new bottom
+        }
+
+        @Override
+        public void copy(int slot, int doc) throws IOException {
+            values[slot] = getValueForDoc(doc);
+        }
+
+        @Override
+        public void setScorer(Scorable scorer) throws IOException {
+            if (iteratorCost == -1) {
+                if (scorer instanceof Scorer) {
+                    iteratorCost = ((Scorer) scorer).iterator().cost(); // starting iterator cost is the scorer's cost
+                } else {
+                    iteratorCost = maxDoc;
+                }
+                updateCompetitiveIterator(); // update an iterator when we have a new segment
+            }
+        }
+
+        @Override
+        public void setHitsThresholdReached() throws IOException {
+            hitsThresholdReached = true;
+            updateCompetitiveIterator();
+        }
+
+        @Override
+        public int compareBottom(int doc) throws IOException {
+            return Long.compare(bottom, getValueForDoc(doc));
+        }
+
+        @Override
+        public int compareTop(int doc) throws IOException {
+            return Long.compare(topValue, getValueForDoc(doc));
+        }
+
+        private long getValueForDoc(int doc) throws IOException {
+            boolean found = timestamps.advanceExact(doc);
+            assert found;
+            return timestamps.longValue();
+        }
+
+        // update its iterator to include possibly only docs that are "stronger" than the current bottom
+        // entry
+        private void updateCompetitiveIterator() throws IOException {
+            if (hitsThresholdReached == false || (leafTopSet == false && queueFull == false)) {
+                return;
+            }
+
+            long minTimestamp;
+            long maxTimestamp;
+            if (reverse) {
+                minTimestamp = bottom;
+                maxTimestamp = Long.MAX_VALUE;
+            } else {
+                minTimestamp = Long.MIN_VALUE;
+                maxTimestamp = bottom;
+            }
+
+            if (timestampSkipper.maxValue() < bottom) {
+                competitiveIterator = DocIdSetIterator.empty();
+            } else if (useTimestampFieldOnly) {
+                competitiveIterator = TimestampQuery.getIteratorIfTimestampIfPrimarySort(
+                    maxDoc,
+                    timestamps,
+                    timestampSkipper,
+                    minTimestamp,
+                    maxTimestamp
+                );
+            } else {
+                competitiveIterator = new TimestampIterator(timestamps, timestampSkipper, primaryFieldSkipper, minTimestamp, maxTimestamp);
+            }
+            iteratorCost = competitiveIterator.cost();
+        }
+
+        @Override
+        public DocIdSetIterator competitiveIterator() {
+            return new DocIdSetIterator() {
+                private int docID = competitiveIterator.docID();
+
+                @Override
+                public int nextDoc() throws IOException {
+                    return advance(docID + 1);
+                }
+
+                @Override
+                public int docID() {
+                    return docID;
+                }
+
+                @Override
+                public long cost() {
+                    return competitiveIterator.cost();
+                }
+
+                @Override
+                public int advance(int target) throws IOException {
+                    return docID = competitiveIterator.advance(target);
+                }
+
+                @Override
+                public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException {
+                    // The competitive iterator is usually a BitSetIterator, which has an optimized
+                    // implementation of #intoBitSet.
+                    if (competitiveIterator.docID() < docID) {
+                        competitiveIterator.advance(docID);
+                    }
+                    competitiveIterator.intoBitSet(upTo, bitSet, offset);
+                    docID = competitiveIterator.docID();
+                }
+            };
+        }
+
+    }
+}