DATE_RANGE data type for esql

docs/changelog/133309.yaml

@@ -0,0 +1,5 @@
+pr: 133309
+summary: support DATE_RANGE field type 
+area: ES|QL
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java

@@ -496,6 +496,8 @@ interface BlockFactory {
         SortedSetOrdinalsBuilder sortedSetOrdinalsBuilder(SortedSetDocValues ordinals, int count);
 
         AggregateMetricDoubleBuilder aggregateMetricDoubleBuilder(int count);
+
+        DateRangeBuilder dateRangeBuilder(int count);
     }
 
     /**
@@ -623,4 +625,10 @@ interface AggregateMetricDoubleBuilder extends Builder {
 
         IntBuilder count();
     }
+
+    interface DateRangeBuilder extends Builder {
+        LongBuilder from();
+
+        LongBuilder to();
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java

@@ -9,10 +9,13 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.network.InetAddresses;
@@ -61,6 +64,8 @@ public class RangeFieldMapper extends FieldMapper {
     public static final boolean DEFAULT_INCLUDE_UPPER = true;
     public static final boolean DEFAULT_INCLUDE_LOWER = true;
 
+    public static final TransportVersion ESQL_DATE_RANGE_CREATED_VERSION = TransportVersion.fromName("esql_date_range_created_version");
+
     public static class Defaults {
         public static final DateFormatter DATE_FORMATTER = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER;
         public static final Locale LOCALE = DateFieldMapper.DEFAULT_LOCALE;
@@ -342,6 +347,96 @@ public Query rangeQuery(
                 context
             );
         }
+
+        public static class DateRangeDocValuesLoader extends BlockDocValuesReader.DocValuesBlockLoader {
+            private final String fieldName;
+
+            public DateRangeDocValuesLoader(String fieldName) {
+                this.fieldName = fieldName;
+            }
+
+            @Override
+            public Builder builder(BlockFactory factory, int expectedCount) {
+                return factory.longsFromDocValues(expectedCount);
+            }
+
+            @Override
+            public AllReader reader(LeafReaderContext context) throws IOException {
+                var docValues = context.reader().getBinaryDocValues(fieldName);
+                return new DateRangeDocValuesReader(docValues);
+            }
+        }
+
+        @Override
+        public BlockLoader blockLoader(BlockLoaderContext blContext) {
+            if (hasDocValues()) {
+                return new DateRangeDocValuesLoader(name());
+            }
+            throw new IllegalStateException("Cannot load blocks without doc values");
+        }
+    }
+
+    public static class DateRangeDocValuesReader extends BlockDocValuesReader {
+        private final BytesRef spare = new BytesRef();
+
+        private final BinaryDocValues numericDocValues;
+
+        public DateRangeDocValuesReader(BinaryDocValues numericDocValues) {
+            this.numericDocValues = numericDocValues;
+        }
+
+        private int docId = -1;
+
+        @Override
+        protected int docId() {
+            return docId;
+        }
+
+        @Override
+        public String toString() {
+            return "BlockDocValuesReader.DateRangeDocValuesReader";
+        }
+
+        @Override
+        public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs, int offset, boolean nullsFiltered)
+            throws IOException {
+            try (BlockLoader.DateRangeBuilder builder = factory.dateRangeBuilder(docs.count() - offset)) {
+                int lastDoc = -1;
+                for (int i = offset; i < docs.count(); i++) {
+                    int doc = docs.get(i);
+                    if (doc < lastDoc) {
+                        throw new IllegalStateException("docs within same block must be in order");
+                    }
+                    if (false == numericDocValues.advanceExact(doc)) {
+                        builder.appendNull();
+                    } else {
+                        BytesRef ref = numericDocValues.binaryValue();
+                        var ranges = BinaryRangeUtil.decodeLongRanges(ref);
+                        for (var range : ranges) {
+                            lastDoc = doc;
+                            this.docId = doc;
+                            builder.from().appendLong((long) range.from);
+                            builder.to().appendLong((long) range.to);
+                        }
+                    }
+                }
+                return builder.build();
+            }
+        }
+
+        @Override
+        public void read(int doc, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException {
+            var blockBuilder = (BlockLoader.DateRangeBuilder) builder;
+            this.docId = doc;
+            if (false == numericDocValues.advanceExact(doc)) {
+                blockBuilder.appendNull();
+            } else {
+                var range = BinaryRangeUtil.decodeLongRanges(numericDocValues.binaryValue());
+                assert range.size() == 1 : "stored fields should only have a single range";
+                blockBuilder.from().appendLong((long) range.getFirst().from);
+                blockBuilder.to().appendLong((long) range.getFirst().to);
+            }
+        }
     }
 
     private final RangeType type;

server/src/main/resources/transport/definitions/referable/esql_date_range_created_version.csv

@@ -0,0 +1 @@
+9192000

server/src/main/resources/transport/upper_bounds/9.3.csv

@@ -1 +1 @@
-add_cross_cluster_api_key_signature,9191000
+esql_date_range_created_version,9192000

test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java

@@ -414,9 +414,15 @@ public SortedSetOrdinalBuilder appendOrd(int value) {
                 return new SortedSetOrdinalBuilder();
             }
 
+            @Override
             public BlockLoader.AggregateMetricDoubleBuilder aggregateMetricDoubleBuilder(int expectedSize) {
                 return new AggregateMetricDoubleBlockBuilder(expectedSize);
             }
+
+            @Override
+            public BlockLoader.DateRangeBuilder dateRangeBuilder(int expectedSize) {
+                return new DateRangeBuilder(expectedSize);
+            }
         };
     }
 
@@ -624,4 +630,68 @@ public void close() {
 
         }
     }
+
+    public static class DateRangeBuilder implements BlockLoader.DateRangeBuilder {
+        private final LongBuilder from;
+        private final LongBuilder to;
+
+        DateRangeBuilder(int expectedSize) {
+            from = new LongBuilder(expectedSize);
+            to = new LongBuilder(expectedSize);
+        }
+
+        @Override
+        public BlockLoader.LongBuilder from() {
+            return from;
+        }
+
+        @Override
+        public BlockLoader.LongBuilder to() {
+            return to;
+        }
+
+        @Override
+        public BlockLoader.Block build() {
+            var fromBlock = from.build();
+            var toBlock = to.build();
+            assert fromBlock.size() == toBlock.size();
+            var values = new ArrayList<>(fromBlock.size());
+            for (int i = 0; i < fromBlock.size(); i++) {
+                values.add(List.of(fromBlock.values.get(i), toBlock.values.get(i)));
+            }
+            return new TestBlock(values);
+        }
+
+        @Override
+        public BlockLoader.Builder appendNull() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public BlockLoader.Builder beginPositionEntry() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public BlockLoader.Builder endPositionEntry() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void close() {
+
+        }
+
+        private static class LongBuilder extends TestBlock.Builder implements BlockLoader.LongBuilder {
+            private LongBuilder(int expectedSize) {
+                super(expectedSize);
+            }
+
+            @Override
+            public BlockLoader.LongBuilder appendLong(long value) {
+                add(value);
+                return this;
+            }
+        }
+    };
 }

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java

@@ -21,6 +21,7 @@
 import static org.elasticsearch.xpack.esql.core.type.DataType.AGGREGATE_METRIC_DOUBLE;
 import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
 import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_RANGE;
 import static org.elasticsearch.xpack.esql.core.type.DataType.DENSE_VECTOR;
 import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
 import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;
@@ -85,10 +86,10 @@ public static TypeResolution isRepresentableExceptCountersDenseVectorAndAggregat
     ) {
         return isType(
             e,
-            dt -> isRepresentable(dt) && dt != DENSE_VECTOR && dt != AGGREGATE_METRIC_DOUBLE,
+            dt -> isRepresentable(dt) && dt != DENSE_VECTOR && dt != AGGREGATE_METRIC_DOUBLE && dt != DATE_RANGE,
             operationName,
             paramOrd,
-            "any type except counter types, dense_vector, or aggregate_metric_double"
+            "any type except counter types, dense_vector, aggregate_metric_double or date_range"
         );
     }
 
@@ -99,10 +100,14 @@ public static TypeResolution isRepresentableExceptCountersSpatialDenseVectorAndA
     ) {
         return isType(
             e,
-            (t) -> isSpatialOrGrid(t) == false && DataType.isRepresentable(t) && t != DENSE_VECTOR && t != AGGREGATE_METRIC_DOUBLE,
+            (t) -> isSpatialOrGrid(t) == false
+                && DataType.isRepresentable(t)
+                && t != DENSE_VECTOR
+                && t != AGGREGATE_METRIC_DOUBLE
+                && t != DATE_RANGE,
             operationName,
             paramOrd,
-            "any type except counter, spatial types, dense_vector, or aggregate_metric_double"
+            "any type except counter, spatial types, dense_vector, aggregate_metric_double or date_range"
         );
     }
 

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java

@@ -33,6 +33,7 @@
 import java.util.stream.Collectors;
 
 import static java.util.stream.Collectors.toMap;
+import static org.elasticsearch.index.mapper.RangeFieldMapper.ESQL_DATE_RANGE_CREATED_VERSION;
 
 /**
  * This enum represents data types the ES|QL query processing layer is able to
@@ -270,6 +271,7 @@ public enum DataType implements Writeable {
      * Nanosecond precision date, stored as a 64-bit signed number.
      */
     DATE_NANOS(builder().esType("date_nanos").estimatedSize(Long.BYTES).docValues().supportedOnAllNodes()),
+    DATE_RANGE(builder().esType("date_range").estimatedSize(2 * Long.BYTES).docValues().supportedOn(ESQL_DATE_RANGE_CREATED_VERSION)),
     /**
      * IP addresses. IPv4 address are always
      * <a href="https://datatracker.ietf.org/doc/html/rfc4291#section-2.5.5">embedded</a>
@@ -982,5 +984,6 @@ private static class DataTypesTransportVersions {
         public static final TransportVersion ESQL_AGGREGATE_METRIC_DOUBLE_CREATED_VERSION = TransportVersion.fromName(
             "esql_aggregate_metric_double_created_version"
         );
+
     }
 }

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java

@@ -481,6 +481,22 @@ public final AggregateMetricDoubleBlock newAggregateMetricDoubleBlock(
         return new AggregateMetricDoubleArrayBlock(min, max, sum, count);
     }
 
+    public DateRangeBlockBuilder newDateRangeBlockBuilder(int estimatedSize) {
+        return new DateRangeBlockBuilder(estimatedSize, this);
+    }
+
+    public DateRangeBlock newConstantDateRangeBlock(DateRangeBlockBuilder.DateRangeLiteral value, int positions) {
+        var fromBuilder = newConstantLongBlockWith(value.from(), positions);
+        var toBuilder = newConstantLongBlockWith(value.to(), positions);
+        return new DateRangeArrayBlock(fromBuilder, toBuilder);
+    }
+
+    public DateRangeBlock newDateRangeBlock(long[] fromValues, long[] toValues, int positions) {
+        var from = newLongArrayVector(fromValues, positions).asBlock();
+        var to = newLongArrayVector(toValues, positions).asBlock();
+        return new DateRangeArrayBlock(from, to);
+    }
+
     /**
      * Returns the maximum number of bytes that a Block should be backed by a primitive array before switching to using BigArrays.
      */

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java

@@ -224,6 +224,7 @@ public static void appendValue(Block.Builder builder, Object val, ElementType ty
             case DOUBLE -> ((DoubleBlock.Builder) builder).appendDouble((Double) val);
             case BOOLEAN -> ((BooleanBlock.Builder) builder).appendBoolean((Boolean) val);
             case AGGREGATE_METRIC_DOUBLE -> ((AggregateMetricDoubleBlockBuilder) builder).appendLiteral((AggregateMetricDoubleLiteral) val);
+            case DATE_RANGE -> ((DateRangeBlockBuilder) builder).appendDateRange((DateRangeBlockBuilder.DateRangeLiteral) val);
             default -> throw new UnsupportedOperationException("unsupported element type [" + type + "]");
         }
     }
@@ -253,6 +254,7 @@ private static Block constantBlock(BlockFactory blockFactory, ElementType type,
             case BOOLEAN -> blockFactory.newConstantBooleanBlockWith((boolean) val, size);
             case AGGREGATE_METRIC_DOUBLE -> blockFactory.newConstantAggregateMetricDoubleBlock((AggregateMetricDoubleLiteral) val, size);
             case FLOAT -> blockFactory.newConstantFloatBlockWith((float) val, size);
+            case DATE_RANGE -> blockFactory.newConstantDateRangeBlock((DateRangeBlockBuilder.DateRangeLiteral) val, size);
             default -> throw new UnsupportedOperationException("unsupported element type [" + type + "]");
         };
     }
@@ -306,6 +308,12 @@ yield new AggregateMetricDoubleLiteral(
                     aggBlock.countBlock().getInt(offset)
                 );
             }
+            case DATE_RANGE -> {
+                DateRangeBlock b = (DateRangeBlock) block;
+                LongBlock fromBlock = b.getFromBlock();
+                LongBlock toBlock = b.getToBlock();
+                yield new DateRangeBlockBuilder.DateRangeLiteral(fromBlock.getLong(offset), toBlock.getLong(offset));
+            }
             case UNKNOWN -> throw new IllegalArgumentException("can't read values from [" + block + "]");
         };
     }

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java

@@ -26,7 +26,8 @@ public final class ConstantNullBlock extends AbstractNonThreadSafeRefCounted
         FloatBlock,
         DoubleBlock,
         BytesRefBlock,
-        AggregateMetricDoubleBlock {
+        AggregateMetricDoubleBlock,
+        DateRangeBlock {
 
     private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantNullBlock.class);
     private final int positionCount;
@@ -118,6 +119,16 @@ public ConstantNullBlock expand() {
         return this;
     }
 
+    @Override
+    public LongBlock getFromBlock() {
+        return this;
+    }
+
+    @Override
+    public LongBlock getToBlock() {
+        return this;
+    }
+
     @Override
     public long ramBytesUsed() {
         return BASE_RAM_BYTES_USED;