diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index a49c635204522..b2668cd6791cd 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -190,6 +190,8 @@ private static Version parseUnchecked(String version) { public static final IndexVersion KEYWORD_MULTI_FIELDS_NOT_STORED_WHEN_IGNORED = def(9_040_0_00, Version.LUCENE_10_3_0); public static final IndexVersion UPGRADE_TO_LUCENE_10_3_1 = def(9_041_0_00, Version.LUCENE_10_3_1); + public static final IndexVersion REENABLED_TIMESTAMP_DOC_VALUES_SPARSE_INDEX = def(9_042_0_00, Version.LUCENE_10_3_1); + /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index e69793fb799cf..c6810ed0af4cf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -1164,7 +1164,7 @@ private DateFieldMapper( * Determines whether the doc values skipper (sparse index) should be used for the {@code @timestamp} field. *

* The doc values skipper is enabled only if {@code index.mapping.use_doc_values_skipper} is set to {@code true}, - * the index was created on or after {@link IndexVersions#TIMESTAMP_DOC_VALUES_SPARSE_INDEX}, and the + * the index was created on or after {@link IndexVersions#REENABLED_TIMESTAMP_DOC_VALUES_SPARSE_INDEX}, and the * field has doc values enabled. Additionally, the index mode must be {@link IndexMode#LOGSDB} or {@link IndexMode#TIME_SERIES}, and * the index sorting configuration must include the {@code @timestamp} field. * @@ -1186,7 +1186,7 @@ private static boolean shouldUseDocValuesSkipper( final IndexSortConfig indexSortConfig, final String fullFieldName ) { - return indexCreatedVersion.onOrAfter(IndexVersions.TIMESTAMP_DOC_VALUES_SPARSE_INDEX) + return indexCreatedVersion.onOrAfter(IndexVersions.REENABLED_TIMESTAMP_DOC_VALUES_SPARSE_INDEX) && useDocValuesSkipper && hasDocValues && (IndexMode.LOGSDB.equals(indexMode) || IndexMode.TIME_SERIES.equals(indexMode)) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedDocValuesRangeQuery.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedDocValuesRangeQuery.java new file mode 100644 index 0000000000000..1e0f3c3d12aae --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedDocValuesRangeQuery.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.search.aggregations.bucket.filter; + +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.core.SuppressForbidden; + +import java.lang.reflect.Field; +import java.util.Objects; + +class MergedDocValuesRangeQuery { + + @SuppressForbidden(reason = "Uses reflection to access package-private lucene class") + public static Query merge(Query query, Query extraQuery) { + Class queryClass = query.getClass(); + Class extraQueryClass = extraQuery.getClass(); + + if (queryClass.equals(extraQueryClass) == false + || queryClass.getCanonicalName().equals("org.apache.lucene.document.SortedNumericDocValuesRangeQuery") == false) { + return null; + } + + try { + Field fieldName = queryClass.getDeclaredField("field"); + fieldName.setAccessible(true); + + String field = fieldName.get(query).toString(); + if (Objects.equals(field, fieldName.get(extraQuery)) == false) { + return null; + } + + Field lowerValue = queryClass.getDeclaredField("lowerValue"); + Field upperValue = queryClass.getDeclaredField("upperValue"); + lowerValue.setAccessible(true); + upperValue.setAccessible(true); + + long q1LowerValue = lowerValue.getLong(query); + long q1UpperValue = upperValue.getLong(query); + long q2LowerValue = lowerValue.getLong(extraQuery); + long q2UpperValue = upperValue.getLong(extraQuery); + + if (q1UpperValue < q2LowerValue || q2UpperValue < q1LowerValue) { + return new MatchNoDocsQuery("Non-overlapping range queries"); + } + + return NumericDocValuesField.newSlowRangeQuery( + field, + Math.max(q1LowerValue, q2LowerValue), + Math.min(q1UpperValue, q2UpperValue) + ); + + } catch (NoSuchFieldException | IllegalAccessException e) { + return null; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java index 4b7b41c9022bc..84e4563169862 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java @@ -132,13 +132,12 @@ QueryToFilterAdapter union(Query extraQuery) throws IOException { extraQuery = searcher().rewrite(new ConstantScoreQuery(extraQuery)); Query unwrappedExtraQuery = unwrap(extraQuery); Query unwrappedQuery = unwrap(query); - if (unwrappedQuery instanceof PointRangeQuery q1 && unwrappedExtraQuery instanceof PointRangeQuery q2) { - Query merged = MergedPointRangeQuery.merge(q1, q2); - if (merged != null) { - // Should we rewrap here? - return new QueryToFilterAdapter(searcher(), key(), merged); - } + + Query merged = maybeMergeRangeQueries(unwrappedQuery, unwrappedExtraQuery); + if (merged != null) { + return new QueryToFilterAdapter(searcher(), key(), merged); } + BooleanQuery.Builder builder = new BooleanQuery.Builder(); builder.add(query, BooleanClause.Occur.FILTER); builder.add(extraQuery, BooleanClause.Occur.FILTER); @@ -155,6 +154,13 @@ public boolean isInefficientUnion() { }; } + private static Query maybeMergeRangeQueries(Query query, Query extraQuery) throws IOException { + if (query instanceof PointRangeQuery q1 && extraQuery instanceof PointRangeQuery q2) { + return MergedPointRangeQuery.merge(q1, q2); + } + return MergedDocValuesRangeQuery.merge(query, extraQuery); + } + private static Query unwrap(Query query) { while (true) { switch (query) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java index c711d990eec37..eb2923b3703d3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java @@ -12,6 +12,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipper; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PointValues; import org.apache.lucene.search.BooleanClause; @@ -314,6 +315,12 @@ public Function roundingPreparer(AggregationContext range[0] = dft.resolution().parsePointAsMillis(min); range[1] = dft.resolution().parsePointAsMillis(max); } + } else if (dft.hasDocValuesSkipper()) { + log.trace("Attempting to apply skipper-based data rounding"); + range[0] = dft.resolution() + .roundDownToMillis(DocValuesSkipper.globalMinValue(context.searcher(), fieldContext.field())); + range[1] = dft.resolution() + .roundDownToMillis(DocValuesSkipper.globalMaxValue(context.searcher(), fieldContext.field())); } log.trace("Bounds after index bound date rounding: {}, {}", range[0], range[1]); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java index cff972b18df9d..2768f7d668244 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java @@ -13,6 +13,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.RangeFieldMapper; @@ -429,7 +430,11 @@ public Function getPointReaderOrNull() { * the ordering. */ public boolean alignsWithSearchIndex() { - return script() == null && missing() == null && fieldType() != null && fieldType().indexType().supportsSortShortcuts(); + boolean hasDocValuesSkipper = fieldType() instanceof DateFieldMapper.DateFieldType dft && dft.hasDocValuesSkipper(); + return script() == null + && missing() == null + && fieldType() != null + && (fieldType().indexType().supportsSortShortcuts() || hasDocValuesSkipper); } /** diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/MergedDocValuesRangeQueryTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/MergedDocValuesRangeQueryTests.java new file mode 100644 index 0000000000000..37caa9e360057 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/MergedDocValuesRangeQueryTests.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.search.aggregations.bucket.filter; + +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.TermQuery; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.nullValue; + +public class MergedDocValuesRangeQueryTests extends ESTestCase { + + public void testNotRangeQueries() { + assertThat( + MergedDocValuesRangeQuery.merge(LongPoint.newRangeQuery("field", 1, 4), new TermQuery(new Term("field", "foo"))), + nullValue() + ); + + assertThat( + MergedDocValuesRangeQuery.merge(NumericDocValuesField.newSlowRangeQuery("field", 1, 4), LongPoint.newRangeQuery("field", 2, 4)), + nullValue() + ); + + assertThat( + MergedDocValuesRangeQuery.merge(LongPoint.newRangeQuery("field", 2, 4), NumericDocValuesField.newSlowRangeQuery("field", 1, 4)), + nullValue() + ); + } + + public void testDifferentFields() { + assertThat( + MergedDocValuesRangeQuery.merge( + NumericDocValuesField.newSlowRangeQuery("field1", 1, 4), + NumericDocValuesField.newSlowRangeQuery("field2", 2, 4) + ), + nullValue() + ); + } + + public void testNoOverlap() { + assertThat( + MergedDocValuesRangeQuery.merge( + NumericDocValuesField.newSlowRangeQuery("field", 1, 4), + NumericDocValuesField.newSlowRangeQuery("field", 6, 8) + ), + instanceOf(MatchNoDocsQuery.class) + ); + } + + public void testOverlap() { + assertThat( + MergedDocValuesRangeQuery.merge( + NumericDocValuesField.newSlowRangeQuery("field", 1, 6), + NumericDocValuesField.newSlowRangeQuery("field", 4, 8) + ), + equalTo(NumericDocValuesField.newSlowRangeQuery("field", 4, 6)) + ); + + assertThat( + MergedDocValuesRangeQuery.merge( + NumericDocValuesField.newSlowRangeQuery("field", 4, 8), + NumericDocValuesField.newSlowRangeQuery("field", 1, 6) + ), + equalTo(NumericDocValuesField.newSlowRangeQuery("field", 4, 6)) + ); + + assertThat( + MergedDocValuesRangeQuery.merge( + NumericDocValuesField.newSlowRangeQuery("field", 1, 8), + NumericDocValuesField.newSlowRangeQuery("field", 4, 6) + ), + equalTo(NumericDocValuesField.newSlowRangeQuery("field", 4, 6)) + ); + + assertThat( + MergedDocValuesRangeQuery.merge( + NumericDocValuesField.newSlowRangeQuery("field", 4, 6), + NumericDocValuesField.newSlowRangeQuery("field", 1, 8) + ), + equalTo(NumericDocValuesField.newSlowRangeQuery("field", 4, 6)) + ); + } + +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java index 31f90edd43b6a..c452ab4f6a292 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.stats; +import org.apache.lucene.index.DocValuesSkipper; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -17,6 +18,7 @@ import org.apache.lucene.index.PointValues; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; import org.elasticsearch.index.mapper.ConstantFieldType; @@ -201,21 +203,29 @@ public Object min(FieldName field) { var stat = cache.computeIfAbsent(field.string(), this::makeFieldStats); // Consolidate min for indexed date fields only, skip the others and mixed-typed fields. MappedFieldType fieldType = stat.config.fieldType; - if (fieldType == null || stat.config.indexed == false || fieldType instanceof DateFieldType == false) { + boolean hasDocValueSkipper = fieldType instanceof DateFieldType dft && dft.hasDocValuesSkipper(); + if (fieldType == null + || (hasDocValueSkipper == false && stat.config.indexed == false) + || fieldType instanceof DateFieldType == false) { return null; } if (stat.min == null) { var min = new long[] { Long.MAX_VALUE }; Holder foundMinValue = new Holder<>(false); doWithContexts(r -> { - byte[] minPackedValue = PointValues.getMinPackedValue(r, field.string()); - if (minPackedValue != null && minPackedValue.length == 8) { - long minValue = NumericUtils.sortableBytesToLong(minPackedValue, 0); - if (minValue <= min[0]) { - min[0] = minValue; - foundMinValue.set(true); + long minValue = Long.MAX_VALUE; + if (hasDocValueSkipper) { + minValue = DocValuesSkipper.globalMinValue(new IndexSearcher(r), field.string()); + } else { + byte[] minPackedValue = PointValues.getMinPackedValue(r, field.string()); + if (minPackedValue != null && minPackedValue.length == 8) { + minValue = NumericUtils.sortableBytesToLong(minPackedValue, 0); } } + if (minValue <= min[0]) { + min[0] = minValue; + foundMinValue.set(true); + } return true; }, true); stat.min = foundMinValue.get() ? min[0] : null; @@ -228,21 +238,29 @@ public Object max(FieldName field) { var stat = cache.computeIfAbsent(field.string(), this::makeFieldStats); // Consolidate max for indexed date fields only, skip the others and mixed-typed fields. MappedFieldType fieldType = stat.config.fieldType; - if (fieldType == null || stat.config.indexed == false || fieldType instanceof DateFieldType == false) { + boolean hasDocValueSkipper = fieldType instanceof DateFieldType dft && dft.hasDocValuesSkipper(); + if (fieldType == null + || (hasDocValueSkipper == false && stat.config.indexed == false) + || fieldType instanceof DateFieldType == false) { return null; } if (stat.max == null) { var max = new long[] { Long.MIN_VALUE }; Holder foundMaxValue = new Holder<>(false); doWithContexts(r -> { - byte[] maxPackedValue = PointValues.getMaxPackedValue(r, field.string()); - if (maxPackedValue != null && maxPackedValue.length == 8) { - long maxValue = NumericUtils.sortableBytesToLong(maxPackedValue, 0); - if (maxValue >= max[0]) { - max[0] = maxValue; - foundMaxValue.set(true); + long maxValue = Long.MIN_VALUE; + if (hasDocValueSkipper) { + maxValue = DocValuesSkipper.globalMaxValue(new IndexSearcher(r), field.string()); + } else { + byte[] maxPackedValue = PointValues.getMaxPackedValue(r, field.string()); + if (maxPackedValue != null && maxPackedValue.length == 8) { + maxValue = NumericUtils.sortableBytesToLong(maxPackedValue, 0); } } + if (maxValue >= max[0]) { + max[0] = maxValue; + foundMaxValue.set(true); + } return true; }, true); stat.max = foundMaxValue.get() ? max[0] : null;