diff --git a/docs/changelog/137533.yaml b/docs/changelog/137533.yaml new file mode 100644 index 0000000000000..cfd603ee4950f --- /dev/null +++ b/docs/changelog/137533.yaml @@ -0,0 +1,5 @@ +pr: 137533 +summary: Speed up sorts on secondary sort fields +area: Search +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java index 0ef0e3ffa28c8..517077bfe5401 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -634,12 +634,8 @@ private static SortField rewriteMergeSortField(SortField sortField) { SortField newSortField = new SortField(sortField.getField(), SortField.Type.STRING, sortField.getReverse()); newSortField.setMissingValue(sortField.getMissingValue()); return newSortField; - } else if (sortField.getClass() == SortedNumericSortField.class) { - SortField newSortField = new SortField( - sortField.getField(), - ((SortedNumericSortField) sortField).getNumericType(), - sortField.getReverse() - ); + } else if (sortField instanceof SortedNumericSortField snsf) { + SortField newSortField = new SortField(sortField.getField(), snsf.getNumericType(), sortField.getReverse()); newSortField.setMissingValue(sortField.getMissingValue()); return newSortField; } else if (sortField.getClass() == ShardDocSortField.class) { @@ -651,9 +647,6 @@ private static SortField rewriteMergeSortField(SortField sortField) { static void writeSortField(StreamOutput out, SortField sortField) throws IOException { sortField = rewriteMergeSortField(sortField); - if (sortField.getClass() != SortField.class) { - throw new IllegalArgumentException("Cannot serialize SortField impl [" + sortField + "]"); - } out.writeOptionalString(sortField.getField()); if (sortField.getComparatorSource() != null) { IndexFieldData.XFieldComparatorSource comparatorSource = (IndexFieldData.XFieldComparatorSource) sortField diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java index c5fcb0207ce4d..a1f2b30332630 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java @@ -80,7 +80,8 @@ public FieldComparator newComparator(String fieldname, int numHits, Pruning e final double dMissingValue = (Double) missingObject(missingValue, reversed); // NOTE: it's important to pass null as a missing value in the constructor so that // the comparator doesn't check docsWithField since we replace missing values in select() - return new DoubleComparator(numHits, null, null, reversed, Pruning.NONE) { + // TODO we can re-enable pruning here if we allow NumericDoubleValues to expose an iterator + return new DoubleComparator(numHits, fieldname, null, reversed, Pruning.NONE) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new DoubleLeafComparator(context) { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java index 79f1fdb25a0a6..6e30c94411953 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/FloatValuesComparatorSource.java @@ -73,7 +73,8 @@ public FieldComparator newComparator(String fieldname, int numHits, Pruning e final float fMissingValue = (Float) missingObject(missingValue, reversed); // NOTE: it's important to pass null as a missing value in the constructor so that // the comparator doesn't check docsWithField since we replace missing values in select() - return new FloatComparator(numHits, null, null, reversed, Pruning.NONE) { + // TODO we can re-enable pruning here if we allow NumericDoubleValues to expose an iterator + return new FloatComparator(numHits, fieldname, null, reversed, Pruning.NONE) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new FloatLeafComparator(context) { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/HalfFloatValuesComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/HalfFloatValuesComparatorSource.java index ade3f5ccc5a3a..aa2c7c9bdef5d 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/HalfFloatValuesComparatorSource.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/HalfFloatValuesComparatorSource.java @@ -39,7 +39,8 @@ public FieldComparator newComparator(String fieldname, int numHits, Pruning e final float fMissingValue = (Float) missingObject(missingValue, reversed); // NOTE: it's important to pass null as a missing value in the constructor so that // the comparator doesn't check docsWithField since we replace missing values in select() - return new HalfFloatComparator(numHits, fieldname, null, reversed, enableSkipping) { + // TODO we can re-enable pruning here if we allow NumericDoubleValues to expose an iterator + return new HalfFloatComparator(numHits, fieldname, null, reversed, Pruning.NONE) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { return new HalfFloatLeafComparator(context) { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java index 544b933772065..d2fdd2f2900a6 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/IntValuesComparatorSource.java @@ -60,7 +60,7 @@ public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws I return new IntLeafComparator(context) { @Override protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException { - return wrap(getLongValues(context, iMissingValue)); + return wrap(getLongValues(context, iMissingValue), context.reader().maxDoc()); } }; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java index d27c1b4784311..82830d5bfe3bd 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/LongValuesComparatorSource.java @@ -8,6 +8,7 @@ */ package org.elasticsearch.index.fielddata.fieldcomparator; +import org.apache.lucene.index.DocValuesSkipper; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.search.DocIdSetIterator; @@ -15,8 +16,8 @@ import org.apache.lucene.search.LeafFieldComparator; import org.apache.lucene.search.LongValues; import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; -import org.apache.lucene.search.comparators.LongComparator; import org.apache.lucene.util.BitSet; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.util.BigArrays; @@ -28,6 +29,8 @@ import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.SortedNumericLongValues; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; +import org.elasticsearch.lucene.comparators.XLongComparator; +import org.elasticsearch.lucene.comparators.XNumericComparator; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.sort.BucketedSort; @@ -103,13 +106,48 @@ public FieldComparator newComparator(String fieldname, int numHits, Pruning e final long lMissingValue = (Long) missingObject(missingValue, reversed); // NOTE: it's important to pass null as a missing value in the constructor so that // the comparator doesn't check docsWithField since we replace missing values in select() - return new LongComparator(numHits, null, null, reversed, Pruning.NONE) { + return new XLongComparator(numHits, fieldname, null, reversed, enableSkipping) { @Override public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { + final int maxDoc = context.reader().maxDoc(); return new LongLeafComparator(context) { @Override protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException { - return wrap(getLongValues(context, lMissingValue)); + return wrap(getLongValues(context, lMissingValue), maxDoc); + } + + @Override + protected XNumericComparator.CompetitiveDISIBuilder buildCompetitiveDISIBuilder(LeafReaderContext context) + throws IOException { + Sort indexSort = context.reader().getMetaData().sort(); + if (indexSort == null) { + return super.buildCompetitiveDISIBuilder(context); + } + SortField[] sortFields = indexSort.getSort(); + if (sortFields.length != 2) { + return super.buildCompetitiveDISIBuilder(context); + } + if (sortFields[1].getField().equals(field) == false) { + return super.buildCompetitiveDISIBuilder(context); + } + DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field); + DocValuesSkipper primaryFieldSkipper = context.reader().getDocValuesSkipper(sortFields[0].getField()); + if (primaryFieldSkipper == null || skipper.docCount() != maxDoc || primaryFieldSkipper.docCount() != maxDoc) { + return super.buildCompetitiveDISIBuilder(context); + } + return new CompetitiveDISIBuilder(this) { + @Override + protected int docCount() { + return skipper.docCount(); + } + + @Override + protected void doUpdateCompetitiveIterator() { + competitiveIterator.update( + new SecondarySortIterator(docValues, skipper, primaryFieldSkipper, minValueAsLong, maxValueAsLong) + ); + } + }; } }; } @@ -163,8 +201,11 @@ public Object missingObject(Object missingValue, boolean reversed) { return super.missingObject(missingValue, reversed); } - protected static NumericDocValues wrap(LongValues longValues) { + protected static NumericDocValues wrap(LongValues longValues, int maxDoc) { return new NumericDocValues() { + + int doc = -1; + @Override public long longValue() throws IOException { return longValues.longValue(); @@ -172,22 +213,31 @@ public long longValue() throws IOException { @Override public boolean advanceExact(int target) throws IOException { + doc = target; return longValues.advanceExact(target); } @Override public int docID() { - throw new UnsupportedOperationException(); + return doc; } @Override public int nextDoc() throws IOException { - throw new UnsupportedOperationException(); + return advance(doc + 1); } @Override public int advance(int target) throws IOException { - throw new UnsupportedOperationException(); + if (target >= maxDoc) { + return doc = NO_MORE_DOCS; + } + // All documents are guaranteed to have a value, as all invocations of getLongValues + // always return `true` from `advanceExact()` + boolean hasValue = longValues.advanceExact(target); + assert hasValue : "LongValuesComparatorSource#wrap called with a LongValues that has missing values"; + doc = target; + return target; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/SecondarySortIterator.java b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/SecondarySortIterator.java new file mode 100644 index 0000000000000..405cc58080751 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/SecondarySortIterator.java @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.fielddata.fieldcomparator; + +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; + +import java.io.IOException; + +/** + * A competitive DocIdSetIterator that examines the values of a secondary + * sort field and tries to exclude documents with values outside a given + * range, using DocValueSkippers on the primary sort field to advance rapidly + * to the next block of values. + */ +class SecondarySortIterator extends DocIdSetIterator { + + final NumericDocValues values; + + final DocValuesSkipper valueSkipper; + final DocValuesSkipper primaryFieldSkipper; + final long minValue; + final long maxValue; + + int docID = -1; + boolean skipperMatch; + int primaryFieldUpTo = -1; + int valueFieldUpTo = -1; + + SecondarySortIterator( + NumericDocValues values, + DocValuesSkipper valueSkipper, + DocValuesSkipper primaryFieldSkipper, + long minValue, + long maxValue + ) { + this.values = values; + this.valueSkipper = valueSkipper; + this.primaryFieldSkipper = primaryFieldSkipper; + this.minValue = minValue; + this.maxValue = maxValue; + + valueFieldUpTo = valueSkipper.maxDocID(0); + primaryFieldUpTo = primaryFieldSkipper.maxDocID(0); + } + + @Override + public int docID() { + return docID; + } + + @Override + public int nextDoc() throws IOException { + return advance(docID + 1); + } + + @Override + public int advance(int target) throws IOException { + skipperMatch = false; + target = values.advance(target); + if (target == DocIdSetIterator.NO_MORE_DOCS) { + return docID = target; + } + while (true) { + if (target > valueFieldUpTo) { + valueSkipper.advance(target); + valueFieldUpTo = valueSkipper.maxDocID(0); + long minValue = valueSkipper.minValue(0); + long maxValue = valueSkipper.maxValue(0); + if (minValue > this.maxValue || maxValue < this.minValue) { + // outside the desired range, skip forward + for (int level = 1; level < valueSkipper.numLevels(); level++) { + minValue = valueSkipper.minValue(level); + maxValue = valueSkipper.maxValue(level); + if (minValue > this.maxValue || maxValue < this.minValue) { + valueFieldUpTo = valueSkipper.maxDocID(level); + } else { + break; + } + } + + int upTo = valueFieldUpTo; + if (maxValue < this.minValue) { + // We've moved past the end of the valid values in the secondary sort field + // for this primary value. Advance the primary skipper to find the starting point + // for the next primary value, where the secondary field values will have reset + primaryFieldSkipper.advance(target); + primaryFieldUpTo = primaryFieldSkipper.maxDocID(0); + if (primaryFieldSkipper.minValue(0) == primaryFieldSkipper.maxValue(0)) { + for (int level = 1; level < primaryFieldSkipper.numLevels(); level++) { + if (primaryFieldSkipper.minValue(level) == primaryFieldSkipper.maxValue(level)) { + primaryFieldUpTo = primaryFieldSkipper.maxDocID(level); + } else { + break; + } + } + } + if (primaryFieldUpTo > upTo) { + upTo = primaryFieldUpTo; + } + } + + target = values.advance(upTo + 1); + if (target == DocIdSetIterator.NO_MORE_DOCS) { + return docID = target; + } + } else if (minValue >= this.minValue && maxValue <= this.maxValue) { + assert valueSkipper.docCount(0) == valueSkipper.maxDocID(0) - valueSkipper.minDocID(0) + 1; + skipperMatch = true; + return docID = target; + } + } + + long value = values.longValue(); + if (value < minValue && target > primaryFieldUpTo) { + primaryFieldSkipper.advance(target); + primaryFieldUpTo = primaryFieldSkipper.maxDocID(0); + if (primaryFieldSkipper.minValue(0) == primaryFieldSkipper.maxValue(0)) { + for (int level = 1; level < primaryFieldSkipper.numLevels(); level++) { + if (primaryFieldSkipper.minValue(level) == primaryFieldSkipper.maxValue(level)) { + primaryFieldUpTo = primaryFieldSkipper.maxDocID(level); + } else { + break; + } + } + target = values.advance(primaryFieldUpTo + 1); + if (target == DocIdSetIterator.NO_MORE_DOCS) { + return docID = target; + } + } else { + target = values.nextDoc(); + if (target == DocIdSetIterator.NO_MORE_DOCS) { + return docID = target; + } + } + } else if (value >= minValue && value <= maxValue) { + return docID = target; + } else { + target = values.nextDoc(); + if (target == DocIdSetIterator.NO_MORE_DOCS) { + return docID = target; + } + } + } + } + + @Override + public int docIDRunEnd() throws IOException { + if (skipperMatch) { + return valueFieldUpTo + 1; + } + return super.docIDRunEnd(); + } + + @Override + public long cost() { + return values.cost(); + } + +} diff --git a/server/src/main/java/org/elasticsearch/lucene/comparators/XLongComparator.java b/server/src/main/java/org/elasticsearch/lucene/comparators/XLongComparator.java new file mode 100644 index 0000000000000..24bdf921efb16 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/lucene/comparators/XLongComparator.java @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.lucene.comparators; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.comparators.LongComparator; +import org.apache.lucene.util.NumericUtils; + +import java.io.IOException; + +public class XLongComparator extends XNumericComparator { + + private final long[] values; + protected long topValue; + protected long bottom; + + public XLongComparator(int numHits, String field, Long missingValue, boolean reverse, Pruning pruning) { + super(field, missingValue != null ? missingValue : 0L, reverse, pruning, Long.BYTES); + values = new long[numHits]; + } + + @Override + public int compare(int slot1, int slot2) { + return Long.compare(values[slot1], values[slot2]); + } + + @Override + public void setTopValue(Long value) { + super.setTopValue(value); + topValue = value; + } + + @Override + public Long value(int slot) { + return Long.valueOf(values[slot]); + } + + @Override + protected long missingValueAsComparableLong() { + return missingValue; + } + + @Override + protected long sortableBytesToLong(byte[] bytes) { + return NumericUtils.sortableBytesToLong(bytes, 0); + } + + @Override + public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException { + return new LongLeafComparator(context); + } + + /** Leaf comparator for {@link LongComparator} that provides skipping functionality */ + public class LongLeafComparator extends NumericLeafComparator { + + public LongLeafComparator(LeafReaderContext context) throws IOException { + super(context); + } + + @Override + protected XNumericComparator.CompetitiveDISIBuilder buildCompetitiveDISIBuilder(LeafReaderContext context) + throws IOException { + return super.buildCompetitiveDISIBuilder(context); + } + + private long getValueForDoc(int doc) throws IOException { + if (docValues.advanceExact(doc)) { + return docValues.longValue(); + } else { + return missingValue; + } + } + + @Override + public void setBottom(int slot) throws IOException { + bottom = values[slot]; + super.setBottom(slot); + } + + @Override + public int compareBottom(int doc) throws IOException { + return Long.compare(bottom, getValueForDoc(doc)); + } + + @Override + public int compareTop(int doc) throws IOException { + return Long.compare(topValue, getValueForDoc(doc)); + } + + @Override + public void copy(int slot, int doc) throws IOException { + values[slot] = getValueForDoc(doc); + super.copy(slot, doc); + } + + @Override + protected long bottomAsComparableLong() { + return bottom; + } + + @Override + protected long topAsComparableLong() { + return topValue; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/lucene/comparators/XNumericComparator.java b/server/src/main/java/org/elasticsearch/lucene/comparators/XNumericComparator.java new file mode 100644 index 0000000000000..2ac33da3a1e8c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/lucene/comparators/XNumericComparator.java @@ -0,0 +1,538 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.lucene.comparators; + +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DocValuesRangeIterator; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.util.DocIdSetBuilder; +import org.apache.lucene.util.IntsRef; + +import java.io.IOException; + +/** + * Abstract numeric comparator for comparing numeric values. This comparator provides a skipping + * functionality – an iterator that can skip over non-competitive documents. + * + *

Parameter {@code field} provided in the constructor is used as a field name in the default + * implementations of the methods {@code getNumericDocValues} and {@code getPointValues} to retrieve + * doc values and points. You can pass a dummy value for a field name (e.g. when sorting by script), + * but in this case you must override both of these methods. + */ +public abstract class XNumericComparator extends FieldComparator { + + // MIN_SKIP_INTERVAL and MAX_SKIP_INTERVAL both should be powers of 2 + private static final int MIN_SKIP_INTERVAL = 32; + private static final int MAX_SKIP_INTERVAL = 8192; + protected final T missingValue; + private final long missingValueAsLong; + protected final String field; + protected final boolean reverse; + private final int bytesCount; // how many bytes are used to encode this number + + protected boolean topValueSet; + protected boolean singleSort; // singleSort is true, if sort is based on a single sort field. + protected boolean hitsThresholdReached; + protected boolean queueFull; + protected Pruning pruning; + + protected XNumericComparator(String field, T missingValue, boolean reverse, Pruning pruning, int bytesCount) { + this.field = field; + this.missingValue = missingValue; + this.missingValueAsLong = missingValueAsComparableLong(); + this.reverse = reverse; + this.pruning = pruning; + this.bytesCount = bytesCount; + } + + @Override + public void setTopValue(T value) { + topValueSet = true; + } + + @Override + public void setSingleSort() { + singleSort = true; + } + + @Override + public void disableSkipping() { + pruning = Pruning.NONE; + } + + protected abstract long missingValueAsComparableLong(); + + /** + * Decode sortable bytes to long. It should be consistent with the codec that {@link PointValues} + * of this field is using. + */ + protected abstract long sortableBytesToLong(byte[] bytes); + + /** Leaf comparator for {@link org.apache.lucene.search.comparators.NumericComparator} that provides skipping functionality */ + public abstract class NumericLeafComparator implements LeafFieldComparator { + private final LeafReaderContext context; + protected final NumericDocValues docValues; + private final CompetitiveDISIBuilder competitiveDISIBuilder; + + public NumericLeafComparator(LeafReaderContext context) throws IOException { + this.context = context; + this.docValues = getNumericDocValues(context, field); + competitiveDISIBuilder = buildCompetitiveDISIBuilder(context); + } + + protected CompetitiveDISIBuilder buildCompetitiveDISIBuilder(LeafReaderContext context) throws IOException { + if (pruning == Pruning.NONE) { + return null; + } + LeafReader reader = context.reader(); + PointValues pointValues = reader.getPointValues(field); + if (pointValues != null) { + return new PointsCompetitiveDISIBuilder(pointValues, this); + } else { + DocValuesSkipper skipper = reader.getDocValuesSkipper(field); + if (skipper != null) { + return new DVSkipperCompetitiveDISIBuilder(skipper, this); + } + } + return null; + } + + /** + * Retrieves the NumericDocValues for the field in this segment + * + *

If you override this method, you should probably always disable skipping as the comparator + * uses values from the points index to build its competitive iterators, and assumes that the + * values in doc values and points are the same. + * + * @param context – reader context + * @param field - field name + * @return numeric doc values for the field in this segment. + * @throws IOException If there is a low-level I/O error + */ + protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException { + return DocValues.getNumeric(context.reader(), field); + } + + @Override + public void setBottom(int slot) throws IOException { + queueFull = true; // if we are setting bottom, it means that we have collected enough hits + if (competitiveDISIBuilder != null) { + competitiveDISIBuilder.updateCompetitiveIterator(); + } + } + + @Override + public void copy(int slot, int doc) throws IOException { + if (competitiveDISIBuilder != null) { + competitiveDISIBuilder.setMaxDocVisited(doc); + } + } + + @Override + public void setScorer(Scorable scorer) throws IOException { + if (competitiveDISIBuilder != null) { + competitiveDISIBuilder.setScorer(scorer); + } + } + + @Override + public void setHitsThresholdReached() throws IOException { + hitsThresholdReached = true; + if (competitiveDISIBuilder != null) { + competitiveDISIBuilder.updateCompetitiveIterator(); + } + } + + @Override + public DocIdSetIterator competitiveIterator() { + return competitiveDISIBuilder == null ? null : competitiveDISIBuilder.competitiveIterator; + } + + protected abstract long bottomAsComparableLong(); + + protected abstract long topAsComparableLong(); + } + + protected abstract class CompetitiveDISIBuilder { + + final int maxDoc; + final NumericLeafComparator leafComparator; + + /** According to {@link FieldComparator#setTopValue}, topValueSet is final in leafComparator */ + final boolean leafTopSet = topValueSet; + + protected final XUpdateableDocIdSetIterator competitiveIterator = new XUpdateableDocIdSetIterator(); + protected long minValueAsLong = Long.MIN_VALUE; + protected long maxValueAsLong = Long.MAX_VALUE; + int maxDocVisited = -1; + int updateCounter = 0; + int currentSkipInterval = MIN_SKIP_INTERVAL; + + protected CompetitiveDISIBuilder(NumericLeafComparator leafComparator) { + this.leafComparator = leafComparator; + this.maxDoc = leafComparator.context.reader().maxDoc(); + this.competitiveIterator.update(DocIdSetIterator.all(maxDoc)); + if (leafTopSet) { + encodeTop(); + } + } + + void setScorer(Scorable scorer) throws IOException {} + + protected abstract int docCount(); + + final void updateCompetitiveIterator() throws IOException { + if (hitsThresholdReached == false) { + return; + } + if (leafTopSet == false && queueFull == false) { + return; + } + + // if some documents have missing points, check that missing values prohibits optimization + if (docCount() < maxDoc && isMissingValueCompetitive()) { + return; + } + + updateCounter++; + // Start sampling if we get called too much + if (updateCounter > 256 && (updateCounter & (currentSkipInterval - 1)) != currentSkipInterval - 1) { + return; + } + + if (queueFull) { + encodeBottom(); + } + + doUpdateCompetitiveIterator(); + } + + protected abstract void doUpdateCompetitiveIterator() throws IOException; + + private void setMaxDocVisited(int maxDocVisited) { + this.maxDocVisited = maxDocVisited; + } + + /** + * If {@link XNumericComparator#pruning} equals {@link Pruning#GREATER_THAN_OR_EQUAL_TO}, we + * could better tune the {@link #maxValueAsLong}/{@link #minValueAsLong}. For instance, if the + * sort is ascending and bottom value is 5, we will use a range on [MIN_VALUE, 4]. + */ + private void encodeBottom() { + if (reverse == false) { + maxValueAsLong = leafComparator.bottomAsComparableLong(); + if (pruning == Pruning.GREATER_THAN_OR_EQUAL_TO && maxValueAsLong != Long.MIN_VALUE) { + maxValueAsLong--; + } + } else { + minValueAsLong = leafComparator.bottomAsComparableLong(); + if (pruning == Pruning.GREATER_THAN_OR_EQUAL_TO && minValueAsLong != Long.MAX_VALUE) { + minValueAsLong++; + } + } + } + + /** + * If {@link XNumericComparator#pruning} equals {@link Pruning#GREATER_THAN_OR_EQUAL_TO}, we + * could better tune the {@link #minValueAsLong}/{@link #minValueAsLong}. For instance, if the + * sort is ascending and top value is 3, we will use a range on [4, MAX_VALUE]. + */ + private void encodeTop() { + if (reverse == false) { + minValueAsLong = leafComparator.topAsComparableLong(); + if (singleSort && pruning == Pruning.GREATER_THAN_OR_EQUAL_TO && queueFull && minValueAsLong != Long.MAX_VALUE) { + minValueAsLong++; + } + } else { + maxValueAsLong = leafComparator.topAsComparableLong(); + if (singleSort && pruning == Pruning.GREATER_THAN_OR_EQUAL_TO && queueFull && maxValueAsLong != Long.MIN_VALUE) { + maxValueAsLong--; + } + } + } + + boolean isMissingValueCompetitive() { + // if queue is full, compare with bottom first, + // if competitive, then check if we can compare with topValue + if (queueFull) { + int result = Long.compare(missingValueAsLong, leafComparator.bottomAsComparableLong()); + // in reverse (desc) sort missingValue is competitive when it's greater or equal to bottom, + // in asc sort missingValue is competitive when it's smaller or equal to bottom + final boolean competitive = reverse + ? (pruning == Pruning.GREATER_THAN_OR_EQUAL_TO ? result > 0 : result >= 0) + : (pruning == Pruning.GREATER_THAN_OR_EQUAL_TO ? result < 0 : result <= 0); + if (competitive == false) { + return false; + } + } + + if (leafTopSet) { + int result = Long.compare(missingValueAsLong, leafComparator.topAsComparableLong()); + // in reverse (desc) sort missingValue is competitive when it's smaller or equal to + // topValue, + // in asc sort missingValue is competitive when it's greater or equal to topValue + return reverse ? (result <= 0) : (result >= 0); + } + + // by default competitive + return true; + } + } + + private class PointsCompetitiveDISIBuilder extends CompetitiveDISIBuilder { + + private final PointValues pointValues; + // lazily constructed to avoid performance overhead when this is not used + private PointValues.PointTree pointTree; + private long iteratorCost = -1; + // helps to be conservative about increasing the sampling interval + private int tryUpdateFailCount = 0; + + PointsCompetitiveDISIBuilder(PointValues pointValues, NumericLeafComparator comparator) throws IOException { + super(comparator); + LeafReaderContext context = comparator.context; + FieldInfo info = context.reader().getFieldInfos().fieldInfo(field); + if (info == null || info.getPointDimensionCount() == 0) { + throw new IllegalStateException( + "Field " + field + " doesn't index points according to FieldInfos yet returns non-null PointValues" + ); + } else if (info.getPointDimensionCount() > 1) { + throw new IllegalArgumentException("Field " + field + " is indexed with multiple dimensions, sorting is not supported"); + } else if (info.getPointNumBytes() != bytesCount) { + throw new IllegalArgumentException( + "Field " + + field + + " is indexed with " + + info.getPointNumBytes() + + " bytes per dimension, but " + + this + + " expected " + + bytesCount + ); + } + this.pointValues = pointValues; + postInitializeCompetitiveIterator(); + } + + @Override + void setScorer(Scorable scorer) throws IOException { + if (iteratorCost == -1) { + if (scorer instanceof Scorer) { + iteratorCost = ((Scorer) scorer).iterator().cost(); // starting iterator cost is the scorer's cost + } else { + iteratorCost = maxDoc; + } + updateCompetitiveIterator(); // update an iterator when we have a new segment + } + } + + @Override + protected int docCount() { + return pointValues.getDocCount(); + } + + /** + * If queue is full and global min/max point values are not competitive with bottom then set an + * empty iterator as competitive iterator. + * + * @throws IOException i/o exception while fetching min and max values from point values + */ + void postInitializeCompetitiveIterator() throws IOException { + if (queueFull && hitsThresholdReached) { + // if some documents have missing doc values, check that missing values prohibits + // optimization + if (docCount() < maxDoc && isMissingValueCompetitive()) { + return; + } + long bottom = leafComparator.bottomAsComparableLong(); + long minValue = sortableBytesToLong(pointValues.getMinPackedValue()); + long maxValue = sortableBytesToLong(pointValues.getMaxPackedValue()); + if (reverse == false && bottom < minValue) { + competitiveIterator.update(DocIdSetIterator.empty()); + } else if (reverse && bottom > maxValue) { + competitiveIterator.update(DocIdSetIterator.empty()); + } + } + } + + @Override + protected void doUpdateCompetitiveIterator() throws IOException { + DocIdSetBuilder result = new DocIdSetBuilder(maxDoc); + PointValues.IntersectVisitor visitor = new PointValues.IntersectVisitor() { + DocIdSetBuilder.BulkAdder adder; + + @Override + public void grow(int count) { + adder = result.grow(count); + } + + @Override + public void visit(int docID) { + if (docID <= maxDocVisited) { + return; // Already visited or skipped + } + adder.add(docID); + } + + @Override + public void visit(int docID, byte[] packedValue) { + if (docID <= maxDocVisited) { + return; // already visited or skipped + } + long l = sortableBytesToLong(packedValue); + if (l >= minValueAsLong && l <= maxValueAsLong) { + adder.add(docID); // doc is competitive + } + } + + @Override + public void visit(DocIdSetIterator iterator) throws IOException { + if (iterator.advance(maxDocVisited + 1) != DocIdSetIterator.NO_MORE_DOCS) { + adder.add(iterator.docID()); + adder.add(iterator); + } + } + + @Override + public void visit(IntsRef ref) { + adder.add(ref, maxDocVisited + 1); + } + + @Override + public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + long min = sortableBytesToLong(minPackedValue); + long max = sortableBytesToLong(maxPackedValue); + + if (min > maxValueAsLong || max < minValueAsLong) { + // 1. cmp ==0 and pruning==Pruning.GREATER_THAN_OR_EQUAL_TO : if the sort is + // ascending then maxValueAsLong is bottom's next less value, so it is competitive + // 2. cmp ==0 and pruning==Pruning.GREATER_THAN: maxValueAsLong equals to + // bottom, but there are multiple comparators, so it could be competitive + return PointValues.Relation.CELL_OUTSIDE_QUERY; + } + + if (min < minValueAsLong || max > maxValueAsLong) { + return PointValues.Relation.CELL_CROSSES_QUERY; + } + return PointValues.Relation.CELL_INSIDE_QUERY; + } + }; + + final long threshold = iteratorCost >>> 3; + + if (PointValues.isEstimatedPointCountGreaterThanOrEqualTo(visitor, getPointTree(), threshold)) { + // the new range is not selective enough to be worth materializing, it doesn't reduce number + // of docs at least 8x + updateSkipInterval(false); + if (pointValues.getDocCount() < iteratorCost) { + // Use the set of doc with values to help drive iteration + competitiveIterator.update(leafComparator.getNumericDocValues(leafComparator.context, field)); + iteratorCost = pointValues.getDocCount(); + } + return; + } + pointValues.intersect(visitor); + competitiveIterator.update(result.build().iterator()); + iteratorCost = competitiveIterator.cost(); + updateSkipInterval(true); + } + + private PointValues.PointTree getPointTree() throws IOException { + if (pointTree == null) { + pointTree = pointValues.getPointTree(); + } + return pointTree; + } + + private void updateSkipInterval(boolean success) { + if (updateCounter > 256) { + if (success) { + currentSkipInterval = Math.max(currentSkipInterval / 2, MIN_SKIP_INTERVAL); + tryUpdateFailCount = 0; + } else { + if (tryUpdateFailCount >= 3) { + currentSkipInterval = Math.min(currentSkipInterval * 2, MAX_SKIP_INTERVAL); + tryUpdateFailCount = 0; + } else { + tryUpdateFailCount++; + } + } + } + } + } + + private class DVSkipperCompetitiveDISIBuilder extends CompetitiveDISIBuilder { + + private final DocValuesSkipper skipper; + private final TwoPhaseIterator innerTwoPhase; + + DVSkipperCompetitiveDISIBuilder(DocValuesSkipper skipper, NumericLeafComparator leafComparator) throws IOException { + super(leafComparator); + this.skipper = skipper; + NumericDocValues docValues = leafComparator.getNumericDocValues(leafComparator.context, field); + innerTwoPhase = new TwoPhaseIterator(docValues) { + @Override + public boolean matches() throws IOException { + final long value = docValues.longValue(); + return value >= minValueAsLong && value <= maxValueAsLong; + } + + @Override + public float matchCost() { + return 2; // 2 comparisons + } + }; + postInitializeCompetitiveIterator(); + } + + @Override + protected int docCount() { + return skipper.docCount(); + } + + /** + * If queue is full and global min/max skipper are not competitive with bottom then set an empty + * iterator as competitive iterator. + */ + void postInitializeCompetitiveIterator() { + if (queueFull && hitsThresholdReached) { + // if some documents have missing doc values, check that missing values prohibits + // optimization + if (docCount() < maxDoc && isMissingValueCompetitive()) { + return; + } + long bottom = leafComparator.bottomAsComparableLong(); + if (reverse == false && bottom < skipper.minValue()) { + competitiveIterator.update(DocIdSetIterator.empty()); + } else if (reverse && bottom > skipper.maxValue()) { + competitiveIterator.update(DocIdSetIterator.empty()); + } + } + } + + @Override + protected void doUpdateCompetitiveIterator() { + TwoPhaseIterator twoPhaseIterator = new DocValuesRangeIterator(innerTwoPhase, skipper, minValueAsLong, maxValueAsLong, false); + competitiveIterator.update(TwoPhaseIterator.asDocIdSetIterator(twoPhaseIterator)); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/lucene/comparators/XUpdateableDocIdSetIterator.java b/server/src/main/java/org/elasticsearch/lucene/comparators/XUpdateableDocIdSetIterator.java new file mode 100644 index 0000000000000..d7ce698d0c7af --- /dev/null +++ b/server/src/main/java/org/elasticsearch/lucene/comparators/XUpdateableDocIdSetIterator.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.lucene.comparators; + +import org.apache.lucene.search.AbstractDocIdSetIterator; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.FixedBitSet; + +import java.io.IOException; +import java.util.Objects; + +public final class XUpdateableDocIdSetIterator extends AbstractDocIdSetIterator { + + private DocIdSetIterator in = DocIdSetIterator.empty(); + + /** + * Update the wrapped {@link DocIdSetIterator}. It doesn't need to be positioned on the same doc + * ID as this iterator. + */ + public void update(DocIdSetIterator iterator) { + this.in = Objects.requireNonNull(iterator); + } + + @Override + public int nextDoc() throws IOException { + return advance(doc + 1); + } + + @Override + public int advance(int target) throws IOException { + int curDoc = in.docID(); + if (curDoc < target) { + curDoc = in.advance(target); + } + return this.doc = curDoc; + } + + @Override + public long cost() { + return in.cost(); + } + + @Override + public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException { + // #update may have been just called + if (in.docID() < doc) { + in.advance(doc); + } + in.intoBitSet(upTo, bitSet, offset); + doc = in.docID(); + } + + @Override + public int docIDRunEnd() throws IOException { + // #update may have been just called + if (in.docID() < doc) { + in.advance(doc); + } + if (in.docID() == doc) { + return in.docIDRunEnd(); + } else { + return super.docIDRunEnd(); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/SecondarySortIteratorTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/SecondarySortIteratorTests.java new file mode 100644 index 0000000000000..54c7892b7b2b3 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/fielddata/fieldcomparator/SecondarySortIteratorTests.java @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.fielddata.fieldcomparator; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DocValuesRangeIterator; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +public class SecondarySortIteratorTests extends ESTestCase { + + public void testAgainstDocValuesRangeIterator() throws IOException { + + Directory dir = newDirectory(); + Sort indexSort = new Sort(new SortField("hostname", SortField.Type.STRING), new SortField("@timestamp", SortField.Type.LONG, true)); + IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())).setIndexSort(indexSort); + RandomIndexWriter indexWriter = new RandomIndexWriter(random(), dir, iwc); + + int numDocs = atLeast(1000); + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + doc.add(SortedDocValuesField.indexedField("hostname", new BytesRef("host1"))); + doc.add(NumericDocValuesField.indexedField("@timestamp", 1_000_000 + i)); + indexWriter.addDocument(doc); + } + + indexWriter.forceMerge(1); + + IndexReader reader = indexWriter.getReader(); + long start = 1_000_400; + long end = 1_000_499; + DocIdSetIterator dvIt = docValuesRangeIterator(reader.leaves().getFirst(), start, end); + DocIdSetIterator ssIt = secondarySortIterator(reader.leaves().getFirst(), start, end); + + // Because the primary sort field has only a single value, we should get exactly the same + // results from the secondary sort iterator as from a standard DVRangeIterator over the + // secondary field + for (int doc = dvIt.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = dvIt.nextDoc()) { + assertEquals(doc, ssIt.nextDoc()); + } + assertEquals(DocIdSetIterator.NO_MORE_DOCS, ssIt.nextDoc()); + + reader.close(); + indexWriter.close(); + dir.close(); + } + + private static DocIdSetIterator secondarySortIterator(LeafReaderContext ctx, long start, long end) throws IOException { + NumericDocValues timeStampDV = ctx.reader().getNumericDocValues("@timestamp"); + DocValuesSkipper primarySkipper = ctx.reader().getDocValuesSkipper("hostname"); + DocValuesSkipper secondarySkipper = ctx.reader().getDocValuesSkipper("@timestamp"); + return new SecondarySortIterator(timeStampDV, secondarySkipper, primarySkipper, start, end); + } + + private static DocIdSetIterator docValuesRangeIterator(LeafReaderContext ctx, long start, long end) throws IOException { + NumericDocValues timeStampDV = ctx.reader().getNumericDocValues("@timestamp"); + TwoPhaseIterator twoPhaseIterator = new TwoPhaseIterator(timeStampDV) { + @Override + public boolean matches() throws IOException { + return timeStampDV.longValue() >= start && timeStampDV.longValue() <= end; + } + + @Override + public float matchCost() { + return 2; + } + }; + DocValuesSkipper skipper = ctx.reader().getDocValuesSkipper("@timestamp"); + return TwoPhaseIterator.asDocIdSetIterator(new DocValuesRangeIterator(twoPhaseIterator, skipper, start, end, false)); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index f3e2633717a4b..5a117765424ab 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -918,6 +918,15 @@ public void testSingletonLongBulkBlockReadingManyValues() throws Exception { @Override protected List getSortShortcutSupport() { return List.of( + new SortShortcutSupport( + IndexVersion.current(), + Settings.builder().put("index.mode", "logsdb").put("index.logsdb.sort_on_host_name", true).build(), + "@timestamp", + b -> b.field("type", "date").field("ignore_malformed", false), + b -> b.startObject("host.name").field("type", "keyword").endObject(), + b -> b.field("@timestamp", "2025-10-30T00:00:00").field("host.name", "foo"), + true + ), new SortShortcutSupport(b -> b.field("type", "date"), b -> b.field("field", "2025-10-30T00:00:00"), true), new SortShortcutSupport(b -> b.field("type", "date_nanos"), b -> b.field("field", "2025-10-30T00:00:00"), true), new SortShortcutSupport( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java index c56ad3a1e56eb..5a42bd66d0fa0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.script.DoubleFieldScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; @@ -164,4 +165,12 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean ignoreMalformed, Mapper.SourceKeepMode sourceKeepMode) { return new NumberSyntheticSourceSupportForKeepTests(Number::doubleValue, ignoreMalformed, sourceKeepMode); } + + @Override + protected List getSortShortcutSupport() { + return List.of( + new SortShortcutSupport(this::minimalMapping, this::writeField, true), + new SortShortcutSupport(IndexVersion.fromId(5000099), this::minimalMapping, this::writeField, false) + ); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java index 148c7b1b86e44..42183543ccf10 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.xcontent.XContentBuilder; import org.junit.AssumptionViolatedException; @@ -69,4 +70,12 @@ protected SyntheticSourceSupport syntheticSourceSupportForKeepTests(boolean igno protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of( + new SortShortcutSupport(this::minimalMapping, this::writeField, true), + new SortShortcutSupport(IndexVersion.fromId(5000099), this::minimalMapping, this::writeField, false) + ); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java index 8264f53661320..329379a9c56cb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java @@ -10,6 +10,7 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.sandbox.document.HalfFloatPoint; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.xcontent.XContentBuilder; import org.junit.AssumptionViolatedException; @@ -73,4 +74,13 @@ protected IngestScriptSupport ingestScriptSupport() { protected boolean supportsBulkDoubleBlockReading() { return true; } + + @Override + protected List getSortShortcutSupport() { + return List.of( + // TODO enable pruning here + new SortShortcutSupport(this::minimalMapping, this::writeField, false), + new SortShortcutSupport(IndexVersion.fromId(5000099), this::minimalMapping, this::writeField, false) + ); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 2b94f469e6c18..48019f2de4adc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -25,6 +25,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.Query; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; @@ -767,6 +768,10 @@ public void doTestIndexSortRangeQueries(NumberType type, Supplier valueS } } + var comparator = sortField.getComparator(10, Pruning.GREATER_THAN_OR_EQUAL_TO); + var leafComparator = comparator.getLeafComparator(reader.getContext().leaves().get(0)); + assertNotNull(leafComparator.competitiveIterator()); + reader.close(); w.close(); dir.close(); diff --git a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java index 521f6e9b5eb2f..f79d02b8c29af 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java @@ -175,6 +175,14 @@ public void testBuildSortFieldMissingValue() throws IOException { assertEquals(expectedSortField, sortField); } + // Specialised assertEquals to handle subclasses of SortedNumericSortField + public static void assertEquals(SortedNumericSortField expected, SortField actual) { + assertEquals(expected.getField(), actual.getField()); + assertEquals(expected.getMissingValue(), actual.getMissingValue()); + assertEquals(expected.getType(), actual.getType()); + assertEquals(expected.getReverse(), actual.getReverse()); + } + /** * Test that the sort builder order gets transferred correctly to the SortField */ diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 8ccab79c1c6ae..02afc6981a198 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -17,6 +17,7 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; import org.elasticsearch.TransportVersion; @@ -40,6 +41,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -395,9 +397,17 @@ protected static void withLuceneIndex( CheckedConsumer builder, CheckedConsumer test ) throws IOException { + IndexSortConfig sortConfig = new IndexSortConfig(mapperService.getIndexSettings()); + Sort indexSort = sortConfig.buildIndexSort( + mapperService::fieldType, + (ft, s) -> ft.fielddataBuilder(FieldDataContext.noRuntimeFields("")).build(null, null) + ); IndexWriterConfig iwc = new IndexWriterConfig(IndexShard.buildIndexAnalyzer(mapperService)).setCodec( new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE) ); + if (indexSort != null) { + iwc.setIndexSort(indexSort); + } try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc)) { builder.accept(iw); try (DirectoryReader reader = iw.getReader()) { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index ea5c4616f0989..7e32e5e2b1c00 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1712,7 +1712,9 @@ protected T compileOtherScript(Script script, ScriptContext context) { public record SortShortcutSupport( IndexVersion indexVersion, Settings settings, - CheckedConsumer mappings, + String fieldname, + CheckedConsumer fieldMapping, + CheckedConsumer additionalMappings, CheckedConsumer document, boolean supportsShortcut ) { @@ -1721,7 +1723,7 @@ public SortShortcutSupport( CheckedConsumer document, boolean supportsShortcut ) { - this(IndexVersion.current(), SETTINGS, mappings, document, supportsShortcut); + this(IndexVersion.current(), SETTINGS, "field", mappings, b -> {}, document, supportsShortcut); } public SortShortcutSupport( @@ -1730,7 +1732,7 @@ public SortShortcutSupport( CheckedConsumer document, boolean supportsShortcut ) { - this(indexVersion, SETTINGS, mappings, document, supportsShortcut); + this(indexVersion, SETTINGS, "field", mappings, b -> {}, document, supportsShortcut); } } @@ -1757,7 +1759,12 @@ public final void testSortShortcuts() throws IOException { sortShortcutSupport.settings, () -> true ); - merge(mapperService, fieldMapping(sortShortcutSupport.mappings)); + merge(mapperService, mapping(b -> { + b.startObject(sortShortcutSupport.fieldname); + sortShortcutSupport.fieldMapping.accept(b); + b.endObject(); + sortShortcutSupport.additionalMappings.accept(b); + })); withLuceneIndex(mapperService, iw -> { iw.addDocument( mapperService.documentParser() @@ -1766,7 +1773,7 @@ public final void testSortShortcuts() throws IOException { ); }, reader -> { IndexSearcher searcher = newSearcher(reader); - MappedFieldType ft = mapperService.fieldType("field"); + MappedFieldType ft = mapperService.fieldType(sortShortcutSupport.fieldname); SortField sortField = ft.fielddataBuilder(new FieldDataContext("", mapperService.getIndexSettings(), () -> { throw new UnsupportedOperationException(); }, Set::of, MappedFieldType.FielddataOperation.SEARCH)) diff --git a/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java b/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java index 4470e5928c0b6..7dd91f6594cce 100644 --- a/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java +++ b/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.datastreams.CreateDataStreamAction; import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Template; @@ -28,6 +29,7 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.license.LicenseSettings; @@ -160,7 +162,7 @@ public void testHostnameMessageTimestampSortConfig() throws IOException { assertOrder(backingIndex, orderedDocs); } - public void testHostnameTimestampSortConfig() throws IOException { + public void testHostnameTimestampSortConfig() throws Exception { final String dataStreamName = "test-logsdb-sort-hostname-timestamp"; final String MAPPING = """ @@ -210,6 +212,12 @@ public void testHostnameTimestampSortConfig() throws IOException { } assertOrder(backingIndex, orderedDocs); + + SearchRequest searchRequest = new SearchRequest(dataStreamName); + searchRequest.source().sort("@timestamp", SortOrder.DESC).query(new TermQueryBuilder("host.name", "aaa")); + var response = client().search(searchRequest).get(); + assertEquals(4, response.getHits().getHits().length); + response.decRef(); } public void testTimestampOnlySortConfig() throws IOException {