diff --git a/docs/changelog/135940.yaml b/docs/changelog/135940.yaml new file mode 100644 index 0000000000000..d754c98f320f5 --- /dev/null +++ b/docs/changelog/135940.yaml @@ -0,0 +1,5 @@ +pr: 135940 +summary: Enable directIO and bfloat16 for bbq and unquantized vector field types +area: Vector Search +type: feature +issues: [] diff --git a/docs/changelog/136632.yaml b/docs/changelog/136632.yaml new file mode 100644 index 0000000000000..1a63e135fefcc --- /dev/null +++ b/docs/changelog/136632.yaml @@ -0,0 +1,6 @@ +pr: 136632 +summary: Field caps transport changes to return for each original expression what + it was resolved to +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/137023.yaml b/docs/changelog/137023.yaml new file mode 100644 index 0000000000000..cab7620233562 --- /dev/null +++ b/docs/changelog/137023.yaml @@ -0,0 +1,5 @@ +pr: 137023 +summary: Support choosing the downsampling method in data stream lifecycle +area: "Data streams" +type: enhancement +issues: [] diff --git a/docs/changelog/137220.yaml b/docs/changelog/137220.yaml new file mode 100644 index 0000000000000..895a634d91501 --- /dev/null +++ b/docs/changelog/137220.yaml @@ -0,0 +1,5 @@ +pr: 137220 +summary: Skip dataframes when disabled +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/137442.yaml b/docs/changelog/137442.yaml new file mode 100644 index 0000000000000..b999927dff74a --- /dev/null +++ b/docs/changelog/137442.yaml @@ -0,0 +1,5 @@ +pr: 137442 +summary: Handle ._original stored fields with fls +area: "Authorization" +type: bug +issues: [] diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MSBitToInt4ESNextOSQVectorsScorer.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MSBitToInt4ESNextOSQVectorsScorer.java index 14b0cc3499ae5..888388ce2caa6 100644 --- a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MSBitToInt4ESNextOSQVectorsScorer.java +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MSBitToInt4ESNextOSQVectorsScorer.java @@ -14,7 +14,6 @@ import jdk.incubator.vector.LongVector; import jdk.incubator.vector.ShortVector; import jdk.incubator.vector.VectorOperators; -import jdk.incubator.vector.VectorSpecies; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.IndexInput; @@ -31,23 +30,6 @@ /** Panamized scorer for quantized vectors stored as a {@link MemorySegment}. */ final class MSBitToInt4ESNextOSQVectorsScorer extends MemorySegmentESNextOSQVectorsScorer.MemorySegmentScorer { - private static final int BULK_SIZE = MemorySegmentESNextOSQVectorsScorer.BULK_SIZE; - private static final float FOUR_BIT_SCALE = 1f / ((1 << 4) - 1); - - private static final VectorSpecies INT_SPECIES_128 = IntVector.SPECIES_128; - - private static final VectorSpecies LONG_SPECIES_128 = LongVector.SPECIES_128; - private static final VectorSpecies LONG_SPECIES_256 = LongVector.SPECIES_256; - - private static final VectorSpecies BYTE_SPECIES_128 = ByteVector.SPECIES_128; - private static final VectorSpecies BYTE_SPECIES_256 = ByteVector.SPECIES_256; - - private static final VectorSpecies SHORT_SPECIES_128 = ShortVector.SPECIES_128; - private static final VectorSpecies SHORT_SPECIES_256 = ShortVector.SPECIES_256; - - private static final VectorSpecies FLOAT_SPECIES_128 = FloatVector.SPECIES_128; - private static final VectorSpecies FLOAT_SPECIES_256 = FloatVector.SPECIES_256; - MSBitToInt4ESNextOSQVectorsScorer(IndexInput in, int dimensions, int dataLength, MemorySegment memorySegment) { super(in, dimensions, dataLength, memorySegment); } diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MSInt4SymmetricESNextOSQVectorsScorer.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MSInt4SymmetricESNextOSQVectorsScorer.java new file mode 100644 index 0000000000000..9263c4ce56282 --- /dev/null +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MSInt4SymmetricESNextOSQVectorsScorer.java @@ -0,0 +1,419 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.simdvec.internal.vectorization; + +import jdk.incubator.vector.ByteVector; +import jdk.incubator.vector.FloatVector; +import jdk.incubator.vector.IntVector; +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.ShortVector; +import jdk.incubator.vector.VectorOperators; + +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.BitUtil; +import org.apache.lucene.util.VectorUtil; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.nio.ByteOrder; + +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; +import static org.apache.lucene.index.VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT; + +/** Panamized scorer for quantized vectors stored as a {@link MemorySegment}. */ +final class MSInt4SymmetricESNextOSQVectorsScorer extends MemorySegmentESNextOSQVectorsScorer.MemorySegmentScorer { + + MSInt4SymmetricESNextOSQVectorsScorer(IndexInput in, int dimensions, int dataLength, MemorySegment memorySegment) { + super(in, dimensions, dataLength, memorySegment); + } + + @Override + public long quantizeScore(byte[] q) throws IOException { + assert q.length == length; + // 128 / 8 == 16 + if (length >= 16 && PanamaESVectorUtilSupport.HAS_FAST_INTEGER_VECTORS) { + if (PanamaESVectorUtilSupport.VECTOR_BITSIZE >= 256) { + return quantizeScoreSymmetric256(q); + } else if (PanamaESVectorUtilSupport.VECTOR_BITSIZE == 128) { + return quantizeScoreSymmetric128(q); + } + } + return Long.MIN_VALUE; + } + + private long quantizeScoreSymmetric128(byte[] q) throws IOException { + int stripe0 = (int) quantizeScore128(q); + int stripe1 = (int) quantizeScore128(q); + int stripe2 = (int) quantizeScore128(q); + int stripe3 = (int) quantizeScore128(q); + return stripe0 + ((long) stripe1 << 1) + ((long) stripe2 << 2) + ((long) stripe3 << 3); + } + + private long quantizeScoreSymmetric256(byte[] q) throws IOException { + int stripe0 = (int) quantizeScore256(q); + int stripe1 = (int) quantizeScore256(q); + int stripe2 = (int) quantizeScore256(q); + int stripe3 = (int) quantizeScore256(q); + return stripe0 + ((long) stripe1 << 1) + ((long) stripe2 << 2) + ((long) stripe3 << 3); + } + + private long quantizeScore256(byte[] q) throws IOException { + long subRet0 = 0; + long subRet1 = 0; + long subRet2 = 0; + long subRet3 = 0; + int i = 0; + long offset = in.getFilePointer(); + int size = length / 4; + if (size >= ByteVector.SPECIES_256.vectorByteSize() * 2) { + int limit = ByteVector.SPECIES_256.loopBound(size); + var sum0 = LongVector.zero(LONG_SPECIES_256); + var sum1 = LongVector.zero(LONG_SPECIES_256); + var sum2 = LongVector.zero(LONG_SPECIES_256); + var sum3 = LongVector.zero(LONG_SPECIES_256); + for (; i < limit; i += ByteVector.SPECIES_256.length(), offset += LONG_SPECIES_256.vectorByteSize()) { + var vq0 = ByteVector.fromArray(BYTE_SPECIES_256, q, i).reinterpretAsLongs(); + var vq1 = ByteVector.fromArray(BYTE_SPECIES_256, q, i + size).reinterpretAsLongs(); + var vq2 = ByteVector.fromArray(BYTE_SPECIES_256, q, i + size * 2).reinterpretAsLongs(); + var vq3 = ByteVector.fromArray(BYTE_SPECIES_256, q, i + size * 3).reinterpretAsLongs(); + var vd = LongVector.fromMemorySegment(LONG_SPECIES_256, memorySegment, offset, ByteOrder.LITTLE_ENDIAN); + sum0 = sum0.add(vq0.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum1 = sum1.add(vq1.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum2 = sum2.add(vq2.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum3 = sum3.add(vq3.and(vd).lanewise(VectorOperators.BIT_COUNT)); + } + subRet0 += sum0.reduceLanes(VectorOperators.ADD); + subRet1 += sum1.reduceLanes(VectorOperators.ADD); + subRet2 += sum2.reduceLanes(VectorOperators.ADD); + subRet3 += sum3.reduceLanes(VectorOperators.ADD); + } + + if (size - i >= ByteVector.SPECIES_128.vectorByteSize()) { + var sum0 = LongVector.zero(LONG_SPECIES_128); + var sum1 = LongVector.zero(LONG_SPECIES_128); + var sum2 = LongVector.zero(LONG_SPECIES_128); + var sum3 = LongVector.zero(LONG_SPECIES_128); + int limit = ByteVector.SPECIES_128.loopBound(size); + for (; i < limit; i += ByteVector.SPECIES_128.length(), offset += LONG_SPECIES_128.vectorByteSize()) { + var vq0 = ByteVector.fromArray(BYTE_SPECIES_128, q, i).reinterpretAsLongs(); + var vq1 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + size).reinterpretAsLongs(); + var vq2 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + size * 2).reinterpretAsLongs(); + var vq3 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + size * 3).reinterpretAsLongs(); + var vd = LongVector.fromMemorySegment(LONG_SPECIES_128, memorySegment, offset, ByteOrder.LITTLE_ENDIAN); + sum0 = sum0.add(vq0.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum1 = sum1.add(vq1.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum2 = sum2.add(vq2.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum3 = sum3.add(vq3.and(vd).lanewise(VectorOperators.BIT_COUNT)); + } + subRet0 += sum0.reduceLanes(VectorOperators.ADD); + subRet1 += sum1.reduceLanes(VectorOperators.ADD); + subRet2 += sum2.reduceLanes(VectorOperators.ADD); + subRet3 += sum3.reduceLanes(VectorOperators.ADD); + } + // process scalar tail + in.seek(offset); + for (final int upperBound = size & -Long.BYTES; i < upperBound; i += Long.BYTES) { + final long value = in.readLong(); + subRet0 += Long.bitCount((long) BitUtil.VH_LE_LONG.get(q, i) & value); + subRet1 += Long.bitCount((long) BitUtil.VH_LE_LONG.get(q, i + size) & value); + subRet2 += Long.bitCount((long) BitUtil.VH_LE_LONG.get(q, i + 2 * size) & value); + subRet3 += Long.bitCount((long) BitUtil.VH_LE_LONG.get(q, i + 3 * size) & value); + } + for (final int upperBound = size & -Integer.BYTES; i < upperBound; i += Integer.BYTES) { + final int value = in.readInt(); + subRet0 += Integer.bitCount((int) BitUtil.VH_LE_INT.get(q, i) & value); + subRet1 += Integer.bitCount((int) BitUtil.VH_LE_INT.get(q, i + size) & value); + subRet2 += Integer.bitCount((int) BitUtil.VH_LE_INT.get(q, i + 2 * size) & value); + subRet3 += Integer.bitCount((int) BitUtil.VH_LE_INT.get(q, i + 3 * size) & value); + } + for (; i < size; i++) { + int dValue = in.readByte() & 0xFF; + subRet0 += Integer.bitCount((q[i] & dValue) & 0xFF); + subRet1 += Integer.bitCount((q[i + size] & dValue) & 0xFF); + subRet2 += Integer.bitCount((q[i + 2 * size] & dValue) & 0xFF); + subRet3 += Integer.bitCount((q[i + 3 * size] & dValue) & 0xFF); + } + return subRet0 + (subRet1 << 1) + (subRet2 << 2) + (subRet3 << 3); + } + + private long quantizeScore128(byte[] q) throws IOException { + long subRet0 = 0; + long subRet1 = 0; + long subRet2 = 0; + long subRet3 = 0; + int i = 0; + long offset = in.getFilePointer(); + + var sum0 = IntVector.zero(INT_SPECIES_128); + var sum1 = IntVector.zero(INT_SPECIES_128); + var sum2 = IntVector.zero(INT_SPECIES_128); + var sum3 = IntVector.zero(INT_SPECIES_128); + int size = length / 4; + int limit = ByteVector.SPECIES_128.loopBound(size); + for (; i < limit; i += ByteVector.SPECIES_128.length(), offset += INT_SPECIES_128.vectorByteSize()) { + var vd = IntVector.fromMemorySegment(INT_SPECIES_128, memorySegment, offset, ByteOrder.LITTLE_ENDIAN); + var vq0 = ByteVector.fromArray(BYTE_SPECIES_128, q, i).reinterpretAsInts(); + var vq1 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + size).reinterpretAsInts(); + var vq2 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + size * 2).reinterpretAsInts(); + var vq3 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + size * 3).reinterpretAsInts(); + sum0 = sum0.add(vd.and(vq0).lanewise(VectorOperators.BIT_COUNT)); + sum1 = sum1.add(vd.and(vq1).lanewise(VectorOperators.BIT_COUNT)); + sum2 = sum2.add(vd.and(vq2).lanewise(VectorOperators.BIT_COUNT)); + sum3 = sum3.add(vd.and(vq3).lanewise(VectorOperators.BIT_COUNT)); + } + subRet0 += sum0.reduceLanes(VectorOperators.ADD); + subRet1 += sum1.reduceLanes(VectorOperators.ADD); + subRet2 += sum2.reduceLanes(VectorOperators.ADD); + subRet3 += sum3.reduceLanes(VectorOperators.ADD); + // process scalar tail + in.seek(offset); + for (final int upperBound = size & -Long.BYTES; i < upperBound; i += Long.BYTES) { + final long value = in.readLong(); + subRet0 += Long.bitCount((long) BitUtil.VH_LE_LONG.get(q, i) & value); + subRet1 += Long.bitCount((long) BitUtil.VH_LE_LONG.get(q, i + size) & value); + subRet2 += Long.bitCount((long) BitUtil.VH_LE_LONG.get(q, i + 2 * size) & value); + subRet3 += Long.bitCount((long) BitUtil.VH_LE_LONG.get(q, i + 3 * size) & value); + } + for (final int upperBound = size & -Integer.BYTES; i < upperBound; i += Integer.BYTES) { + final int value = in.readInt(); + subRet0 += Integer.bitCount((int) BitUtil.VH_LE_INT.get(q, i) & value); + subRet1 += Integer.bitCount((int) BitUtil.VH_LE_INT.get(q, i + size) & value); + subRet2 += Integer.bitCount((int) BitUtil.VH_LE_INT.get(q, i + 2 * size) & value); + subRet3 += Integer.bitCount((int) BitUtil.VH_LE_INT.get(q, i + 3 * size) & value); + } + for (; i < size; i++) { + int dValue = in.readByte() & 0xFF; + subRet0 += Integer.bitCount((q[i] & dValue) & 0xFF); + subRet1 += Integer.bitCount((q[i + size] & dValue) & 0xFF); + subRet2 += Integer.bitCount((q[i + 2 * size] & dValue) & 0xFF); + subRet3 += Integer.bitCount((q[i + 3 * size] & dValue) & 0xFF); + } + return subRet0 + (subRet1 << 1) + (subRet2 << 2) + (subRet3 << 3); + } + + @Override + public boolean quantizeScoreBulk(byte[] q, int count, float[] scores) throws IOException { + assert q.length == length; + // 128 / 8 == 16 + if (length >= 16 && PanamaESVectorUtilSupport.HAS_FAST_INTEGER_VECTORS) { + if (PanamaESVectorUtilSupport.VECTOR_BITSIZE >= 256) { + quantizeScore256Bulk(q, count, scores); + return true; + } else if (PanamaESVectorUtilSupport.VECTOR_BITSIZE == 128) { + quantizeScore128Bulk(q, count, scores); + return true; + } + } + return false; + } + + private void quantizeScore128Bulk(byte[] q, int count, float[] scores) throws IOException { + for (int iter = 0; iter < count; iter++) { + scores[iter] = quantizeScoreSymmetric128(q); + } + } + + private void quantizeScore256Bulk(byte[] q, int count, float[] scores) throws IOException { + for (int iter = 0; iter < count; iter++) { + scores[iter] = quantizeScoreSymmetric256(q); + } + } + + @Override + public float scoreBulk( + byte[] q, + float queryLowerInterval, + float queryUpperInterval, + int queryComponentSum, + float queryAdditionalCorrection, + VectorSimilarityFunction similarityFunction, + float centroidDp, + float[] scores + ) throws IOException { + assert q.length == length; + // 128 / 8 == 16 + if (length >= 16 && PanamaESVectorUtilSupport.HAS_FAST_INTEGER_VECTORS) { + if (PanamaESVectorUtilSupport.VECTOR_BITSIZE >= 256) { + return score256Bulk( + q, + queryLowerInterval, + queryUpperInterval, + queryComponentSum, + queryAdditionalCorrection, + similarityFunction, + centroidDp, + scores + ); + } else if (PanamaESVectorUtilSupport.VECTOR_BITSIZE == 128) { + return score128Bulk( + q, + queryLowerInterval, + queryUpperInterval, + queryComponentSum, + queryAdditionalCorrection, + similarityFunction, + centroidDp, + scores + ); + } + } + return Float.NEGATIVE_INFINITY; + } + + private float score128Bulk( + byte[] q, + float queryLowerInterval, + float queryUpperInterval, + int queryComponentSum, + float queryAdditionalCorrection, + VectorSimilarityFunction similarityFunction, + float centroidDp, + float[] scores + ) throws IOException { + quantizeScore128Bulk(q, BULK_SIZE, scores); + int limit = FLOAT_SPECIES_128.loopBound(BULK_SIZE); + int i = 0; + long offset = in.getFilePointer(); + float ay = queryLowerInterval; + float ly = (queryUpperInterval - ay) * FOUR_BIT_SCALE; + float y1 = queryComponentSum; + float maxScore = Float.NEGATIVE_INFINITY; + for (; i < limit; i += FLOAT_SPECIES_128.length()) { + var ax = FloatVector.fromMemorySegment(FLOAT_SPECIES_128, memorySegment, offset + i * Float.BYTES, ByteOrder.LITTLE_ENDIAN); + var lx = FloatVector.fromMemorySegment( + FLOAT_SPECIES_128, + memorySegment, + offset + 4 * BULK_SIZE + i * Float.BYTES, + ByteOrder.LITTLE_ENDIAN + ).sub(ax).mul(FOUR_BIT_SCALE); + var targetComponentSums = ShortVector.fromMemorySegment( + SHORT_SPECIES_128, + memorySegment, + offset + 8 * BULK_SIZE + i * Short.BYTES, + ByteOrder.LITTLE_ENDIAN + ).convert(VectorOperators.S2I, 0).reinterpretAsInts().and(0xffff).convert(VectorOperators.I2F, 0); + var additionalCorrections = FloatVector.fromMemorySegment( + FLOAT_SPECIES_128, + memorySegment, + offset + 10 * BULK_SIZE + i * Float.BYTES, + ByteOrder.LITTLE_ENDIAN + ); + var qcDist = FloatVector.fromArray(FLOAT_SPECIES_128, scores, i); + // ax * ay * dimensions + ay * lx * (float) targetComponentSum + ax * ly * y1 + lx * ly * + // qcDist; + var res1 = ax.mul(ay).mul(dimensions); + var res2 = lx.mul(ay).mul(targetComponentSums); + var res3 = ax.mul(ly).mul(y1); + var res4 = lx.mul(ly).mul(qcDist); + var res = res1.add(res2).add(res3).add(res4); + // For euclidean, we need to invert the score and apply the additional correction, which is + // assumed to be the squared l2norm of the centroid centered vectors. + if (similarityFunction == EUCLIDEAN) { + res = res.mul(-2).add(additionalCorrections).add(queryAdditionalCorrection).add(1f); + res = FloatVector.broadcast(FLOAT_SPECIES_128, 1).div(res).max(0); + maxScore = Math.max(maxScore, res.reduceLanes(VectorOperators.MAX)); + res.intoArray(scores, i); + } else { + // For cosine and max inner product, we need to apply the additional correction, which is + // assumed to be the non-centered dot-product between the vector and the centroid + res = res.add(queryAdditionalCorrection).add(additionalCorrections).sub(centroidDp); + if (similarityFunction == MAXIMUM_INNER_PRODUCT) { + res.intoArray(scores, i); + // not sure how to do it better + for (int j = 0; j < FLOAT_SPECIES_128.length(); j++) { + scores[i + j] = VectorUtil.scaleMaxInnerProductScore(scores[i + j]); + maxScore = Math.max(maxScore, scores[i + j]); + } + } else { + res = res.add(1f).mul(0.5f).max(0); + res.intoArray(scores, i); + maxScore = Math.max(maxScore, res.reduceLanes(VectorOperators.MAX)); + } + } + } + in.seek(offset + 14L * BULK_SIZE); + return maxScore; + } + + private float score256Bulk( + byte[] q, + float queryLowerInterval, + float queryUpperInterval, + int queryComponentSum, + float queryAdditionalCorrection, + VectorSimilarityFunction similarityFunction, + float centroidDp, + float[] scores + ) throws IOException { + quantizeScore256Bulk(q, BULK_SIZE, scores); + int limit = FLOAT_SPECIES_256.loopBound(BULK_SIZE); + int i = 0; + long offset = in.getFilePointer(); + float ay = queryLowerInterval; + float ly = (queryUpperInterval - ay) * FOUR_BIT_SCALE; + float y1 = queryComponentSum; + float maxScore = Float.NEGATIVE_INFINITY; + for (; i < limit; i += FLOAT_SPECIES_256.length()) { + var ax = FloatVector.fromMemorySegment(FLOAT_SPECIES_256, memorySegment, offset + i * Float.BYTES, ByteOrder.LITTLE_ENDIAN); + var lx = FloatVector.fromMemorySegment( + FLOAT_SPECIES_256, + memorySegment, + offset + 4 * BULK_SIZE + i * Float.BYTES, + ByteOrder.LITTLE_ENDIAN + ).sub(ax).mul(FOUR_BIT_SCALE); + var targetComponentSums = ShortVector.fromMemorySegment( + SHORT_SPECIES_256, + memorySegment, + offset + 8 * BULK_SIZE + i * Short.BYTES, + ByteOrder.LITTLE_ENDIAN + ).convert(VectorOperators.S2I, 0).reinterpretAsInts().and(0xffff).convert(VectorOperators.I2F, 0); + var additionalCorrections = FloatVector.fromMemorySegment( + FLOAT_SPECIES_256, + memorySegment, + offset + 10 * BULK_SIZE + i * Float.BYTES, + ByteOrder.LITTLE_ENDIAN + ); + var qcDist = FloatVector.fromArray(FLOAT_SPECIES_256, scores, i); + // ax * ay * dimensions + ay * lx * (float) targetComponentSum + ax * ly * y1 + lx * ly * + // qcDist; + var res1 = ax.mul(ay).mul(dimensions); + var res2 = lx.mul(ay).mul(targetComponentSums); + var res3 = ax.mul(ly).mul(y1); + var res4 = lx.mul(ly).mul(qcDist); + var res = res1.add(res2).add(res3).add(res4); + // For euclidean, we need to invert the score and apply the additional correction, which is + // assumed to be the squared l2norm of the centroid centered vectors. + if (similarityFunction == EUCLIDEAN) { + res = res.mul(-2).add(additionalCorrections).add(queryAdditionalCorrection).add(1f); + res = FloatVector.broadcast(FLOAT_SPECIES_256, 1).div(res).max(0); + maxScore = Math.max(maxScore, res.reduceLanes(VectorOperators.MAX)); + res.intoArray(scores, i); + } else { + // For cosine and max inner product, we need to apply the additional correction, which is + // assumed to be the non-centered dot-product between the vector and the centroid + res = res.add(queryAdditionalCorrection).add(additionalCorrections).sub(centroidDp); + if (similarityFunction == MAXIMUM_INNER_PRODUCT) { + res.intoArray(scores, i); + // not sure how to do it better + for (int j = 0; j < FLOAT_SPECIES_256.length(); j++) { + scores[i + j] = VectorUtil.scaleMaxInnerProductScore(scores[i + j]); + maxScore = Math.max(maxScore, scores[i + j]); + } + } else { + res = res.add(1f).mul(0.5f).max(0); + maxScore = Math.max(maxScore, res.reduceLanes(VectorOperators.MAX)); + res.intoArray(scores, i); + } + } + } + in.seek(offset + 14L * BULK_SIZE); + return maxScore; + } +} diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MemorySegmentESNextOSQVectorsScorer.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MemorySegmentESNextOSQVectorsScorer.java index 52563fa959f27..492e19dd649ad 100644 --- a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MemorySegmentESNextOSQVectorsScorer.java +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MemorySegmentESNextOSQVectorsScorer.java @@ -25,20 +25,6 @@ /** Panamized scorer for quantized vectors stored as a {@link MemorySegment}. */ public final class MemorySegmentESNextOSQVectorsScorer extends ESNextOSQVectorsScorer { - private static final VectorSpecies INT_SPECIES_128 = IntVector.SPECIES_128; - - private static final VectorSpecies LONG_SPECIES_128 = LongVector.SPECIES_128; - private static final VectorSpecies LONG_SPECIES_256 = LongVector.SPECIES_256; - - private static final VectorSpecies BYTE_SPECIES_128 = ByteVector.SPECIES_128; - private static final VectorSpecies BYTE_SPECIES_256 = ByteVector.SPECIES_256; - - private static final VectorSpecies SHORT_SPECIES_128 = ShortVector.SPECIES_128; - private static final VectorSpecies SHORT_SPECIES_256 = ShortVector.SPECIES_256; - - private static final VectorSpecies FLOAT_SPECIES_128 = FloatVector.SPECIES_128; - private static final VectorSpecies FLOAT_SPECIES_256 = FloatVector.SPECIES_256; - private final MemorySegment memorySegment; private final MemorySegmentScorer scorer; @@ -54,6 +40,10 @@ public MemorySegmentESNextOSQVectorsScorer( this.memorySegment = memorySegment; if (queryBits == 4 && indexBits == 1) { this.scorer = new MSBitToInt4ESNextOSQVectorsScorer(in, dimensions, dataLength, memorySegment); + } else if (queryBits == 4 && indexBits == 4) { + this.scorer = new MSInt4SymmetricESNextOSQVectorsScorer(in, dimensions, dataLength, memorySegment); + } else if (queryBits == 4 && indexBits == 2) { + throw new IllegalArgumentException("Only symmetric 4-bit query and 1-bit index supported"); } else { throw new IllegalArgumentException("Only asymmetric 4-bit query and 1-bit index supported"); } @@ -112,7 +102,24 @@ public float scoreBulk( ); } - abstract static sealed class MemorySegmentScorer permits MSBitToInt4ESNextOSQVectorsScorer { + abstract static sealed class MemorySegmentScorer permits MSBitToInt4ESNextOSQVectorsScorer, MSInt4SymmetricESNextOSQVectorsScorer { + + static final int BULK_SIZE = MemorySegmentESNextOSQVectorsScorer.BULK_SIZE; + static final float FOUR_BIT_SCALE = 1f / ((1 << 4) - 1); + static final VectorSpecies INT_SPECIES_128 = IntVector.SPECIES_128; + + static final VectorSpecies LONG_SPECIES_128 = LongVector.SPECIES_128; + static final VectorSpecies LONG_SPECIES_256 = LongVector.SPECIES_256; + + static final VectorSpecies BYTE_SPECIES_128 = ByteVector.SPECIES_128; + static final VectorSpecies BYTE_SPECIES_256 = ByteVector.SPECIES_256; + + static final VectorSpecies SHORT_SPECIES_128 = ShortVector.SPECIES_128; + static final VectorSpecies SHORT_SPECIES_256 = ShortVector.SPECIES_256; + + static final VectorSpecies FLOAT_SPECIES_128 = FloatVector.SPECIES_128; + static final VectorSpecies FLOAT_SPECIES_256 = FloatVector.SPECIES_256; + protected final MemorySegment memorySegment; protected final IndexInput in; protected final int length; diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java index 17838c5bef05b..44e6937b5b23e 100644 --- a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java @@ -40,7 +40,7 @@ public ESNextOSQVectorsScorer newESNextOSQVectorsScorer(IndexInput input, byte q if (PanamaESVectorUtilSupport.HAS_FAST_INTEGER_VECTORS && input instanceof MemorySegmentAccessInput msai && queryBits == 4 - && indexBits == 1) { + && (indexBits == 1 || indexBits == 4)) { MemorySegment ms = msai.segmentSliceOrNull(0, input.length()); if (ms != null) { return new MemorySegmentESNextOSQVectorsScorer(input, queryBits, indexBits, dimension, dataLength, ms); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java index 27cfc62822441..fe99d78fa6823 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.delete.DeleteRequest; @@ -44,6 +45,7 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.resizeRequest; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; @@ -267,12 +269,10 @@ public void testIndexingGettingAndSearchingShrunkIndex() throws Exception { assertThat(updateSettingsResponse.isAcknowledged(), is(true)); String shrunkenTarget = "k8s-shrunken"; - var shrinkIndexResponse = client().admin() - .indices() - .prepareResizeIndex(sourceIndex, shrunkenTarget) - .setResizeType(ResizeType.SHRINK) - .setSettings(indexSettings(2, 0).build()) - .get(); + final var shrinkIndexResponse = client().execute( + TransportResizeAction.TYPE, + resizeRequest(ResizeType.SHRINK, sourceIndex, shrunkenTarget, indexSettings(2, 0)) + ).actionGet(); assertThat(shrinkIndexResponse.isAcknowledged(), is(true)); assertThat(shrinkIndexResponse.index(), equalTo(shrunkenTarget)); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java index 776ee03928b27..5a998f43d0103 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java @@ -524,7 +524,10 @@ Set maybeExecuteDownsampling(ProjectState projectState, DataStream dataSt // - has matching downsample rounds // - is read-only // So let's wait for an in-progress downsampling operation to succeed or trigger the last matching round - affectedIndices.addAll(waitForInProgressOrTriggerDownsampling(dataStream, backingIndexMeta, downsamplingRounds, project)); + var downsamplingMethod = dataStream.getDataLifecycle().downsamplingMethod(); + affectedIndices.addAll( + waitForInProgressOrTriggerDownsampling(dataStream, backingIndexMeta, downsamplingRounds, downsamplingMethod, project) + ); } } @@ -541,6 +544,7 @@ private Set waitForInProgressOrTriggerDownsampling( DataStream dataStream, IndexMetadata backingIndex, List downsamplingRounds, + DownsampleConfig.SamplingMethod downsamplingMethod, ProjectMetadata project ) { assert dataStream.getIndices().contains(backingIndex.getIndex()) @@ -556,7 +560,7 @@ private Set waitForInProgressOrTriggerDownsampling( String downsampleIndexName = DownsampleConfig.generateDownsampleIndexName( DOWNSAMPLED_INDEX_PREFIX, backingIndex, - round.config().getFixedInterval() + round.fixedInterval() ); IndexMetadata targetDownsampleIndexMeta = project.index(downsampleIndexName); boolean targetDownsampleIndexExists = targetDownsampleIndexMeta != null; @@ -568,7 +572,8 @@ private Set waitForInProgressOrTriggerDownsampling( INDEX_DOWNSAMPLE_STATUS.get(targetDownsampleIndexMeta.getSettings()), round, lastRound, - index, + downsamplingMethod, + backingIndex, targetDownsampleIndexMeta.getIndex() ); if (downsamplingNotComplete.isEmpty() == false) { @@ -580,7 +585,7 @@ private Set waitForInProgressOrTriggerDownsampling( // no maintenance needed for previously started downsampling actions and we are on the last matching round so it's time // to kick off downsampling affectedIndices.add(index); - downsampleIndexOnce(round, project.id(), indexName, downsampleIndexName); + downsampleIndexOnce(round, downsamplingMethod, project.id(), backingIndex, downsampleIndexName); } } } @@ -592,16 +597,30 @@ private Set waitForInProgressOrTriggerDownsampling( */ private void downsampleIndexOnce( DataStreamLifecycle.DownsamplingRound round, + DownsampleConfig.SamplingMethod requestedDownsamplingMethod, ProjectId projectId, - String sourceIndex, + IndexMetadata sourceIndexMetadata, String downsampleIndexName ) { + // When an index is already downsampled with a method, we require all later downsampling rounds to use the same method. + // This is necessary to preserve the relation of the downsampled index to the raw data. For example, if an index is already + // downsampled and downsampled it again to 1 hour; we know that a document represents either the aggregated raw data of an hour + // or the last value of the raw data within this hour. If we mix the methods, we cannot derive any meaning from them. + // Furthermore, data stream lifecycle is configured on the data stream level and not on the individual index level, meaning that + // when a user changes downsampling method, some indices would not be able to be downsampled anymore. + // For this reason, when we encounter an already downsampled index, we use the source downsampling method which might be different + // from the requested one. + var sourceIndexSamplingMethod = DownsampleConfig.SamplingMethod.fromIndexMetadata(sourceIndexMetadata); + String sourceIndex = sourceIndexMetadata.getIndex().getName(); DownsampleAction.Request request = new DownsampleAction.Request( TimeValue.THIRTY_SECONDS /* TODO should this be longer/configurable? */, sourceIndex, downsampleIndexName, null, - round.config() + new DownsampleConfig( + round.fixedInterval(), + sourceIndexSamplingMethod == null ? requestedDownsamplingMethod : sourceIndexSamplingMethod + ) ); transportActionsDeduplicator.executeOnce( Tuple.tuple(projectId, request), @@ -632,11 +651,12 @@ private Set evaluateDownsampleStatus( IndexMetadata.DownsampleTaskStatus downsampleStatus, DataStreamLifecycle.DownsamplingRound currentRound, DataStreamLifecycle.DownsamplingRound lastRound, - Index backingIndex, + DownsampleConfig.SamplingMethod downsamplingMethod, + IndexMetadata backingIndex, Index downsampleIndex ) { Set affectedIndices = new HashSet<>(); - String indexName = backingIndex.getName(); + String indexName = backingIndex.getIndex().getName(); String downsampleIndexName = downsampleIndex.getName(); return switch (downsampleStatus) { case UNKNOWN -> { @@ -683,15 +703,15 @@ private Set evaluateDownsampleStatus( // NOTE that the downsample request is made through the deduplicator so it will only really be executed if // there isn't one already in-flight. This can happen if a previous request timed-out, failed, or there was a // master failover and data stream lifecycle needed to restart - downsampleIndexOnce(currentRound, projectId, indexName, downsampleIndexName); - affectedIndices.add(backingIndex); + downsampleIndexOnce(currentRound, downsamplingMethod, projectId, backingIndex, downsampleIndexName); + affectedIndices.add(backingIndex.getIndex()); yield affectedIndices; } case SUCCESS -> { if (dataStream.getIndices().contains(downsampleIndex) == false) { // at this point the source index is part of the data stream and the downsample index is complete but not // part of the data stream. we need to replace the source index with the downsample index in the data stream - affectedIndices.add(backingIndex); + affectedIndices.add(backingIndex.getIndex()); replaceBackingIndexWithDownsampleIndexOnce(projectId, dataStream, indexName, downsampleIndexName); } yield affectedIndices; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java index 2d975845fcd27..0d1cd1be1d95e 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.PUT; import static org.elasticsearch.rest.RestUtils.getAckTimeout; @@ -29,6 +30,9 @@ @ServerlessScope(Scope.PUBLIC) public class RestPutDataStreamLifecycleAction extends BaseRestHandler { + private static final String SUPPORTS_DOWNSAMPLING_METHOD = "dlm.downsampling_method"; + private static final Set CAPABILITIES = Set.of(SUPPORTS_DOWNSAMPLING_METHOD); + @Override public String getName() { return "put_data_lifecycles_action"; @@ -44,13 +48,14 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli try (XContentParser parser = request.contentParser()) { PutDataStreamLifecycleAction.Request putLifecycleRequest = PutDataStreamLifecycleAction.Request.parseRequest( parser, - (dataRetention, enabled, downsampling) -> new PutDataStreamLifecycleAction.Request( + (dataRetention, enabled, downsamplingRounds, downsamplingMethod) -> new PutDataStreamLifecycleAction.Request( getMasterNodeTimeout(request), getAckTimeout(request), Strings.splitStringByCommaToArray(request.param("name")), dataRetention, enabled, - downsampling + downsamplingRounds, + downsamplingMethod ) ); putLifecycleRequest.indicesOptions(IndicesOptions.fromRequest(request, putLifecycleRequest.indicesOptions())); @@ -61,4 +66,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli ); } } + + @Override + public Set supportedCapabilities() { + return CAPABILITIES; + } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java index fd29a2087db86..574c30376137c 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java @@ -43,6 +43,8 @@ import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.composeDataLifecycles; import static org.elasticsearch.common.settings.Settings.builder; import static org.elasticsearch.datastreams.MetadataDataStreamRolloverServiceTests.createSettingsProvider; +import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.randomResettable; +import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.randomSamplingMethod; import static org.elasticsearch.indices.ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -146,51 +148,63 @@ public void testLifecycleComposition() { List lifecycles = List.of(); assertThat(composeDataLifecycles(lifecycles), nullValue()); } - // One lifecycle results to this lifecycle as the final + // One lifecycle results in this lifecycle as the final { + ResettableValue> downsamplingRounds = randomDownsampling(); DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.createDataLifecycleTemplate( true, randomRetention(), - randomDownsampling() + downsamplingRounds, + randomResettable(() -> randomSamplingMethod(downsamplingRounds.get())) ); List lifecycles = List.of(lifecycle); DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); // Defaults to true assertThat(result.enabled(), equalTo(true)); assertThat(result.dataRetention(), equalTo(lifecycle.dataRetention().get())); - assertThat(result.downsampling(), equalTo(lifecycle.downsampling().get())); + assertThat(result.downsamplingRounds(), equalTo(lifecycle.downsamplingRounds().get())); + assertThat(result.downsamplingMethod(), equalTo(lifecycle.downsamplingMethod().get())); } // If the last lifecycle is missing a property (apart from enabled) we keep the latest from the previous ones // Enabled is always true unless it's explicitly set to false { + List downsamplingRounds = randomRounds(); DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.createDataLifecycleTemplate( false, randomPositiveTimeValue(), - randomRounds() + downsamplingRounds, + randomSamplingMethod(downsamplingRounds) ); List lifecycles = List.of(lifecycle, DataStreamLifecycle.Template.DATA_DEFAULT); DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); assertThat(result.enabled(), equalTo(true)); assertThat(result.dataRetention(), equalTo(lifecycle.dataRetention().get())); - assertThat(result.downsampling(), equalTo(lifecycle.downsampling().get())); + assertThat(result.downsamplingRounds(), equalTo(lifecycle.downsamplingRounds().get())); + assertThat(result.downsamplingMethod(), equalTo(lifecycle.downsamplingMethod().get())); } - // If both lifecycle have all properties, then the latest one overwrites all the others + // If both lifecycles have all properties, then the latest one overwrites all the others { + DownsampleConfig.SamplingMethod downsamplingMethod1 = randomFrom(DownsampleConfig.SamplingMethod.LAST_VALUE); DataStreamLifecycle.Template lifecycle1 = DataStreamLifecycle.createDataLifecycleTemplate( false, randomPositiveTimeValue(), - randomRounds() + randomRounds(), + downsamplingMethod1 ); DataStreamLifecycle.Template lifecycle2 = DataStreamLifecycle.createDataLifecycleTemplate( true, randomPositiveTimeValue(), - randomRounds() + randomRounds(), + downsamplingMethod1 == DownsampleConfig.SamplingMethod.LAST_VALUE + ? DownsampleConfig.SamplingMethod.AGGREGATE + : DownsampleConfig.SamplingMethod.LAST_VALUE ); List lifecycles = List.of(lifecycle1, lifecycle2); DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); assertThat(result.enabled(), equalTo(lifecycle2.enabled())); assertThat(result.dataRetention(), equalTo(lifecycle2.dataRetention().get())); - assertThat(result.downsampling(), equalTo(lifecycle2.downsampling().get())); + assertThat(result.downsamplingRounds(), equalTo(lifecycle2.downsamplingRounds().get())); + assertThat(result.downsamplingMethod(), equalTo(lifecycle2.downsamplingMethod().get())); } } @@ -255,7 +269,7 @@ private static List randomRounds() { List rounds = new ArrayList<>(); var previous = new DataStreamLifecycle.DownsamplingRound( TimeValue.timeValueDays(randomIntBetween(1, 365)), - new DownsampleConfig(new DateHistogramInterval(randomIntBetween(1, 24) + "h")) + new DateHistogramInterval(randomIntBetween(1, 24) + "h") ); rounds.add(previous); for (int i = 0; i < count; i++) { @@ -268,9 +282,7 @@ private static List randomRounds() { private static DataStreamLifecycle.DownsamplingRound nextRound(DataStreamLifecycle.DownsamplingRound previous) { var after = TimeValue.timeValueDays(previous.after().days() + randomIntBetween(1, 10)); - var fixedInterval = new DownsampleConfig( - new DateHistogramInterval((previous.config().getFixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms") - ); + var fixedInterval = new DateHistogramInterval((previous.fixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms"); return new DataStreamLifecycle.DownsamplingRound(after, fixedInterval); } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java index 8f6605cefae9f..10b772249e436 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java @@ -154,7 +154,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti .setGeneration(3) .setAllowCustomRouting(true) .setIndexMode(IndexMode.STANDARD) - .setLifecycle(DataStreamLifecycle.createDataLifecycle(false, null, null)) + .setLifecycle(DataStreamLifecycle.createDataLifecycle(false, null, null, null)) .setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED) .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build()) .build(); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java index 163bff3dfd2cb..c0625e34f3153 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java @@ -38,7 +38,9 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance; import static org.elasticsearch.test.ESIntegTestCase.client; +import static org.elasticsearch.test.ESTestCase.between; import static org.elasticsearch.test.ESTestCase.frequently; +import static org.elasticsearch.test.ESTestCase.randomFrom; import static org.elasticsearch.test.ESTestCase.randomIntBetween; import static org.junit.Assert.assertTrue; @@ -143,14 +145,18 @@ static void putComposableIndexTemplate( } static DataStreamLifecycle.Template randomDataLifecycleTemplate() { + ResettableValue> downsampling = randomResettable( + DataStreamLifecycleFixtures::randomDownsamplingRounds + ); return DataStreamLifecycle.createDataLifecycleTemplate( frequently(), randomResettable(ESTestCase::randomTimeValue), - randomResettable(DataStreamLifecycleFixtures::randomDownsamplingRounds) + downsampling, + randomResettable(() -> randomSamplingMethod(downsampling.get())) ); } - private static ResettableValue randomResettable(Supplier supplier) { + public static ResettableValue randomResettable(Supplier supplier) { return switch (randomIntBetween(0, 2)) { case 0 -> ResettableValue.undefined(); case 1 -> ResettableValue.reset(); @@ -164,7 +170,7 @@ private static List randomDownsamplingRou List rounds = new ArrayList<>(); var previous = new DataStreamLifecycle.DownsamplingRound( TimeValue.timeValueDays(randomIntBetween(1, 365)), - new DownsampleConfig(new DateHistogramInterval(randomIntBetween(1, 24) + "h")) + new DateHistogramInterval(randomIntBetween(1, 24) + "h") ); rounds.add(previous); for (int i = 0; i < count; i++) { @@ -177,9 +183,19 @@ private static List randomDownsamplingRou private static DataStreamLifecycle.DownsamplingRound nextRound(DataStreamLifecycle.DownsamplingRound previous) { var after = TimeValue.timeValueDays(previous.after().days() + randomIntBetween(1, 10)); - var fixedInterval = new DownsampleConfig( - new DateHistogramInterval((previous.config().getFixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms") - ); + var fixedInterval = new DateHistogramInterval((previous.fixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms"); return new DataStreamLifecycle.DownsamplingRound(after, fixedInterval); } + + /** + * In order to produce valid data stream lifecycle configurations, the sampling method can be defined only when + * the downsampling rounds are also defined. + */ + public static DownsampleConfig.SamplingMethod randomSamplingMethod(List downsamplingRounds) { + if (downsamplingRounds == null || between(0, DownsampleConfig.SamplingMethod.values().length) == 0) { + return null; + } else { + return randomFrom(DownsampleConfig.SamplingMethod.values()); + } + } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index 0ce32cf1f94ab..35c709462d5d9 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -1228,9 +1228,7 @@ public void testDownsampling() throws Exception { .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "@timestamp"), DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( - List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m")))) - ) + .downsamplingRounds(List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")))) .dataRetention(TimeValue.MAX_VALUE) .build(), now @@ -1377,9 +1375,7 @@ public void testDownsamplingWhenTargetIndexNameClashYieldsException() throws Exc .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "@timestamp"), DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( - List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m")))) - ) + .downsamplingRounds(List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")))) .dataRetention(TimeValue.MAX_VALUE) .build(), now @@ -1662,9 +1658,7 @@ private ClusterState downsampleSetup(ProjectId projectId, String dataStreamName, settings(IndexVersion.current()).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "@timestamp"), DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( - List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m")))) - ) + .downsamplingRounds(List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")))) .dataRetention(TimeValue.timeValueMillis(1)) .build(), now diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml index 4bf6ccfbfa7ce..56c8f764a3d2d 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml @@ -171,6 +171,58 @@ teardown: - match: { data_streams.1.lifecycle.downsampling.1.after: '100d'} - match: { data_streams.1.lifecycle.downsampling.1.fixed_interval: '10h'} +--- +"Put data stream lifecycle with downsampling method": + - requires: + capabilities: + - method: PUT + path: /_data_stream/{name}/_lifecycle + capabilities: [ "dlm.downsampling_method" ] + test_runner_features: [ "capabilities" ] + reason: "Downsampling method was added to data stream lifecycle was available from 9.3" + + - do: + indices.put_data_lifecycle: + name: "*" + body: > + { + "downsampling": [ + { + "after": "10d", + "fixed_interval": "1h" + }, + { + "after": "100d", + "fixed_interval": "10h" + } + ], + "downsampling_method": "aggregate", + "data_retention": "30d", + "enabled": false + } + + - is_true: acknowledged + + - do: + indices.get_data_lifecycle: + name: "*" + - length: { data_streams: 2 } + - match: { data_streams.0.name: data-stream-with-lifecycle } + - match: { data_streams.0.lifecycle.data_retention: "30d" } + - match: { data_streams.0.lifecycle.enabled: false} + - match: { data_streams.0.lifecycle.downsampling.0.after: '10d'} + - match: { data_streams.0.lifecycle.downsampling.0.fixed_interval: '1h'} + - match: { data_streams.0.lifecycle.downsampling.1.after: '100d'} + - match: { data_streams.0.lifecycle.downsampling.1.fixed_interval: '10h'} + - match: { data_streams.1.name: simple-data-stream1 } + - match: { data_streams.1.lifecycle.data_retention: "30d" } + - match: { data_streams.1.lifecycle.enabled: false} + - match: { data_streams.1.lifecycle.downsampling_method: 'aggregate'} + - match: { data_streams.1.lifecycle.downsampling.0.after: '10d'} + - match: { data_streams.1.lifecycle.downsampling.0.fixed_interval: '1h'} + - match: { data_streams.1.lifecycle.downsampling.1.after: '100d'} + - match: { data_streams.1.lifecycle.downsampling.1.fixed_interval: '10h'} + --- "Enable lifecycle": diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java index c97b0a28d22de..dcbf5bcb87e41 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java @@ -678,4 +678,9 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java index ef72e234f8d6b..3a567490fc218 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java @@ -385,4 +385,9 @@ public void testLoadSyntheticSourceFromStringOrBytesRef() throws IOException { } } } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java index 917941b39e888..174bc44cbaecc 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java @@ -215,4 +215,9 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java index 6818b7936d124..c229752917560 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java @@ -209,4 +209,9 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean syntheticSource) protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java index dce105f339a52..7f988ebcb04f4 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java @@ -649,4 +649,12 @@ private double encodeDecode(double value, double scalingFactor) { private static double randomValue() { return randomBoolean() ? randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true) : randomFloat(); } + + @Override + protected List getSortShortcutSupport() { + return List.of( + // TODO doubles currently disable pruning, can we re-enable? + new SortShortcutSupport(this::minimalMapping, this::writeField, false) + ); + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java index 3e2bb422c2062..9965259e294da 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java @@ -877,4 +877,9 @@ protected RandomIndexWriter indexWriterForSyntheticSource(Directory directory) t protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java index 8105eee707cc3..b77ab875926e8 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java @@ -264,4 +264,9 @@ public void testAggregationsDocValuesDisabled() throws IOException { })); assertAggregatableConsistency(mapperService.fieldType("field")); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/muted-tests.yml b/muted-tests.yml index 231138c6b0efd..d43272b21542a 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -417,33 +417,6 @@ tests: - class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT method: test {p0=field_caps/10_basic/Field caps for number field with only doc values} issue: https://github.com/elastic/elasticsearch/issues/136244 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testDataStreams {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/136353 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testServiceAccountApiKey {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/136390 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testWatcher {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/136391 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testSingleDoc {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/136392 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testTransformLegacyTemplateCleanup {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/136393 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testApiKeySuperuser {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/136394 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testSecurityNativeRealm {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/136395 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testSlmPolicyAndStats {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/136399 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testRollupAfterRestart {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/136437 - class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT method: test {p0=search.vectors/200_dense_vector_docvalue_fields/Enable docvalue_fields parameter for dense_vector fields} issue: https://github.com/elastic/elasticsearch/issues/136443 @@ -492,9 +465,6 @@ tests: - class: org.elasticsearch.readiness.ReadinessClusterIT method: testReadinessDuringRestartsNormalOrder issue: https://github.com/elastic/elasticsearch/issues/136955 -- class: org.elasticsearch.xpack.ilm.TimeSeriesDataStreamsIT - method: testSearchableSnapshotAction - issue: https://github.com/elastic/elasticsearch/issues/137167 - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT method: test {yaml=indices.validate_query/20_query_string/validate_query with query_string parameters} issue: https://github.com/elastic/elasticsearch/issues/137391 @@ -504,6 +474,12 @@ tests: - class: org.elasticsearch.xpack.esql.qa.single_node.PushQueriesIT method: testEqualityAndOther {SEMANTIC_TEXT_WITH_KEYWORD} issue: https://github.com/elastic/elasticsearch/issues/137491 +- class: org.elasticsearch.xpack.ilm.CCRIndexLifecycleIT + method: testBasicCCRAndILMIntegration {targetCluster=FOLLOWER} + issue: https://github.com/elastic/elasticsearch/issues/137494 +- class: org.elasticsearch.xpack.ilm.CCRIndexLifecycleIT + method: testCcrAndIlmWithRollover {targetCluster=FOLLOWER} + issue: https://github.com/elastic/elasticsearch/issues/137528 # Examples: # diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java index 034a25e854298..275a70eb43f57 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java @@ -318,4 +318,9 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index ae3d03c3a503f..b808cffd75c7a 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -704,4 +704,9 @@ protected void validateRoundTripReader(String syntheticSource, DirectoryReader r protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index d04f2067d8aed..2978ee458dda9 100644 --- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -148,4 +148,9 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml index 800dec2a795a4..c230371cc4003 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml @@ -142,6 +142,38 @@ - match: {component_templates.0.component_template.template.lifecycle.enabled: true} - match: {component_templates.0.component_template.template.lifecycle.data_retention: "10d"} +--- +"Add data stream lifecycle with downsampling": + - requires: + capabilities: + - method: PUT + path: /_component_template/{name} + capabilities: [ "dlm.downsampling_method" ] + test_runner_features: [ "capabilities" ] + reason: "Downsampling method was added to data stream lifecycle was available from 9.3" + + - do: + cluster.put_component_template: + name: test-lifecycle + body: + template: + lifecycle: + data_retention: "10d" + downsampling_method: last_value + downsampling: + - {"after": "1d", "fixed_interval": "5m"} + + - do: + cluster.get_component_template: + name: test-lifecycle + + - match: {component_templates.0.name: test-lifecycle} + - match: {component_templates.0.component_template.template.lifecycle.enabled: true} + - match: {component_templates.0.component_template.template.lifecycle.data_retention: "10d"} + - match: {component_templates.0.component_template.template.lifecycle.downsampling_method: "last_value"} + - match: {component_templates.0.component_template.template.lifecycle.downsampling.0.after: "1d"} + - match: {component_templates.0.component_template.template.lifecycle.downsampling.0.fixed_interval: "5m"} + --- "Get data stream lifecycle with default rollover": - requires: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml index 81068f460b2b3..b538293d668ea 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml @@ -197,3 +197,37 @@ name: baz - match: {index_templates.0.name: baz} + +--- +"Add data stream lifecycle with downsampling": + - requires: + capabilities: + - method: PUT + path: /_index_template/{name} + capabilities: [ "dlm.downsampling_method" ] + test_runner_features: [ "capabilities" ] + reason: "Downsampling method was added to data stream lifecycle was available from 9.3" + + - do: + indices.put_index_template: + name: test-lifecycle + body: + index_patterns: downsampling-* + data_stream: {} + template: + lifecycle: + data_retention: "10d" + downsampling_method: last_value + downsampling: + - {"after": "1d", "fixed_interval": "5m"} + + - do: + indices.get_index_template: + name: test-lifecycle + + - match: {index_templates.0.name: test-lifecycle} + - match: {index_templates.0.index_template.template.lifecycle.enabled: true} + - match: {index_templates.0.index_template.template.lifecycle.data_retention: "10d"} + - match: {index_templates.0.index_template.template.lifecycle.downsampling_method: "last_value"} + - match: {index_templates.0.index_template.template.lifecycle.downsampling.0.after: "1d"} + - match: {index_templates.0.index_template.template.lifecycle.downsampling.0.fixed_interval: "5m"} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/200_dense_vector_docvalue_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/200_dense_vector_docvalue_fields.yml index 161fc23a84651..59b9840fdfd28 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/200_dense_vector_docvalue_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/200_dense_vector_docvalue_fields.yml @@ -123,7 +123,6 @@ setup: - match: {hits.hits.0.fields.vector5.0: [1, 111, -13, 15, -128]} - match: {hits.hits.0.fields.vector6.0: [-1, 11, 0, 12, 111]} - - match: {hits.hits.1._id: "2"} - match: {hits.hits.1.fields.name.0: "moose.jpg"} @@ -143,7 +142,6 @@ setup: - match: {hits.hits.1.fields.vector4.0: [-1, 50, -1, 1, 120]} - match: {hits.hits.1.fields.vector5.0: [1, 111, -13, 15, -128]} - - match: {hits.hits.2._id: "3"} - match: {hits.hits.2.fields.name.0: "rabbit.jpg"} @@ -161,3 +159,103 @@ setup: - close_to: { hits.hits.2.fields.vector2.0.4: { value: -100.0, error: 0.001 } } - match: {hits.hits.2.fields.vector3.0: [-1, 100, -13, 15, -128]} + +--- +"dense_vector docvalues with bfloat16": + - requires: + cluster_features: [ "mapper.vectors.hnsw_bfloat16_on_disk_rescoring" ] + reason: Needs bfloat16 support + - do: + indices.create: + index: test-bfloat16 + body: + mappings: + properties: + name: + type: keyword + vector7: + type: dense_vector + element_type: bfloat16 + dims: 5 + index: true + vector8: + type: dense_vector + element_type: bfloat16 + dims: 5 + index: false + + - do: + index: + index: test-bfloat16 + id: "1" + body: + name: cow.jpg + vector7: [230.0, 300.33, -34.8988, 15.555, -200.0] + vector8: [130.0, 115.0, -1.02, 15.555, -100.0] + - do: + index: + index: test-bfloat16 + id: "2" + body: + name: moose.jpg + vector7: [-0.5, 100.0, -13, 14.8, -156.0] + - do: + index: + index: test-bfloat16 + id: "3" + body: + name: rabbit.jpg + vector8: [130.0, 115.0, -1.02, 15.555, -100.0] + + - do: + indices.refresh: {} + + - do: + search: + _source: false + index: test-bfloat16 + body: + docvalue_fields: [name, vector7, vector8] + sort: name + + - match: {hits.hits.0._id: "1"} + - match: {hits.hits.0.fields.name.0: "cow.jpg"} + + - length: {hits.hits.0.fields.vector7.0: 5} + - length: {hits.hits.0.fields.vector8.0: 5} + + - close_to: { hits.hits.0.fields.vector7.0.0: { value: 230.0, error: 0.1 } } + - close_to: { hits.hits.0.fields.vector7.0.1: { value: 300.33, error: 0.1 } } + - close_to: { hits.hits.0.fields.vector7.0.2: { value: -34.8988, error: 0.1 } } + - close_to: { hits.hits.0.fields.vector7.0.3: { value: 15.555, error: 0.1 } } + - close_to: { hits.hits.0.fields.vector7.0.4: { value: -200.0, error: 0.1 } } + + - close_to: { hits.hits.0.fields.vector8.0.0: { value: 130.0, error: 0.1 } } + - close_to: { hits.hits.0.fields.vector8.0.1: { value: 115.0, error: 0.1 } } + - close_to: { hits.hits.0.fields.vector8.0.2: { value: -1.02, error: 0.1 } } + - close_to: { hits.hits.0.fields.vector8.0.3: { value: 15.555, error: 0.1 } } + - close_to: { hits.hits.0.fields.vector8.0.4: { value: -100.0, error: 0.1 } } + + - match: {hits.hits.1._id: "2"} + - match: {hits.hits.1.fields.name.0: "moose.jpg"} + + - length: {hits.hits.1.fields.vector7.0: 5} + - match: {hits.hits.1.fields.vector8: null} + + - close_to: { hits.hits.1.fields.vector7.0.0: { value: -0.5, error: 0.1 } } + - close_to: { hits.hits.1.fields.vector7.0.1: { value: 100.0, error: 0.1 } } + - close_to: { hits.hits.1.fields.vector7.0.2: { value: -13, error: 0.1 } } + - close_to: { hits.hits.1.fields.vector7.0.3: { value: 14.8, error: 0.1 } } + - close_to: { hits.hits.1.fields.vector7.0.4: { value: -156.0, error: 0.1 } } + + - match: {hits.hits.2._id: "3"} + - match: {hits.hits.2.fields.name.0: "rabbit.jpg"} + + - length: {hits.hits.2.fields.vector8.0: 5} + - match: {hits.hits.2.fields.vector7: null} + + - close_to: { hits.hits.2.fields.vector8.0.0: { value: 130.0, error: 0.1 } } + - close_to: { hits.hits.2.fields.vector8.0.1: { value: 115.0, error: 0.1 } } + - close_to: { hits.hits.2.fields.vector8.0.2: { value: -1.02, error: 0.1 } } + - close_to: { hits.hits.2.fields.vector8.0.3: { value: 15.555, error: 0.1 } } + - close_to: { hits.hits.2.fields.vector8.0.4: { value: -100.0, error: 0.1 } } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search_bfloat16.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search_bfloat16.yml new file mode 100644 index 0000000000000..aa80764f14136 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search_bfloat16.yml @@ -0,0 +1,707 @@ +setup: + - requires: + cluster_features: ["mapper.vectors.hnsw_bfloat16_on_disk_rescoring"] + reason: 'bfloat16 needs to be supported' + - do: + indices.create: + index: test + body: + mappings: + properties: + name: + type: keyword + vector: + type: dense_vector + element_type: bfloat16 + dims: 5 + index: true + similarity: l2_norm + index_options: + type: hnsw + m: 16 + ef_construction: 200 + another_vector: + type: dense_vector + element_type: bfloat16 + dims: 5 + index: true + similarity: l2_norm + index_options: + type: hnsw + m: 16 + ef_construction: 200 + - do: + index: + index: test + id: "1" + body: + name: cow.jpg + vector: [ 230.0, 300.33, -34.8988, 15.555, -200.0 ] + another_vector: [ 130.0, 115.0, -1.02, 15.555, -100.0 ] + + - do: + index: + index: test + id: "2" + body: + name: moose.jpg + vector: [ -0.5, 100.0, -13, 14.8, -156.0 ] + another_vector: [ -0.5, 50.0, -1, 1, 120 ] + + - do: + index: + index: test + id: "3" + body: + name: rabbit.jpg + vector: [ 0.5, 111.3, -13.0, 14.8, -156.0 ] + another_vector: [ -0.5, 11.0, 0, 12, 111.0 ] + + - do: + indices.refresh: { } + +--- +"kNN search only": + - requires: + cluster_features: "gte_v8.4.0" + reason: 'kNN added to search endpoint in 8.4' + - do: + search: + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + k: 2 + num_candidates: 3 + + - match: { hits.hits.0._id: "2" } + - match: { hits.hits.0.fields.name.0: "moose.jpg" } + + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.1.fields.name.0: "rabbit.jpg" } +--- +"kNN multi-field search only": + - requires: + cluster_features: "gte_v8.7.0" + reason: 'multi-field kNN search added to search endpoint in 8.7' + - do: + search: + index: test + body: + fields: [ "name" ] + knn: + - { field: vector, query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ], k: 2, num_candidates: 3 } + - { field: another_vector, query_vector: [ -0.5, 11.0, 0, 12, 111.0 ], k: 2, num_candidates: 3 } + + - match: { hits.hits.0._id: "3" } + - match: { hits.hits.0.fields.name.0: "rabbit.jpg" } + + - match: { hits.hits.1._id: "2" } + - match: { hits.hits.1.fields.name.0: "moose.jpg" } +--- +"kNN search plus query": + - requires: + cluster_features: "gte_v8.4.0" + reason: 'kNN added to search endpoint in 8.4' + - do: + search: + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + k: 2 + num_candidates: 3 + query: + term: + name: cow.jpg + + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.0.fields.name.0: "cow.jpg" } + + - match: { hits.hits.1._id: "2" } + - match: { hits.hits.1.fields.name.0: "moose.jpg" } + + - match: { hits.hits.2._id: "3" } + - match: { hits.hits.2.fields.name.0: "rabbit.jpg" } +--- +"kNN multi-field search with query": + - requires: + cluster_features: "gte_v8.7.0" + reason: 'multi-field kNN search added to search endpoint in 8.7' + - do: + search: + index: test + body: + fields: [ "name" ] + knn: + - { field: vector, query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ], k: 2, num_candidates: 3 } + - { field: another_vector, query_vector: [ -0.5, 11.0, 0, 12, 111.0 ], k: 2, num_candidates: 3 } + query: + term: + name: cow.jpg + + - match: { hits.hits.0._id: "3" } + - match: { hits.hits.0.fields.name.0: "rabbit.jpg" } + + - match: { hits.hits.1._id: "1" } + - match: { hits.hits.1.fields.name.0: "cow.jpg" } + + - match: { hits.hits.2._id: "2" } + - match: { hits.hits.2.fields.name.0: "moose.jpg" } +--- +"kNN search with filter": + - requires: + cluster_features: "gte_v8.4.0" + reason: 'kNN added to search endpoint in 8.4' + - do: + search: + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + k: 2 + num_candidates: 3 + filter: + term: + name: "rabbit.jpg" + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "3" } + - match: { hits.hits.0.fields.name.0: "rabbit.jpg" } + + - do: + search: + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + k: 2 + num_candidates: 3 + filter: + - term: + name: "rabbit.jpg" + - term: + _id: 2 + + - match: { hits.total.value: 0 } + +--- +"kNN search with explicit search_type": + - requires: + cluster_features: "gte_v8.4.0" + reason: 'kNN added to search endpoint in 8.4' + - do: + catch: bad_request + search: + index: test + search_type: query_then_fetch + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + k: 2 + num_candidates: 3 + + - match: { error.root_cause.0.type: "illegal_argument_exception" } + - match: { error.root_cause.0.reason: "cannot set [search_type] when using [knn] search, since the search type is determined automatically" } + +--- +"kNN search in _knn_search endpoint": + - skip: + features: [ "allowed_warnings", "headers" ] + - do: + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=8" + Accept: "application/vnd.elasticsearch+json;compatible-with=8" + allowed_warnings: + - "The kNN search API has been replaced by the `knn` option in the search API." + knn_search: + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + k: 2 + num_candidates: 3 + + - match: { hits.hits.0._id: "2" } + - match: { hits.hits.0.fields.name.0: "moose.jpg" } + + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.1.fields.name.0: "rabbit.jpg" } + +--- +"kNN search with filter in _knn_search endpoint": + - requires: + cluster_features: "gte_v8.2.0" + reason: 'kNN with filtering added in 8.2' + test_runner_features: [ "allowed_warnings", "headers" ] + - do: + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=8" + Accept: "application/vnd.elasticsearch+json;compatible-with=8" + allowed_warnings: + - "The kNN search API has been replaced by the `knn` option in the search API." + knn_search: + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + k: 2 + num_candidates: 3 + filter: + term: + name: "rabbit.jpg" + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "3" } + - match: { hits.hits.0.fields.name.0: "rabbit.jpg" } + + - do: + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=8" + Accept: "application/vnd.elasticsearch+json;compatible-with=8" + allowed_warnings: + - "The kNN search API has been replaced by the `knn` option in the search API." + knn_search: + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + k: 2 + num_candidates: 3 + filter: + - term: + name: "rabbit.jpg" + - term: + _id: 2 + + - match: { hits.total.value: 0 } + +--- +"Test nonexistent field is match none": + - requires: + cluster_features: "gte_v8.16.0" + reason: 'non-existent field handling improved in 8.16' + - do: + search: + index: test + body: + fields: [ "name" ] + knn: + field: nonexistent + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + k: 2 + num_candidates: 3 + + - length: { hits.hits: 0 } + + - do: + indices.create: + index: test_nonexistent + body: + mappings: + properties: + name: + type: keyword + vector: + type: dense_vector + element_type: float + dims: 5 + index: true + similarity: l2_norm + settings: + index.query.parse.allow_unmapped_fields: false + + - do: + catch: bad_request + search: + index: test_nonexistent + body: + fields: [ "name" ] + knn: + field: nonexistent + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + k: 2 + num_candidates: 3 + + - match: { error.root_cause.0.type: "query_shard_exception" } + - match: { error.root_cause.0.reason: "No field mapping can be found for the field with name [nonexistent]" } + +--- +"KNN Vector similarity search only": + - requires: + cluster_features: "gte_v8.8.0" + reason: 'kNN similarity added in 8.8' + - do: + search: + index: test + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + similarity: 11 + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + + - length: { hits.hits: 1 } + + - match: { hits.hits.0._id: "2" } + - match: { hits.hits.0.fields.name.0: "moose.jpg" } +--- +"Vector similarity with filter only": + - requires: + cluster_features: "gte_v8.8.0" + reason: 'kNN similarity added in 8.8' + - do: + search: + index: test + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + similarity: 11 + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + filter: { "term": { "name": "moose.jpg" } } + + - length: { hits.hits: 1 } + + - match: { hits.hits.0._id: "2" } + - match: { hits.hits.0.fields.name.0: "moose.jpg" } + + - do: + search: + index: test + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + similarity: 110 + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + filter: { "term": { "name": "cow.jpg" } } + + - length: { hits.hits: 0 } +--- +"Knn search with mip": + - requires: + cluster_features: "gte_v8.11.0" + reason: 'mip similarity added in 8.11' + test_runner_features: "close_to" + + - do: + indices.create: + index: mip + body: + mappings: + properties: + name: + type: keyword + vector: + type: dense_vector + dims: 5 + index: true + similarity: max_inner_product + index_options: + type: hnsw + m: 16 + ef_construction: 200 + + - do: + index: + index: mip + id: "1" + body: + name: cow.jpg + vector: [ 230.0, 300.33, -34.8988, 15.555, -200.0 ] + + - do: + index: + index: mip + id: "2" + body: + name: moose.jpg + vector: [ -0.5, 100.0, -13, 14.8, -156.0 ] + + - do: + index: + index: mip + id: "3" + body: + name: rabbit.jpg + vector: [ 0.5, 111.3, -13.0, 14.8, -156.0 ] + + - do: + indices.refresh: { } + + - do: + search: + index: mip + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + + + - length: { hits.hits: 3 } + - match: { hits.hits.0._id: "1" } + - close_to: { hits.hits.0._score: { value: 58694.902, error: 0.01 } } + - match: { hits.hits.1._id: "3" } + - close_to: { hits.hits.1._score: { value: 34702.79, error: 0.01 } } + - match: { hits.hits.2._id: "2" } + - close_to: { hits.hits.2._score: { value: 33686.29, error: 0.01 } } + + - do: + search: + index: mip + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + filter: { "term": { "name": "moose.jpg" } } + + + + - length: { hits.hits: 1 } + - match: { hits.hits.0._id: "2" } + - close_to: { hits.hits.0._score: { value: 33686.29, error: 0.01 } } +--- +"Knn search with _name": + - requires: + cluster_features: "gte_v8.15.0" + reason: 'support for _name in knn was added in 8.15' + test_runner_features: "close_to" + + - do: + search: + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + k: 3 + num_candidates: 3 + _name: "my_knn_query" + query: + term: + name: + term: cow.jpg + _name: "my_query" + + + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.0.fields.name.0: "cow.jpg" } + - match: { hits.hits.0.matched_queries.0: "my_knn_query" } + - match: { hits.hits.0.matched_queries.1: "my_query" } + + - match: { hits.hits.1._id: "2" } + - match: { hits.hits.1.fields.name.0: "moose.jpg" } + - match: { hits.hits.1.matched_queries.0: "my_knn_query" } + + - match: { hits.hits.2._id: "3" } + - match: { hits.hits.2.fields.name.0: "rabbit.jpg" } + - match: { hits.hits.2.matched_queries.0: "my_knn_query" } + +--- +"kNN search on empty index should return 0 results and not an error": + - requires: + cluster_features: "gte_v8.15.1" + reason: 'Error fixed in 8.15.1' + - do: + indices.create: + index: test_empty + body: + mappings: + properties: + vector: + type: dense_vector + - do: + search: + index: test_empty + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + k: 2 + num_candidates: 3 + + - match: { hits.total.value: 0 } +--- +"Vector rescoring has no effect for non-quantized vectors and provides same results as non-rescored knn": + - requires: + reason: 'Quantized vector rescoring is required' + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_search + capabilities: [knn_quantized_vector_rescore_oversample] + - skip: + features: "headers" + + # Non-rescored knn + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 3 + num_candidates: 3 + + # Get scores - hit ordering may change depending on how things are distributed + - match: { hits.total: 3 } + - set: { hits.hits.0._score: knn_score0 } + - set: { hits.hits.1._score: knn_score1 } + - set: { hits.hits.2._score: knn_score2 } + + # Rescored knn + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 3 + num_candidates: 3 + rescore_vector: + oversample: 1.5 + + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $knn_score0 } + - match: { hits.hits.1._score: $knn_score1 } + - match: { hits.hits.2._score: $knn_score2 } + +--- +"Dimensions are dynamically set": + - do: + indices.create: + index: test_index + body: + mappings: + properties: + embedding: + type: dense_vector + + - do: + index: + index: test_index + id: "0" + refresh: true + body: + embedding: [ 0.5, 111.3, -13.0, 14.8, -156.0 ] + + # wait and ensure that the mapping update is replicated + - do: + cluster.health: + wait_for_events: languid + + - do: + indices.get_mapping: + index: test_index + + - match: { test_index.mappings.properties.embedding.type: dense_vector } + - match: { test_index.mappings.properties.embedding.dims: 5 } + + - do: + catch: bad_request + index: + index: test_index + id: "0" + body: + embedding: [ 0.5, 111.3 ] + +--- +"Updating dim to null is not allowed": + - requires: + cluster_features: "mapper.npe_on_dims_update_fix" + reason: "dims update fix" + - do: + indices.create: + index: test_index + + - do: + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + dims: 4 + - do: + catch: bad_request + indices.put_mapping: + index: test_index + body: + properties: + embedding: + type: dense_vector + + +--- +"Searching with no data dimensions specified": + - requires: + cluster_features: "search.vectors.no_dimensions_bugfix" + reason: "Search with no dimensions bugfix" + + - do: + indices.create: + index: empty-test + body: + mappings: + properties: + vector: + type: dense_vector + index: true + + - do: + search: + index: empty-test + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [ -0.5, 90.0, -10, 14.8, -156.0 ] + k: 3 + num_candidates: 3 + rescore_vector: + oversample: 1.5 + similarity: 0.1 + + - match: { hits.total.value: 0 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml index e3c1155ed2000..0a2c9eb31dc72 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml @@ -338,6 +338,68 @@ setup: - match: { hits.hits.1._score: $rescore_score1 } - match: { hits.hits.2._score: $rescore_score2 } --- +"Test index configured rescore vector with on-disk rescoring": + - requires: + cluster_features: ["mapper.vectors.hnsw_bfloat16_on_disk_rescoring"] + reason: Needs on_disk_rescoring feature + - skip: + features: "headers" + - do: + indices.create: + index: bbq_on_disk_rescore_hnsw + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: bbq_hnsw + on_disk_rescore: true + rescore_vector: + oversample: 1.5 + + - do: + bulk: + index: bbq_on_disk_rescore_hnsw + refresh: true + body: | + { "index": {"_id": "1"}} + { "vector": [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] } + { "index": {"_id": "2"}} + { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] } + { "index": {"_id": "3"}} + { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] } + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_on_disk_rescore_hnsw + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + + - match: { hits.total: 3 } + - set: { hits.hits.0._score: rescore_score0 } + - set: { hits.hits.1._score: rescore_score1 } + - set: { hits.hits.2._score: rescore_score2 } +--- "Test index configured rescore vector updateable and settable to 0": - requires: cluster_features: ["mapper.dense_vector.rescore_zero_vector"] diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw_bfloat16.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw_bfloat16.yml new file mode 100644 index 0000000000000..d873158e637c3 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw_bfloat16.yml @@ -0,0 +1,580 @@ +setup: + - requires: + cluster_features: "mapper.vectors.hnsw_bfloat16_on_disk_rescoring" + reason: 'bfloat16 needs to be supported' + - do: + indices.create: + index: bbq_hnsw + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + element_type: bfloat16 + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: bbq_hnsw + + - do: + index: + index: bbq_hnsw + id: "1" + body: + vector: [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, + 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, + 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, + -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, + -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, + -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, + -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_hnsw + + - do: + index: + index: bbq_hnsw + id: "2" + body: + vector: [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, + -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, + 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, + -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, + -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, + -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, + 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, + -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_hnsw + + - do: + index: + index: bbq_hnsw + id: "3" + body: + name: rabbit.jpg + vector: [0.139, 0.178, -0.117, 0.399, 0.014, -0.139, 0.347, -0.33 , + 0.139, 0.34 , -0.052, -0.052, -0.249, 0.327, -0.288, 0.049, + 0.464, 0.338, 0.516, 0.247, -0.104, 0.259, -0.209, -0.246, + -0.11 , 0.323, 0.091, 0.442, -0.254, 0.195, -0.109, -0.058, + -0.279, 0.402, -0.107, 0.308, -0.273, 0.019, 0.082, 0.399, + -0.658, -0.03 , 0.276, 0.041, 0.187, -0.331, 0.165, 0.017, + 0.171, -0.203, -0.198, 0.115, -0.007, 0.337, -0.444, 0.615, + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_hnsw + + - do: + indices.forcemerge: + index: bbq_hnsw + max_num_segments: 1 + + - do: + indices.refresh: { } +--- +"Test knn search": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ optimized_scalar_quantization_bbq ] + test_runner_features: capabilities + reason: "BBQ scoring improved and changed with optimized_scalar_quantization_bbq" + - do: + search: + index: bbq_hnsw + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.2._id: "2" } +--- +"Vector rescoring has same scoring as exact search for kNN section": + - requires: + reason: 'Quantized vector rescoring is required' + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_search + capabilities: [knn_quantized_vector_rescore_oversample] + - skip: + features: "headers" + + # Rescore + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_hnsw + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + rescore_vector: + oversample: 1.5 + + # Get rescoring scores - hit ordering may change depending on how things are distributed + - match: { hits.total: 3 } + - set: { hits.hits.0._score: rescore_score0 } + - set: { hits.hits.1._score: rescore_score1 } + - set: { hits.hits.2._score: rescore_score2 } + + # Exact knn via script score + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1" + params: + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $rescore_score0 } + - match: { hits.hits.1._score: $rescore_score1 } + - match: { hits.hits.2._score: $rescore_score2 } + +--- +"Test bad quantization parameters": + - do: + catch: bad_request + indices.create: + index: bad_bbq_hnsw + body: + mappings: + properties: + vector: + type: dense_vector + element_type: bfloat16 + dims: 64 + index: false + index_options: + type: bbq_hnsw +--- +"Test few dimensions fail indexing": + - do: + catch: bad_request + indices.create: + index: bad_bbq_hnsw + body: + mappings: + properties: + vector: + type: dense_vector + element_type: bfloat16 + dims: 42 + index: true + index_options: + type: bbq_hnsw + + - do: + indices.create: + index: dynamic_dim_bbq_hnsw + body: + mappings: + properties: + vector: + type: dense_vector + element_type: bfloat16 + index: true + similarity: l2_norm + index_options: + type: bbq_hnsw + + - do: + catch: bad_request + index: + index: dynamic_dim_bbq_hnsw + body: + vector: [1.0, 2.0, 3.0, 4.0, 5.0] + + - do: + index: + index: dynamic_dim_bbq_hnsw + body: + vector: [1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0] +--- +"Test index configured rescore vector": + - requires: + cluster_features: ["mapper.dense_vector.rescore_vector"] + reason: Needs rescore_vector feature + - skip: + features: "headers" + - do: + indices.create: + index: bbq_rescore_hnsw + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + element_type: bfloat16 + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: bbq_hnsw + rescore_vector: + oversample: 1.5 + + - do: + bulk: + index: bbq_rescore_hnsw + refresh: true + body: | + { "index": {"_id": "1"}} + { "vector": [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] } + { "index": {"_id": "2"}} + { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] } + { "index": {"_id": "3"}} + { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] } + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_rescore_hnsw + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + + - match: { hits.total: 3 } + - set: { hits.hits.0._score: rescore_score0 } + - set: { hits.hits.1._score: rescore_score1 } + - set: { hits.hits.2._score: rescore_score2 } + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_rescore_hnsw + body: + query: + script_score: + query: {match_all: {} } + script: + source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1" + params: + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $rescore_score0 } + - match: { hits.hits.1._score: $rescore_score1 } + - match: { hits.hits.2._score: $rescore_score2 } +--- +"Test index configured rescore vector updateable and settable to 0": + - requires: + cluster_features: ["mapper.dense_vector.rescore_zero_vector"] + reason: Needs rescore_zero_vector feature + + - do: + indices.create: + index: bbq_rescore_0_hnsw + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + element_type: bfloat16 + index_options: + type: bbq_hnsw + rescore_vector: + oversample: 0 + + - do: + indices.create: + index: bbq_rescore_update_hnsw + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + element_type: bfloat16 + index_options: + type: bbq_hnsw + rescore_vector: + oversample: 1 + + - do: + indices.put_mapping: + index: bbq_rescore_update_hnsw + body: + properties: + vector: + type: dense_vector + element_type: bfloat16 + index_options: + type: bbq_hnsw + rescore_vector: + oversample: 0 + + - do: + indices.get_mapping: + index: bbq_rescore_update_hnsw + + - match: { .bbq_rescore_update_hnsw.mappings.properties.vector.index_options.rescore_vector.oversample: 0 } +--- +"Test index configured rescore vector score consistency": + - requires: + cluster_features: ["mapper.dense_vector.rescore_zero_vector"] + reason: Needs rescore_zero_vector feature + - skip: + features: "headers" + - do: + indices.create: + index: bbq_rescore_zero_hnsw + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + element_type: bfloat16 + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: bbq_hnsw + rescore_vector: + oversample: 0 + + - do: + bulk: + index: bbq_rescore_zero_hnsw + refresh: true + body: | + { "index": {"_id": "1"}} + { "vector": [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] } + { "index": {"_id": "2"}} + { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] } + { "index": {"_id": "3"}} + { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] } + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_rescore_zero_hnsw + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + + - match: { hits.total: 3 } + - set: { hits.hits.0._score: raw_score0 } + - set: { hits.hits.1._score: raw_score1 } + - set: { hits.hits.2._score: raw_score2 } + + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_rescore_zero_hnsw + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + rescore_vector: + oversample: 2 + + - match: { hits.total: 3 } + - set: { hits.hits.0._score: override_score0 } + - set: { hits.hits.1._score: override_score1 } + - set: { hits.hits.2._score: override_score2 } + + - do: + indices.put_mapping: + index: bbq_rescore_zero_hnsw + body: + properties: + vector: + type: dense_vector + element_type: bfloat16 + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: bbq_hnsw + rescore_vector: + oversample: 2 + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_rescore_zero_hnsw + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + + - match: { hits.total: 3 } + - set: { hits.hits.0._score: default_rescore0 } + - set: { hits.hits.1._score: default_rescore1 } + - set: { hits.hits.2._score: default_rescore2 } + + - do: + indices.put_mapping: + index: bbq_rescore_zero_hnsw + body: + properties: + vector: + type: dense_vector + element_type: bfloat16 + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: bbq_hnsw + rescore_vector: + oversample: 0 + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_rescore_zero_hnsw + body: + query: + script_score: + query: {match_all: {} } + script: + source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1" + params: + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $override_score0 } + - match: { hits.hits.0._score: $default_rescore0 } + - match: { hits.hits.1._score: $override_score1 } + - match: { hits.hits.1._score: $default_rescore1 } + - match: { hits.hits.2._score: $override_score2 } + - match: { hits.hits.2._score: $default_rescore2 } + +--- +"default oversample value": + - requires: + cluster_features: ["mapper.dense_vector.default_oversample_value_for_bbq"] + reason: "Needs default_oversample_value_for_bbq feature" + - do: + indices.get_mapping: + index: bbq_hnsw + + - match: { bbq_hnsw.mappings.properties.vector.index_options.rescore_vector.oversample: 3.0 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml index b297770eb9104..6d0b57ff394bd 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml @@ -194,6 +194,9 @@ setup: --- "KNN Vector similarity search only": + - requires: + cluster_features: [ "mapper.vectors.hnsw_bfloat16_on_disk_rescoring" ] + reason: "the quantized values, and so the effective similarity scores, changed with the new lucene quantizer" - do: search: index: hnsw_byte_quantized @@ -203,7 +206,7 @@ setup: num_candidates: 3 k: 3 field: vector - similarity: 10.3 + similarity: 10.7 query_vector: [-0.5, 90.0, -10, 14.8, -156.0] - length: {hits.hits: 1} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat_bfloat16.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat_bfloat16.yml new file mode 100644 index 0000000000000..80fee2c53468f --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat_bfloat16.yml @@ -0,0 +1,512 @@ +setup: + - requires: + cluster_features: "mapper.vectors.hnsw_bfloat16_on_disk_rescoring" + reason: 'bfloat16 needs to be supported' + - do: + indices.create: + index: bbq_flat + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + element_type: bfloat16 + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: bbq_flat + + - do: + index: + index: bbq_flat + id: "1" + body: + vector: [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, + 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, + 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, + -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, + -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, + -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, + -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_flat + + - do: + index: + index: bbq_flat + id: "2" + body: + vector: [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, + -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, + 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, + -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, + -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, + -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, + 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, + -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_flat + + - do: + index: + index: bbq_flat + id: "3" + body: + vector: [0.139, 0.178, -0.117, 0.399, 0.014, -0.139, 0.347, -0.33 , + 0.139, 0.34 , -0.052, -0.052, -0.249, 0.327, -0.288, 0.049, + 0.464, 0.338, 0.516, 0.247, -0.104, 0.259, -0.209, -0.246, + -0.11 , 0.323, 0.091, 0.442, -0.254, 0.195, -0.109, -0.058, + -0.279, 0.402, -0.107, 0.308, -0.273, 0.019, 0.082, 0.399, + -0.658, -0.03 , 0.276, 0.041, 0.187, -0.331, 0.165, 0.017, + 0.171, -0.203, -0.198, 0.115, -0.007, 0.337, -0.444, 0.615, + -0.657, 1.285, 0.2 , -0.062, 0.038, 0.089, -0.068, -0.058] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_flat + + - do: + indices.forcemerge: + index: bbq_flat + max_num_segments: 1 +--- +"Test knn search": + - requires: + capabilities: + - method: POST + path: /_search + capabilities: [ optimized_scalar_quantization_bbq ] + test_runner_features: capabilities + reason: "BBQ scoring improved and changed with optimized_scalar_quantization_bbq" + - do: + search: + index: bbq_flat + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.2._id: "2" } +--- +"Vector rescoring has same scoring as exact search for kNN section": + - requires: + reason: 'Quantized vector rescoring is required' + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_search + capabilities: [knn_quantized_vector_rescore_oversample] + - skip: + features: "headers" + + # Rescore + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_flat + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17, + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + rescore_vector: + oversample: 1.5 + + # Get rescoring scores - hit ordering may change depending on how things are distributed + - match: { hits.total: 3 } + - set: { hits.hits.0._score: rescore_score0 } + - set: { hits.hits.1._score: rescore_score1 } + - set: { hits.hits.2._score: rescore_score2 } + + # Exact knn via script score + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_flat + body: + query: + script_score: + query: { match_all: {} } + script: + source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1" + params: + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17, + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $rescore_score0 } + - match: { hits.hits.1._score: $rescore_score1 } + - match: { hits.hits.2._score: $rescore_score2 } + +--- +"Test bad parameters": + - do: + catch: bad_request + indices.create: + index: bad_bbq_flat + body: + mappings: + properties: + vector: + type: dense_vector + element_type: bfloat16 + dims: 64 + index: true + index_options: + type: bbq_flat + m: 42 +--- +"Test bad raw vector size": + - do: + catch: bad_request + indices.create: + index: bad_bbq_flat + body: + mappings: + properties: + vector: + type: dense_vector + dims: 64 + index: true + index_options: + type: bbq_flat + raw_vector_size: 25 +--- +"Test few dimensions fail indexing": + # verify index creation fails + - do: + catch: bad_request + indices.create: + index: bad_bbq_flat + body: + mappings: + properties: + vector: + type: dense_vector + element_type: bfloat16 + dims: 42 + index: true + similarity: l2_norm + index_options: + type: bbq_flat + + # verify dynamic dimension fails + - do: + indices.create: + index: dynamic_dim_bbq_flat + body: + mappings: + properties: + vector: + type: dense_vector + element_type: bfloat16 + index: true + similarity: l2_norm + index_options: + type: bbq_flat + + # verify index fails for odd dim vector + - do: + catch: bad_request + index: + index: dynamic_dim_bbq_flat + body: + vector: [1.0, 2.0, 3.0, 4.0, 5.0] + + # verify that we can index an even dim vector after the odd dim vector failure + - do: + index: + index: dynamic_dim_bbq_flat + body: + vector: [1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0, 4.0] +--- +"Test index configured rescore vector": + - requires: + cluster_features: ["mapper.dense_vector.rescore_vector"] + reason: Needs rescore_vector feature + - skip: + features: "headers" + - do: + indices.create: + index: bbq_rescore_flat + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + element_type: bfloat16 + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: bbq_flat + rescore_vector: + oversample: 1.5 + + - do: + bulk: + index: bbq_rescore_flat + refresh: true + body: | + { "index": {"_id": "1"}} + { "vector": [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] } + { "index": {"_id": "2"}} + { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] } + { "index": {"_id": "3"}} + { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] } + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_rescore_flat + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + + - match: { hits.total: 3 } + - set: { hits.hits.0._score: rescore_score0 } + - set: { hits.hits.1._score: rescore_score1 } + - set: { hits.hits.2._score: rescore_score2 } + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_rescore_flat + body: + query: + script_score: + query: {match_all: {} } + script: + source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1" + params: + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $rescore_score0 } + - match: { hits.hits.1._score: $rescore_score1 } + - match: { hits.hits.2._score: $rescore_score2 } + +--- +"default oversample value": + - requires: + cluster_features: ["mapper.dense_vector.default_oversample_value_for_bbq"] + reason: "Needs default_oversample_value_for_bbq feature" + - do: + indices.get_mapping: + index: bbq_flat + + - match: { bbq_flat.mappings.properties.vector.index_options.rescore_vector.oversample: 3.0 } +--- +"Test nested queries": + - do: + indices.create: + index: bbq_flat_nested + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + name: + type: keyword + nested: + type: nested + properties: + paragraph_id: + type: keyword + vector: + type: dense_vector + element_type: bfloat16 + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: bbq_flat + + - do: + index: + index: bbq_flat_nested + id: "1" + body: + nested: + - paragraph_id: "1" + vector: [ 0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, + 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, + 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, + -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, + -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, + -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, + -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, + -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45 ] + - paragraph_id: "2" + vector: [ 0.7, 0.2 , 0.205, 0.63 , 0.032, 0.201, 0.167, 0.313, + 0.176, 0.1, 0.375, 0.334, 0.046, 0.078, 0.349, 0.272, + 0.307, 0.083, 0.504, 0.255, 0.404, 0.289, 0.226, 0.132, + 0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , 0.265, + 0.285, 0.336, 0.272, 0.369, -0.282, 0.086, 0.132, 0.475, + 0.224, 0.203, 0.439, 0.064, 0.246, 0.396, 0.297, 0.242, + 0.224, 0.203, 0.439, 0.064, 0.246, 0.396, 0.297, 0.242, + 0.028, 0.321, 0.022, 0.009, 0.001 , 0.031, -0.533, 0.45] + - do: + index: + index: bbq_flat_nested + id: "2" + body: + nested: + - paragraph_id: 0 + vector: [ 0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, + -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, + 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, + -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, + -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, + -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, + 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, + -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27, -0.013 ] + - paragraph_id: 2 + vector: [ 0.196, 0.514, 0.039, 0.555, 0.042, 0.242, 0.463, -0.348, + -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, + 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, 0.438, + -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, 0.138, + -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, + -0.602, 0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, 0.166, + 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, + -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27, 0.013 ] + - paragraph_id: 3 + vector: [ 0.196, 0.514, 0.039, 0.555, 0.042, 0.242, 0.463, -0.348, + 0.08 , 0.442, -0.067, -0.05 , 0.001, 0.298, -0.377, 0.048, + 0.307, 0.159, 0.278, 0.119, 0.057, 0.333, -0.289, -0.438, + -0.014, 0.361, -0.169, 0.292, 0.229, 0.123, 0.031, -0.138, + -0.139, 0.315, -0.216, 0.322, 0.445, -0.059, 0.071, 0.429, + -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, + 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, + -0.489, 0.901, 0.208, 0.011, 0.209, -0.153, -0.27, -0.013 ] + + - do: + index: + index: bbq_flat_nested + id: "3" + body: + nested: + - paragraph_id: 0 + vector: [ 0.139, 0.178, -0.117, 0.399, 0.014, -0.139, 0.347, -0.33 , + 0.139, 0.34 , -0.052, -0.052, -0.249, 0.327, -0.288, 0.049, + 0.464, 0.338, 0.516, 0.247, -0.104, 0.259, -0.209, -0.246, + -0.11 , 0.323, 0.091, 0.442, -0.254, 0.195, -0.109, -0.058, + -0.279, 0.402, -0.107, 0.308, -0.273, 0.019, 0.082, 0.399, + -0.658, -0.03 , 0.276, 0.041, 0.187, -0.331, 0.165, 0.017, + 0.171, -0.203, -0.198, 0.115, -0.007, 0.337, -0.444, 0.615, + -0.657, 1.285, 0.2 , -0.062, 0.038, 0.089, -0.068, -0.058 ] + + - do: + indices.flush: + index: bbq_flat_nested + + - do: + indices.forcemerge: + index: bbq_flat_nested + max_num_segments: 1 + + - do: + search: + index: bbq_flat_nested + body: + query: + nested: + path: nested + query: + knn: + field: nested.vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + num_candidates: 3 + k: 2 + + - match: {hits.hits.0._id: "3"} + + - do: + search: + index: bbq_flat_nested + body: + knn: + field: nested.vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + num_candidates: 3 + k: 2 + + - match: {hits.hits.0._id: "3"} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml index 4e0c9426c80b7..46d4e0ef3f57d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml @@ -180,6 +180,9 @@ setup: --- "KNN Vector similarity search only": + - requires: + cluster_features: [ "mapper.vectors.hnsw_bfloat16_on_disk_rescoring" ] + reason: "the quantized values, and so the effective similarity scores, changed with the new lucene quantizer" - do: search: index: int8_flat @@ -189,7 +192,7 @@ setup: num_candidates: 3 k: 3 field: vector - similarity: 10.3 + similarity: 10.7 query_vector: [-0.5, 90.0, -10, 14.8, -156.0] - length: {hits.hits: 1} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/46_knn_search_bbq_ivf_bfloat16.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/46_knn_search_bbq_ivf_bfloat16.yml new file mode 100644 index 0000000000000..aa59bceb00598 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/46_knn_search_bbq_ivf_bfloat16.yml @@ -0,0 +1,629 @@ +setup: + - requires: + cluster_features: ["mapper.vectors.hnsw_bfloat16_on_disk_rescoring"] + reason: 'bfloat16 needs to be supported' + - skip: + features: "headers" + - do: + indices.create: + index: bbq_disk + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + element_type: bfloat16 + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: bbq_disk + + - do: + index: + index: bbq_disk + id: "1" + body: + vector: [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, + 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, + 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, + -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, + -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, + -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, + -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_disk + + - do: + index: + index: bbq_disk + id: "2" + body: + vector: [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, + -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, + 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, + -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, + -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, + -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, + 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, + -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_disk + + - do: + index: + index: bbq_disk + id: "3" + body: + name: rabbit.jpg + vector: [0.139, 0.178, -0.117, 0.399, 0.014, -0.139, 0.347, -0.33 , + 0.139, 0.34 , -0.052, -0.052, -0.249, 0.327, -0.288, 0.049, + 0.464, 0.338, 0.516, 0.247, -0.104, 0.259, -0.209, -0.246, + -0.11 , 0.323, 0.091, 0.442, -0.254, 0.195, -0.109, -0.058, + -0.279, 0.402, -0.107, 0.308, -0.273, 0.019, 0.082, 0.399, + -0.658, -0.03 , 0.276, 0.041, 0.187, -0.331, 0.165, 0.017, + 0.171, -0.203, -0.198, 0.115, -0.007, 0.337, -0.444, 0.615, + -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: bbq_disk + + - do: + indices.forcemerge: + index: bbq_disk + max_num_segments: 1 + + - do: + indices.refresh: { } +--- +"Test knn search": + - do: + search: + index: bbq_disk + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.2._id: "2" } +--- +"Test knn search with visit_percentage": + - do: + search: + index: bbq_disk + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + visit_percentage: 1.0 + + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.2._id: "2" } +--- +"Vector rescoring has same scoring as exact search for kNN section": + - skip: + features: "headers" + + # Rescore + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_disk + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + rescore_vector: + oversample: 1.5 + + # Get rescoring scores - hit ordering may change depending on how things are distributed + - match: { hits.total: 3 } + - set: { hits.hits.0._score: rescore_score0 } + - set: { hits.hits.1._score: rescore_score1 } + - set: { hits.hits.2._score: rescore_score2 } + + # Exact knn via script score + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1" + params: + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $rescore_score0 } + - match: { hits.hits.1._score: $rescore_score1 } + - match: { hits.hits.2._score: $rescore_score2 } + +--- +"Test bad quantization parameters": + - do: + catch: bad_request + indices.create: + index: bad_bbq_ivf + body: + mappings: + properties: + vector: + type: dense_vector + dims: 64 + element_type: byte + index: true + index_options: + type: bbq_disk + + - do: + catch: bad_request + indices.create: + index: bad_bbq_ivf + body: + mappings: + properties: + vector: + type: dense_vector + dims: 64 + index: false + index_options: + type: bbq_disk +--- +"Test index configured rescore vector": + - skip: + features: "headers" + - do: + indices.create: + index: bbq_rescore_ivf + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: bbq_disk + rescore_vector: + oversample: 1.5 + + - do: + bulk: + index: bbq_rescore_ivf + refresh: true + body: | + { "index": {"_id": "1"}} + { "vector": [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] } + { "index": {"_id": "2"}} + { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] } + { "index": {"_id": "3"}} + { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] } + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_rescore_ivf + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + + - match: { hits.total: 3 } + - set: { hits.hits.0._score: rescore_score0 } + - set: { hits.hits.1._score: rescore_score1 } + - set: { hits.hits.2._score: rescore_score2 } + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_rescore_ivf + body: + query: + script_score: + query: {match_all: {} } + script: + source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1" + params: + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $rescore_score0 } + - match: { hits.hits.1._score: $rescore_score1 } + - match: { hits.hits.2._score: $rescore_score2 } + +--- +"Test index configured rescore vector with on-disk rescore": + - requires: + cluster_features: [ "mapper.vectors.diskbbq_on_disk_rescoring" ] + reason: Needs on_disk_rescoring feature for DiskBBQ + - skip: + features: "headers" + - do: + indices.create: + index: bbq_on_disk_rescore_ivf + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: bbq_disk + on_disk_rescore: true + rescore_vector: + oversample: 1.5 + + - do: + bulk: + index: bbq_on_disk_rescore_ivf + refresh: true + body: | + { "index": {"_id": "1"}} + { "vector": [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] } + { "index": {"_id": "2"}} + { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] } + { "index": {"_id": "3"}} + { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] } + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_on_disk_rescore_ivf + body: + knn: + field: vector + query_vector: [ 0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158 ] + k: 3 + num_candidates: 3 + + - match: { hits.total: 3 } + - set: { hits.hits.0._score: rescore_score0 } + - set: { hits.hits.1._score: rescore_score1 } + - set: { hits.hits.2._score: rescore_score2 } + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_on_disk_rescore_ivf + body: + query: + script_score: + query: { match_all: { } } + script: + source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1" + params: + query_vector: [ 0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158 ] + + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $rescore_score0 } + - match: { hits.hits.1._score: $rescore_score1 } + - match: { hits.hits.2._score: $rescore_score2 } +--- +"Test index configured rescore vector updateable and settable to 0": + - do: + indices.create: + index: bbq_rescore_0_ivf + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + index_options: + type: bbq_disk + rescore_vector: + oversample: 0 + + - do: + indices.create: + index: bbq_rescore_update_ivf + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + index_options: + type: bbq_disk + rescore_vector: + oversample: 1 + + - do: + indices.put_mapping: + index: bbq_rescore_update_ivf + body: + properties: + vector: + type: dense_vector + index_options: + type: bbq_disk + rescore_vector: + oversample: 0 + + - do: + indices.get_mapping: + index: bbq_rescore_update_ivf + + - match: { .bbq_rescore_update_ivf.mappings.properties.vector.index_options.rescore_vector.oversample: 0 } +--- +"Test index configured rescore vector score consistency": + - skip: + features: "headers" + - do: + indices.create: + index: bbq_rescore_zero_ivf + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + vector: + type: dense_vector + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: bbq_disk + rescore_vector: + oversample: 0 + + - do: + bulk: + index: bbq_rescore_zero_ivf + refresh: true + body: | + { "index": {"_id": "1"}} + { "vector": [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] } + { "index": {"_id": "2"}} + { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] } + { "index": {"_id": "3"}} + { "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] } + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_rescore_zero_ivf + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + + - match: { hits.total: 3 } + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_rescore_zero_ivf + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + rescore_vector: + oversample: 2 + + - match: { hits.total: 3 } + - set: { hits.hits.0._score: override_score0 } + - set: { hits.hits.1._score: override_score1 } + - set: { hits.hits.2._score: override_score2 } + + - do: + indices.put_mapping: + index: bbq_rescore_zero_ivf + body: + properties: + vector: + type: dense_vector + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: bbq_disk + rescore_vector: + oversample: 2 + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_rescore_zero_ivf + body: + knn: + field: vector + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + k: 3 + num_candidates: 3 + + - match: { hits.total: 3 } + - set: { hits.hits.0._score: default_rescore0 } + - set: { hits.hits.1._score: default_rescore1 } + - set: { hits.hits.2._score: default_rescore2 } + + - do: + indices.put_mapping: + index: bbq_rescore_zero_ivf + body: + properties: + vector: + type: dense_vector + dims: 64 + index: true + similarity: max_inner_product + index_options: + type: bbq_disk + rescore_vector: + oversample: 0 + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + index: bbq_rescore_zero_ivf + body: + query: + script_score: + query: {match_all: {} } + script: + source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1" + params: + query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393, + 0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015, + 0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259, + -0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 , + -0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232, + -0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034, + -0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582, + -0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158] + + # Compare scores as hit IDs may change depending on how things are distributed + - match: { hits.total: 3 } + - match: { hits.hits.0._score: $override_score0 } + - match: { hits.hits.0._score: $default_rescore0 } + - match: { hits.hits.1._score: $override_score1 } + - match: { hits.hits.1._score: $default_rescore1 } + - match: { hits.hits.2._score: $override_score2 } + - match: { hits.hits.2._score: $default_rescore2 } + +--- +"default oversample value": + - do: + indices.get_mapping: + index: bbq_disk + + - match: { bbq_disk.mappings.properties.vector.index_options.rescore_vector.oversample: 3.0 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml index c1fdb8adc8ee9..48f594f8b2cf2 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml @@ -236,8 +236,8 @@ dfs knn vector profiling: --- dfs knn vector profiling description: - requires: - cluster_features: ["lucene_10_upgrade"] - reason: "the profile description changed with Lucene 10" + cluster_features: ["mapper.vectors.hnsw_bfloat16_on_disk_rescoring"] + reason: "the quantized values changed with the new lucene quantizer" - do: indices.create: index: images @@ -272,7 +272,7 @@ dfs knn vector profiling description: num_candidates: 100 - match: { hits.total.value: 1 } - - match: { profile.shards.0.dfs.knn.0.query.0.description: "DocAndScoreQuery[0,...][0.009673266,...],0.009673266" } + - match: { profile.shards.0.dfs.knn.0.query.0.description: "DocAndScoreQuery[0,...][0.008547009,...],0.008547009" } --- dfs knn vector profiling collector name: diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java index 654a1393aa146..0e25b1071a271 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java @@ -25,6 +25,7 @@ import java.util.List; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.executeResize; import static org.elasticsearch.action.admin.indices.create.ShrinkIndexIT.assertNoResizeSourceIndexSettings; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; @@ -67,11 +68,12 @@ public void testCreateCloneIndex() { final boolean createWithReplicas = randomBoolean(); assertAcked( - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.CLONE) - .setSettings( - Settings.builder().put("index.number_of_replicas", createWithReplicas ? 1 : 0).putNull("index.blocks.write").build() - ) + executeResize( + ResizeType.CLONE, + "source", + "target", + Settings.builder().put("index.number_of_replicas", createWithReplicas ? 1 : 0).putNull("index.blocks.write") + ) ); ensureGreen(); assertNoResizeSourceIndexSettings("target"); @@ -125,9 +127,10 @@ public void testResizeChangeIndexMode() { Settings.builder().put("index.mode", "lookup").build() ); for (Settings settings : indexSettings) { - IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> { - indicesAdmin().prepareResizeIndex("source", "target").setResizeType(ResizeType.CLONE).setSettings(settings).get(); - }); + IllegalArgumentException error = expectThrows( + IllegalArgumentException.class, + () -> executeResize(ResizeType.CLONE, "source", "target", Settings.builder().put(settings)).actionGet() + ); assertThat(error.getMessage(), equalTo("can't change setting [index.mode] during resize")); } } @@ -137,12 +140,15 @@ public void testResizeChangeSyntheticSource() { .setMapping("@timestamp", "type=date", "host.name", "type=keyword") .get(); updateIndexSettings(Settings.builder().put("index.blocks.write", true), "source"); - IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> { - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.CLONE) - .setSettings(Settings.builder().put("index.mapping.source.mode", "synthetic").putNull("index.blocks.write").build()) - .get(); - }); + IllegalArgumentException error = expectThrows( + IllegalArgumentException.class, + () -> executeResize( + ResizeType.CLONE, + "source", + "target", + Settings.builder().put("index.mapping.source.mode", "synthetic").putNull("index.blocks.write") + ).actionGet() + ); assertThat(error.getMessage(), containsString("can't change setting [index.mapping.source.mode] during resize")); } @@ -159,26 +165,26 @@ public void testResizeChangeRecoveryUseSyntheticSource() { ) ).setMapping("@timestamp", "type=date", "host.name", "type=keyword").get(); updateIndexSettings(Settings.builder().put("index.blocks.write", true), "source"); - IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> { - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.CLONE) - .setSettings( - Settings.builder() - .put( - "index.version.created", - IndexVersionUtils.randomVersionBetween( - random(), - IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY, - IndexVersion.current() - ) + IllegalArgumentException error = expectThrows( + IllegalArgumentException.class, + () -> executeResize( + ResizeType.CLONE, + "source", + "target", + Settings.builder() + .put( + "index.version.created", + IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY, + IndexVersion.current() ) - .put("index.recovery.use_synthetic_source", true) - .put("index.mode", "logsdb") - .putNull("index.blocks.write") - .build() - ) - .get(); - }); + ) + .put("index.recovery.use_synthetic_source", true) + .put("index.mode", "logsdb") + .putNull("index.blocks.write") + ).actionGet() + ); // The index.recovery.use_synthetic_source setting requires either index.mode or index.mapping.source.mode // to be present in the settings. Since these are all unmodifiable settings with a non-deterministic evaluation // order, any of them may trigger a failure first. @@ -196,12 +202,11 @@ public void testResizeChangeIndexSorts() { .setMapping("@timestamp", "type=date", "host.name", "type=keyword") .get(); updateIndexSettings(Settings.builder().put("index.blocks.write", true), "source"); - ValidationException error = expectThrows(ValidationException.class, () -> { - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.CLONE) - .setSettings(Settings.builder().putList("index.sort.field", List.of("@timestamp")).build()) - .get(); - }); + ValidationException error = expectThrows( + ValidationException.class, + () -> executeResize(ResizeType.CLONE, "source", "target", Settings.builder().putList("index.sort.field", List.of("@timestamp"))) + .actionGet() + ); assertThat(error.getMessage(), containsString("can't override index sort when resizing an index")); } @@ -216,11 +221,13 @@ public void testCloneLogsdbIndexWithNonDefaultTimestamp() { ensureGreen(); // Clone the index - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.CLONE) + executeResize( + ResizeType.CLONE, + "source", + "target", // We need to explicitly set the number of replicas in case the source has 0 replicas and the cluster has only 1 data node - .setSettings(Settings.builder().put("index.number_of_replicas", numberOfReplicas).build()) - .get(); + Settings.builder().put("index.number_of_replicas", numberOfReplicas) + ).actionGet(); // Verify that the target index has the correct @timestamp mapping final var targetMappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "target").get(); @@ -246,11 +253,13 @@ public void testCloneTimeSeriesIndexWithNonDefaultTimestamp() { ensureGreen(); // Clone the index - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.CLONE) + executeResize( + ResizeType.CLONE, + "source", + "target", // We need to explicitly set the number of replicas in case the source has 0 replicas and the cluster has only 1 data node - .setSettings(Settings.builder().put("index.number_of_replicas", numberOfReplicas).build()) - .get(); + Settings.builder().put("index.number_of_replicas", numberOfReplicas) + ).actionGet(); // Verify that the target index has the correct @timestamp mapping final var targetMappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "target").get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java index 31831f7eb6fe5..75f6a949c976d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java @@ -18,11 +18,13 @@ import org.elasticsearch.action.admin.cluster.reroute.TransportClusterRerouteAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.indices.ResizeIndexTestUtils; import org.elasticsearch.action.admin.indices.segments.IndexShardSegments; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse; import org.elasticsearch.action.admin.indices.segments.ShardSegments; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; @@ -58,6 +60,7 @@ import java.util.Map; import java.util.stream.IntStream; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.executeResize; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.containsString; @@ -102,8 +105,7 @@ public void testCreateShrinkIndexToN() { ensureGreen(); // now merge source into a 4 shard index assertAcked( - indicesAdmin().prepareResizeIndex("source", "first_shrink") - .setSettings(indexSettings(shardSplits[1], 0).putNull("index.blocks.write").build()) + executeResize(ResizeType.SHRINK, "source", "first_shrink", indexSettings(shardSplits[1], 0).putNull("index.blocks.write")) ); ensureGreen(); assertHitCount(prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), 20); @@ -128,10 +130,12 @@ public void testCreateShrinkIndexToN() { ensureGreen(); // now merge source into a 2 shard index assertAcked( - indicesAdmin().prepareResizeIndex("first_shrink", "second_shrink") - .setSettings( - indexSettings(shardSplits[2], 0).putNull("index.blocks.write").putNull("index.routing.allocation.require._name").build() - ) + executeResize( + ResizeType.SHRINK, + "first_shrink", + "second_shrink", + indexSettings(shardSplits[2], 0).putNull("index.blocks.write").putNull("index.routing.allocation.require._name") + ) ); ensureGreen(); assertHitCount(prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), 20); @@ -221,8 +225,7 @@ public void testShrinkIndexPrimaryTerm() throws Exception { final long beforeShrinkPrimaryTerm = IntStream.range(0, numberOfShards).mapToLong(indexMetadata::primaryTerm).max().getAsLong(); // now merge source into target - final Settings shrinkSettings = indexSettings(numberOfTargetShards, 0).build(); - assertAcked(indicesAdmin().prepareResizeIndex("source", "target").setSettings(shrinkSettings).get()); + assertAcked(executeResize(ResizeType.SHRINK, "source", "target", indexSettings(numberOfTargetShards, 0))); ensureGreen(TimeValue.timeValueSeconds(120)); @@ -273,15 +276,17 @@ public void testCreateShrinkIndex() { // now merge source into a single shard index final boolean createWithReplicas = randomBoolean(); assertAcked( - indicesAdmin().prepareResizeIndex("source", "target") - .setSettings( - Settings.builder() - .put("index.number_of_replicas", createWithReplicas ? 1 : 0) - .putNull("index.blocks.write") - .putNull("index.routing.allocation.require._name") - .build() - ) + executeResize( + ResizeType.SHRINK, + "source", + "target", + Settings.builder() + .put("index.number_of_replicas", createWithReplicas ? 1 : 0) + .putNull("index.blocks.write") + .putNull("index.routing.allocation.require._name") + ) ); + ensureGreen(); assertNoResizeSourceIndexSettings("target"); @@ -374,16 +379,17 @@ public void testCreateShrinkIndexFails() throws Exception { ensureGreen(); // now merge source into a single shard index - indicesAdmin().prepareResizeIndex("source", "target") - .setWaitForActiveShards(ActiveShardCount.NONE) - .setSettings( - Settings.builder() - .put("index.routing.allocation.exclude._name", mergeNode) // we manually exclude the merge node to forcefully fuck it up - .put("index.number_of_replicas", 0) - .put("index.allocation.max_retries", 1) - .build() - ) - .get(); + final var resizeRequest = ResizeIndexTestUtils.resizeRequest( + ResizeType.SHRINK, + "source", + "target", + Settings.builder() + .put("index.routing.allocation.exclude._name", mergeNode) // we manually exclude the merge node to forcefully mess it up + .put("index.number_of_replicas", 0) + .put("index.allocation.max_retries", 1) + ); + resizeRequest.setWaitForActiveShards(ActiveShardCount.NONE); + client().execute(TransportResizeAction.TYPE, resizeRequest).actionGet(); clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT, "target").setWaitForEvents(Priority.LANGUID).get(); // now we move all shards away from the merge node @@ -464,14 +470,12 @@ public void testCreateShrinkWithIndexSort() throws Exception { // check that index sort cannot be set on the target index IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, - indicesAdmin().prepareResizeIndex("source", "target").setSettings(indexSettings(2, 0).put("index.sort.field", "foo").build()) + executeResize(ResizeType.SHRINK, "source", "target", indexSettings(2, 0).put("index.sort.field", "foo")) ); assertThat(exc.getMessage(), containsString("can't override index sort when resizing an index")); // check that the index sort order of `source` is correctly applied to the `target` - assertAcked( - indicesAdmin().prepareResizeIndex("source", "target").setSettings(indexSettings(2, 0).putNull("index.blocks.write").build()) - ); + assertAcked(executeResize(ResizeType.SHRINK, "source", "target", indexSettings(2, 0).putNull("index.blocks.write"))); ensureGreen(); assertNoResizeSourceIndexSettings("target"); @@ -516,10 +520,7 @@ public void testShrinkCommitsMergeOnIdle() throws Exception { updateClusterSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none")); try { // now merge source into a single shard index - assertAcked( - indicesAdmin().prepareResizeIndex("source", "target") - .setSettings(Settings.builder().put("index.number_of_replicas", 0).build()) - ); + assertAcked(executeResize(ResizeType.SHRINK, "source", "target", Settings.builder().put("index.number_of_replicas", 0))); ensureGreen(); assertNoResizeSourceIndexSettings("target"); @@ -584,13 +585,14 @@ public void testShrinkThenSplitWithFailedNode() throws Exception { ensureGreen(); assertAcked( - indicesAdmin().prepareResizeIndex("original", "shrunk") - .setSettings( - indexSettings(1, 1).putNull( - IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey() - ).build() + executeResize( + ResizeType.SHRINK, + "original", + "shrunk", + indexSettings(1, 1).putNull( + IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey() ) - .setResizeType(ResizeType.SHRINK) + ) ); ensureGreen(); @@ -603,13 +605,14 @@ public void testShrinkThenSplitWithFailedNode() throws Exception { logger.info("--> executing split"); assertAcked( - indicesAdmin().prepareResizeIndex("shrunk", "splitagain") - .setSettings( - indexSettings(shardCount, 0).putNull( - IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey() - ).build() + executeResize( + ResizeType.SPLIT, + "shrunk", + "splitagain", + indexSettings(shardCount, 0).putNull( + IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey() ) - .setResizeType(ResizeType.SPLIT) + ) ); ensureGreen("splitagain"); assertNoResizeSourceIndexSettings("splitagain"); @@ -627,11 +630,13 @@ public void testShrinkLogsdbIndexWithNonDefaultTimestamp() { ensureGreen(); // Shrink the index - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.SHRINK) + executeResize( + ResizeType.SHRINK, + "source", + "target", // We need to explicitly set the number of replicas in case the source has 0 replicas and the cluster has only 1 data node - .setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()) - .get(); + indexSettings(1, 0) + ).actionGet(); // Verify that the target index has the correct @timestamp mapping final var targetMappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "target").get(); @@ -657,11 +662,13 @@ public void testShrinkTimeSeriesIndexWithNonDefaultTimestamp() { ensureGreen(); // Shrink the index - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.SHRINK) + executeResize( + ResizeType.SHRINK, + "source", + "target", // We need to explicitly set the number of replicas in case the source has 0 replicas and the cluster has only 1 data node - .setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()) - .get(); + indexSettings(1, 0) + ).actionGet(); // Verify that the target index has the correct @timestamp mapping final var targetMappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "target").get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java index 5055430310703..ffa479bfe1ff2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java @@ -57,6 +57,7 @@ import java.util.function.BiFunction; import java.util.stream.IntStream; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.executeResize; import static org.elasticsearch.action.admin.indices.create.ShrinkIndexIT.assertNoResizeSourceIndexSettings; import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; @@ -184,11 +185,7 @@ private void splitToN(int sourceShards, int firstSplitShards, int secondSplitSha if (sourceShards == 1 && useRoutingPartition == false && randomBoolean()) { // try to set it if we have a source index with 1 shard firstSplitSettingsBuilder.put("index.number_of_routing_shards", secondSplitShards); } - assertAcked( - indicesAdmin().prepareResizeIndex("source", "first_split") - .setResizeType(ResizeType.SPLIT) - .setSettings(firstSplitSettingsBuilder.build()) - ); + assertAcked(executeResize(ResizeType.SPLIT, "source", "first_split", firstSplitSettingsBuilder)); ensureGreen(); assertHitCount(prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), numDocs); assertNoResizeSourceIndexSettings("first_split"); @@ -212,9 +209,12 @@ private void splitToN(int sourceShards, int firstSplitShards, int secondSplitSha ensureGreen(); // now split source into a new index assertAcked( - indicesAdmin().prepareResizeIndex("first_split", "second_split") - .setResizeType(ResizeType.SPLIT) - .setSettings(indexSettings(secondSplitShards, 0).putNull("index.blocks.write").build()) + executeResize( + ResizeType.SPLIT, + "first_split", + "second_split", + indexSettings(secondSplitShards, 0).putNull("index.blocks.write") + ) ); ensureGreen(); assertHitCount(prepareSearch("second_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), numDocs); @@ -325,8 +325,9 @@ public void testSplitIndexPrimaryTerm() throws Exception { final long beforeSplitPrimaryTerm = IntStream.range(0, numberOfShards).mapToLong(indexMetadata::primaryTerm).max().getAsLong(); // now split source into target - final Settings splitSettings = indexSettings(numberOfTargetShards, 0).putNull("index.blocks.write").build(); - assertAcked(indicesAdmin().prepareResizeIndex("source", "target").setResizeType(ResizeType.SPLIT).setSettings(splitSettings).get()); + assertAcked( + executeResize(ResizeType.SPLIT, "source", "target", indexSettings(numberOfTargetShards, 0).putNull("index.blocks.write")) + ); ensureGreen(TimeValue.timeValueSeconds(120)); // needs more than the default to relocate many shards @@ -372,9 +373,12 @@ public void testCreateSplitIndex() { final boolean createWithReplicas = randomBoolean(); assertAcked( - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.SPLIT) - .setSettings(indexSettings(2, createWithReplicas ? 1 : 0).putNull("index.blocks.write").build()) + executeResize( + ResizeType.SPLIT, + "source", + "target", + indexSettings(2, createWithReplicas ? 1 : 0).putNull("index.blocks.write") + ) ); ensureGreen(); assertNoResizeSourceIndexSettings("target"); @@ -470,18 +474,12 @@ public void testCreateSplitWithIndexSort() throws Exception { // check that index sort cannot be set on the target index IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.SPLIT) - .setSettings(indexSettings(4, 0).put("index.sort.field", "foo").build()) + executeResize(ResizeType.SPLIT, "source", "target", indexSettings(4, 0).put("index.sort.field", "foo")) ); assertThat(exc.getMessage(), containsString("can't override index sort when resizing an index")); // check that the index sort order of `source` is correctly applied to the `target` - assertAcked( - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.SPLIT) - .setSettings(indexSettings(4, 0).putNull("index.blocks.write").build()) - ); + assertAcked(executeResize(ResizeType.SPLIT, "source", "target", indexSettings(4, 0).putNull("index.blocks.write"))); ensureGreen(); flushAndRefresh(); GetSettingsResponse settingsResponse = indicesAdmin().prepareGetSettings(TEST_REQUEST_TIMEOUT, "target").get(); @@ -510,11 +508,13 @@ public void testSplitLogsdbIndexWithNonDefaultTimestamp() { ensureGreen(); // Split the index - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.SPLIT) + executeResize( + ResizeType.SPLIT, + "source", + "target", // We need to explicitly set the number of replicas in case the source has 0 replicas and the cluster has only 1 data node - .setSettings(Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", numberOfReplicas).build()) - .get(); + indexSettings(2, numberOfReplicas) + ).actionGet(); // Verify that the target index has the correct @timestamp mapping final var targetMappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "target").get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java index 8918d995646b8..560e01187febc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java @@ -12,9 +12,10 @@ import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.action.admin.indices.ResizeIndexTestUtils; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.ActiveShardCount; @@ -23,7 +24,6 @@ import org.elasticsearch.cluster.ClusterInfoServiceUtils; import org.elasticsearch.cluster.DiskUsageIntegTestCase; import org.elasticsearch.cluster.InternalClusterInfoService; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; @@ -143,18 +143,16 @@ public void testAllocateCloneIgnoresLowWatermark() throws Exception { refreshDiskUsage(); final var targetIndexName = "target-" + randomIdentifier(); - final var resizeRequest = new ResizeRequest(targetIndexName, sourceIndexName); - resizeRequest.setResizeType(ResizeType.CLONE); - resizeRequest.masterNodeTimeout(TEST_REQUEST_TIMEOUT); - resizeRequest.ackTimeout(TEST_REQUEST_TIMEOUT); + final var resizeRequest = ResizeIndexTestUtils.resizeRequest( + ResizeType.CLONE, + sourceIndexName, + targetIndexName, + indexSettings(1, 0) + ); resizeRequest.setWaitForActiveShards(ActiveShardCount.ALL); - resizeRequest.getTargetIndexRequest() - .settings( - Settings.builder().put(resizeRequest.getTargetIndexRequest().settings()).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - ); safeAwait( - SubscribableListener.newForked(l -> indicesAdmin().resizeIndex(resizeRequest, l)) + SubscribableListener.newForked(l -> client().execute(TransportResizeAction.TYPE, resizeRequest, l)) .andThenAccept( createIndexResponse -> assertThat( true, diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceIT.java index e3586f330a32e..dda581d15a9e0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.cluster.routing.allocation.shards; +import org.elasticsearch.action.admin.indices.ResizeIndexTestUtils; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterStateListener; @@ -109,7 +110,7 @@ public void testIsGreenDuringIndexClone() { updateIndexSettings(Settings.builder().put("index.blocks.write", true), sourceIndex); assertHealthDuring(equalTo(GREEN), () -> { - indicesAdmin().prepareResizeIndex(sourceIndex, targetIndex).setResizeType(ResizeType.CLONE).get(); + ResizeIndexTestUtils.executeResize(ResizeType.CLONE, sourceIndex, targetIndex, Settings.builder()).actionGet(); ensureGreen(targetIndex); }); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java index 583ab18d4f8b3..aec61b877766e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java @@ -9,11 +9,12 @@ package org.elasticsearch.index; +import org.elasticsearch.action.admin.indices.ResizeIndexTestUtils; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; -import org.elasticsearch.action.admin.indices.shrink.ResizeAction; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponse; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; import org.elasticsearch.action.search.SearchResponse; @@ -140,9 +141,7 @@ public void testResizeLookupIndex() { assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createIndexRequest)); client().admin().indices().prepareAddBlock(IndexMetadata.APIBlock.WRITE, "lookup-1").get(); - ResizeRequest clone = new ResizeRequest("lookup-2", "lookup-1"); - clone.setResizeType(ResizeType.CLONE); - assertAcked(client().admin().indices().execute(ResizeAction.INSTANCE, clone).actionGet()); + assertAcked(ResizeIndexTestUtils.executeResize(ResizeType.CLONE, "lookup-1", "lookup-2", Settings.builder())); Settings settings = client().admin() .indices() .prepareGetSettings(TEST_REQUEST_TIMEOUT, "lookup-2") @@ -152,12 +151,15 @@ public void testResizeLookupIndex() { assertThat(settings.get("index.mode"), equalTo("lookup")); assertThat(settings.get("index.number_of_shards"), equalTo("1")); - ResizeRequest split = new ResizeRequest("lookup-3", "lookup-1"); - split.setResizeType(ResizeType.SPLIT); - split.getTargetIndexRequest().settings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3)); + ResizeRequest split = ResizeIndexTestUtils.resizeRequest( + ResizeType.SPLIT, + "lookup-1", + "lookup-3", + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3) + ); IllegalArgumentException error = expectThrows( IllegalArgumentException.class, - () -> client().admin().indices().execute(ResizeAction.INSTANCE, split).actionGet() + () -> client().execute(TransportResizeAction.TYPE, split).actionGet() ); assertThat( error.getMessage(), @@ -186,27 +188,29 @@ public void testResizeRegularIndexToLookup() { .setSettings(Settings.builder().put("index.number_of_replicas", 0)) .get(); - ResizeRequest clone = new ResizeRequest("lookup-3", "regular-1"); - clone.setResizeType(ResizeType.CLONE); - clone.getTargetIndexRequest().settings(Settings.builder().put("index.mode", "lookup")); + ResizeRequest clone = ResizeIndexTestUtils.resizeRequest( + ResizeType.CLONE, + "regular-1", + "lookup-3", + Settings.builder().put("index.mode", "lookup") + ); IllegalArgumentException error = expectThrows( IllegalArgumentException.class, - () -> client().admin().indices().execute(ResizeAction.INSTANCE, clone).actionGet() + () -> client().execute(TransportResizeAction.TYPE, clone).actionGet() ); assertThat( error.getMessage(), equalTo("index with [lookup] mode must have [index.number_of_shards] set to 1 or unset; provided 2") ); - ResizeRequest shrink = new ResizeRequest("lookup-4", "regular-1"); - shrink.setResizeType(ResizeType.SHRINK); - shrink.getTargetIndexRequest() - .settings(Settings.builder().put("index.mode", "lookup").put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)); - - error = expectThrows( - IllegalArgumentException.class, - () -> client().admin().indices().execute(ResizeAction.INSTANCE, shrink).actionGet() + ResizeRequest shrink = ResizeIndexTestUtils.resizeRequest( + ResizeType.SHRINK, + "regular-1", + "lookup-4", + Settings.builder().put("index.mode", "lookup").put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) ); + + error = expectThrows(IllegalArgumentException.class, () -> client().execute(TransportResizeAction.TYPE, shrink).actionGet()); assertThat(error.getMessage(), equalTo("can't change setting [index.mode] during resize")); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/store/DirectIOIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/store/DirectIOIT.java index efbc19b30079c..c8ceb200ea0b3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/store/DirectIOIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/store/DirectIOIT.java @@ -73,7 +73,7 @@ protected boolean useDirectIO(String name, IOContext context, OptionalLong fileL @ParametersFactory public static Iterable parameters() { - return List.of(new Object[] { "bbq_disk" }); + return List.of(new Object[] { "bbq_hnsw" }, new Object[] { "bbq_disk" }); } public DirectIOIT(String type) { @@ -113,15 +113,14 @@ private String indexVectors(boolean directIO) { indexDoc(indexName, Integer.toString(i), "fooVector", IntStream.range(0, 64).mapToDouble(d -> randomFloat()).toArray()); } refresh(); - assertBBQIndexType(indexName, type); // test assertion to ensure that the correct index type is being used + assertIndexType(indexName, type); // test assertion to ensure that the correct index type is being used return indexName; } - @SuppressWarnings("unchecked") - static void assertBBQIndexType(String indexName, String type) { + static void assertIndexType(String indexName, String type) { var response = indicesAdmin().prepareGetFieldMappings(indexName).setFields("fooVector").get(); - var map = (Map) response.fieldMappings(indexName, "fooVector").sourceAsMap().get("fooVector"); - assertThat((String) ((Map) map.get("index_options")).get("type"), is(equalTo(type))); + var map = (Map) response.fieldMappings(indexName, "fooVector").sourceAsMap().get("fooVector"); + assertThat(((Map) map.get("index_options")).get("type"), is(equalTo(type))); } @TestLogging(value = "org.elasticsearch.index.store.FsDirectoryFactory:DEBUG", reason = "to capture trace logging for direct IO") diff --git a/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java index 68bc6656cec7f..adda57d8d290d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java @@ -10,6 +10,7 @@ package org.elasticsearch.routing; import org.apache.lucene.util.Constants; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.IndexScopedSettings; @@ -23,6 +24,7 @@ import java.util.Map; import java.util.Set; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.executeResize; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.hamcrest.CoreMatchers.containsString; @@ -117,9 +119,12 @@ public void testShrinking() throws Exception { index = "index_" + currentShards; logger.info("--> shrinking index [" + previousIndex + "] to [" + index + "]"); - indicesAdmin().prepareResizeIndex(previousIndex, index) - .setSettings(indexSettings(currentShards, numberOfReplicas()).putNull("index.routing.allocation.require._name").build()) - .get(); + executeResize( + ResizeType.SHRINK, + previousIndex, + index, + indexSettings(currentShards, numberOfReplicas()).putNull("index.routing.allocation.require._name") + ).actionGet(); ensureGreen(); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java index 650fad8a496ad..25fe5bdb222cb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java @@ -11,8 +11,11 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.TransportVersion; +import org.elasticsearch.action.ResolvedIndexExpression; +import org.elasticsearch.action.ResolvedIndexExpressions; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.shard.IllegalIndexShardStateException; @@ -22,11 +25,13 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Map; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; @@ -34,6 +39,7 @@ import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; public class CCSFieldCapabilitiesIT extends AbstractMultiClustersTestCase { @@ -268,6 +274,138 @@ public void testReturnAllLocal() { } } + public void testResolvedToMatchingEverywhere() { + String localIndex = "index-local"; + String remoteIndex = "index-remote"; + String remoteClusterAlias = "remote_cluster"; + populateIndices(localIndex, remoteIndex, remoteClusterAlias, false); + String remoteIndexWithCluster = String.join(":", remoteClusterAlias, remoteIndex); + FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndexWithCluster) + .setFields("*") + .setIncludeResolvedTo(true) + .get(); + + assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex, remoteIndexWithCluster)); + + ResolvedIndexExpressions local = response.getResolvedLocally(); + assertThat(local, notNullValue()); + assertThat(local.expressions(), hasSize(1)); + assertEquals( + local.expressions().get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + ); + + List localIndicesList = local.getLocalIndicesList(); + assertThat(localIndicesList, hasSize(1)); + assertThat(localIndicesList, containsInAnyOrder(localIndex)); + + Map remote = response.getResolvedRemotely(); + assertThat(remote, notNullValue()); + assertThat(remote, aMapWithSize(1)); + assertThat(remote.keySet(), contains(remoteClusterAlias)); + + ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias); + List remoteIndicesList = remoteResponse.getLocalIndicesList(); + assertThat(remoteIndicesList, hasSize(1)); + assertEquals( + remoteResponse.expressions().get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + ); + assertThat(remoteIndicesList, containsInAnyOrder(remoteIndex)); + } + + public void testResolvedToMatchingLocallyOnly() { + String localIndex = "index-local"; + String remoteIndex = "index-remote"; + String remoteClusterAlias = "remote_cluster"; + String nonExistentIndex = "non-existent-index"; + populateIndices(localIndex, remoteIndex, remoteClusterAlias, false); + String remoteIndexWithCluster = String.join(":", remoteClusterAlias, nonExistentIndex); + FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndexWithCluster) + .setFields("*") + .setIncludeResolvedTo(true) + .get(); + + assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex)); + + ResolvedIndexExpressions local = response.getResolvedLocally(); + assertThat(local, notNullValue()); + assertThat(local.expressions(), hasSize(1)); + assertEquals( + local.expressions().get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + ); + + List localIndicesList = local.getLocalIndicesList(); + assertThat(localIndicesList, hasSize(1)); + assertThat(localIndicesList, containsInAnyOrder(localIndex)); + + Map remote = response.getResolvedRemotely(); + assertThat(remote, notNullValue()); + assertThat(remote, aMapWithSize(1)); + assertThat(remote.keySet(), contains(remoteClusterAlias)); + + ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias); + List remoteIndicesList = remoteResponse.getLocalIndicesList(); + assertThat(remoteIndicesList, hasSize(0)); + List remoteResolvedExpressions = remoteResponse.expressions(); + assertEquals(1, remoteResolvedExpressions.size()); + assertEquals( + remoteResolvedExpressions.get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE + ); + assertEquals(0, remoteIndicesList.size()); + } + + public void testResolvedToMatchingRemotelyOnly() { + String localIndex = "index-local"; + String remoteIndex = "index-remote"; + String remoteClusterAlias = "remote_cluster"; + String nonExistentIndex = "non-existent-index"; + populateIndices(localIndex, remoteIndex, remoteClusterAlias, false); + String remoteIndexWithCluster = String.join(":", remoteClusterAlias, remoteIndex); + boolean ignoreUnavailable = true; + IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false); + + FieldCapabilitiesResponse response = client().prepareFieldCaps(nonExistentIndex, remoteIndexWithCluster) + .setFields("*") + .setIncludeResolvedTo(true) + .setIndicesOptions(options) // without ignore unavaliable would throw error + .get(); + + assertThat(response.getIndices(), arrayContainingInAnyOrder(remoteIndexWithCluster)); + + ResolvedIndexExpressions local = response.getResolvedLocally(); + assertThat(local, notNullValue()); + assertThat(local.expressions(), hasSize(1)); + assertEquals( + local.expressions().get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE + ); + + List localIndicesList = local.getLocalIndicesList(); + assertThat(localIndicesList, hasSize(0)); + + Map remote = response.getResolvedRemotely(); + assertThat(remote, notNullValue()); + assertThat(remote, aMapWithSize(1)); + assertThat(remote.keySet(), contains(remoteClusterAlias)); + + ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias); + List remoteIndicesList = remoteResponse.getLocalIndicesList(); + assertThat(remoteIndicesList, hasSize(1)); + assertThat(remoteIndicesList, containsInAnyOrder(remoteIndex)); + List remoteResolvedExpressions = remoteResponse.expressions(); + assertEquals(1, remoteResolvedExpressions.size()); + ResolvedIndexExpression remoteExpression = remoteResolvedExpressions.get(0); + assertEquals( + remoteExpression.localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + ); + assertEquals(1, remoteExpression.localExpressions().indices().size()); + assertEquals(remoteIndex, remoteResolvedExpressions.get(0).original()); + } + public void testIncludesMinTransportVersion() { if (randomBoolean()) { assertAcked(client().admin().indices().prepareCreate("index")); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index b9fcec556213a..91a990d326e54 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -13,6 +13,8 @@ import org.apache.http.entity.StringEntity; import org.apache.logging.log4j.Level; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ResolvedIndexExpression; +import org.elasticsearch.action.ResolvedIndexExpressions; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; @@ -102,6 +104,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; @@ -913,6 +916,170 @@ public void testIndexMode() throws Exception { assertThat(actualIndexModes, equalTo(indexModes)); } + public void testResolvedExpressionWithIndexAlias() { + FieldCapabilitiesResponse response = client().prepareFieldCaps("current").setFields("*").setIncludeResolvedTo(true).get(); + assertIndices(response, "new_index"); + + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("current", expression.original()); + assertThat(expression.localExpressions().indices(), containsInAnyOrder("new_index")); + } + + public void testResolvedExpressionWithWildcard() { + FieldCapabilitiesResponse response = client().prepareFieldCaps("*index").setFields("*").setIncludeResolvedTo(true).get(); + assertIndices(response, "new_index", "old_index"); + + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("*index", expression.original()); + assertThat(expression.localExpressions().indices(), containsInAnyOrder("new_index", "old_index")); + } + + public void testResolvedExpressionWithClosedIndices() throws IOException { + // in addition to the existing "old_index" and "new_index", create two where the test query throws an error on rewrite + assertAcked(prepareCreate("index1-error"), prepareCreate("index2-error")); + ensureGreen("index1-error", "index2-error"); + + // Closed shards will result to index error because shards must be in readable state + closeShards(internalCluster(), "index1-error", "index2-error"); + + FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index", "new_index", "index1-error", "index2-error") + .setFields("*") + .setIncludeResolvedTo(true) + .get(); + Set openIndices = Set.of("old_index", "new_index"); + Set closedIndices = Set.of("index1-error", "index2-error"); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(4, resolvedLocally.expressions().size()); + for (ResolvedIndexExpression expression : expressions) { + ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions(); + if (openIndices.contains(expression.original())) { + assertThat(expression.localExpressions().indices(), containsInAnyOrder(expression.original())); + assertEquals(ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS, localExpressions.localIndexResolutionResult()); + } else if (closedIndices.contains(expression.original())) { + Set concreteIndices = localExpressions.indices(); + assertEquals(0, concreteIndices.size()); + assertEquals( + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + localExpressions.localIndexResolutionResult() + ); + } + } + } + + public void testResolvedExpressionWithAllIndices() { + FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("*").setIncludeResolvedTo(true).get(); + assertIndices(response, "new_index", "old_index"); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("_all", expression.original()); // not setting indices means _all + ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions(); + assertThat(expression.localExpressions().indices(), containsInAnyOrder("new_index", "old_index")); + assertEquals(ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS, localExpressions.localIndexResolutionResult()); + } + + public void testResolvedExpressionWithOnlyOneClosedIndexAndIgnoreUnavailable() { + boolean ignoreUnavailable = true; + IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false); + client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet(); + FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index") + .setFields("*") + .setIndicesOptions(options) + .setIncludeResolvedTo(true) + .get(); + + assertIndices(response); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, expressions.size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("old_index", expression.original()); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions(); + Set concreteIndices = localExpressions.indices(); + assertEquals(0, concreteIndices.size()); + assertEquals( + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + localExpressions.localIndexResolutionResult() + ); + } + + public void testResolvedExpressionWithIndexFilter() throws InterruptedException { + assertAcked( + prepareCreate("index-1").setMapping("timestamp", "type=date", "field1", "type=keyword"), + prepareCreate("index-2").setMapping("timestamp", "type=date", "field1", "type=long") + ); + + List reqs = new ArrayList<>(); + reqs.add(prepareIndex("index-1").setSource("timestamp", "2015-07-08")); + reqs.add(prepareIndex("index-1").setSource("timestamp", "2018-07-08")); + reqs.add(prepareIndex("index-2").setSource("timestamp", "2019-10-12")); + reqs.add(prepareIndex("index-2").setSource("timestamp", "2020-07-08")); + indexRandom(true, reqs); + + FieldCapabilitiesResponse response = client().prepareFieldCaps("index-*") + .setFields("*") + .setIndexFilter(QueryBuilders.rangeQuery("timestamp").gte("2019-11-01")) + .setIncludeResolvedTo(true) + .get(); + + assertIndices(response, "index-2"); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("index-*", expression.original()); + assertThat(expression.localExpressions().indices(), containsInAnyOrder("index-1", "index-2")); + } + + public void testNoneExpressionIndices() { + // The auth code injects the pattern ["*", "-*"] which effectively means a request that requests no indices + FieldCapabilitiesResponse response = client().prepareFieldCaps("*", "-*").setFields("*").get(); + + assertThat(response.getIndices().length, is(0)); + } + + public void testExclusion() { + assertAcked(prepareCreate("index-2024"), prepareCreate("index-2025")); + + prepareIndex("index-2024").setSource("timestamp", "2024", "f1", "1").get(); + prepareIndex("index-2025").setSource("timestamp", "2025", "f2", "2").get(); + + var response = client().prepareFieldCaps("index-*", "-*2025").setFields("*").get(); + assertIndices(response, "index-2024"); + } + + public void testExclusionWithResolvedTo() { + assertAcked(prepareCreate("index-2024"), prepareCreate("index-2025")); + + prepareIndex("index-2024").setSource("timestamp", "2024", "f1", "1").get(); + prepareIndex("index-2025").setSource("timestamp", "2025", "f2", "2").get(); + + var response = client().prepareFieldCaps("index-*", "-*2025").setFields("*").setIncludeResolvedTo(true).get(); + assertIndices(response, "index-2024"); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("index-*", expression.original()); + assertThat(expression.localExpressions().indices(), containsInAnyOrder("index-2024", "index-2025")); + } + private void assertIndices(FieldCapabilitiesResponse response, String... indices) { assertNotNull(response.getIndices()); Arrays.sort(indices); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 46d665fde38e8..42ab675008d94 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -18,6 +18,8 @@ import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStats; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; +import org.elasticsearch.action.admin.indices.ResizeIndexTestUtils; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActionTestUtils; @@ -653,7 +655,7 @@ public void testRestoreShrinkIndex() throws Exception { logger.info("--> shrink the index"); updateIndexSettings(Settings.builder().put("index.blocks.write", true), sourceIdx); - assertAcked(indicesAdmin().prepareResizeIndex(sourceIdx, shrunkIdx).get()); + assertAcked(ResizeIndexTestUtils.executeResize(ResizeType.SHRINK, sourceIdx, shrunkIdx, Settings.builder())); logger.info("--> snapshot the shrunk index"); createSnapshot(repo, snapshot, Collections.singletonList(shrunkIdx)); diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index dd722d9ce0f11..31c340d458038 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -465,8 +465,11 @@ org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsFormat, org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat, org.elasticsearch.index.codec.vectors.diskbbq.next.ESNextDiskBBQVectorsFormat, - org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat, + org.elasticsearch.index.codec.vectors.es93.ES93FlatVectorFormat, org.elasticsearch.index.codec.vectors.es93.ES93HnswVectorsFormat, + org.elasticsearch.index.codec.vectors.es93.ES93ScalarQuantizedVectorsFormat, + org.elasticsearch.index.codec.vectors.es93.ES93HnswScalarQuantizedVectorsFormat, + org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93HnswBinaryQuantizedVectorsFormat; provides org.apache.lucene.codecs.Codec diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 8074fee2826f7..7ac39ab974e3a 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -151,7 +151,6 @@ import org.elasticsearch.action.admin.indices.settings.get.TransportGetSettingsAction; import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction; import org.elasticsearch.action.admin.indices.shards.TransportIndicesShardStoresAction; -import org.elasticsearch.action.admin.indices.shrink.ResizeAction; import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.admin.indices.stats.FieldUsageStatsAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; @@ -705,7 +704,7 @@ public void reg actions.register(IndicesSegmentsAction.INSTANCE, TransportIndicesSegmentsAction.class); actions.register(TransportIndicesShardStoresAction.TYPE, TransportIndicesShardStoresAction.class); actions.register(TransportCreateIndexAction.TYPE, TransportCreateIndexAction.class); - actions.register(ResizeAction.INSTANCE, TransportResizeAction.class); + actions.register(TransportResizeAction.TYPE, TransportResizeAction.class); actions.register(RolloverAction.INSTANCE, TransportRolloverAction.class); actions.register(LazyRolloverAction.INSTANCE, LazyRolloverAction.TransportLazyRolloverAction.class); actions.register(TransportDeleteIndexAction.TYPE, TransportDeleteIndexAction.class); diff --git a/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpression.java b/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpression.java index e1db2128143e9..1204ab001e14e 100644 --- a/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpression.java +++ b/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpression.java @@ -71,6 +71,19 @@ public void writeTo(StreamOutput out) throws IOException { * or unauthorized concrete resources. * A wildcard expression resolving to nothing is still considered a successful resolution. * The NONE result indicates that no local resolution was attempted because the expression is known to be remote-only. + * + * This distinction is needed to return either 403 (forbidden) or 404 (not found) to the user, + * and must be propagated by the linked projects to the request coordinator. + * + * CONCRETE_RESOURCE_NOT_VISIBLE: Indicates that a non-wildcard expression was resolved to nothing, + * either because the index does not exist or is closed. + * + * CONCRETE_RESOURCE_UNAUTHORIZED: Indicates that the expression could be resolved to a concrete index, + * but the requesting user is not authorized to access it. + * + * NONE: No local resolution was attempted, typically because the expression is remote-only. + * + * SUCCESS: Local index resolution was successful. */ public enum LocalIndexResolutionResult { NONE, diff --git a/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java b/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java index 301db629d8d69..007db112183dd 100644 --- a/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java +++ b/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java @@ -73,6 +73,14 @@ public void addExpressions( ); } + /** + * Add a new resolved expression. + * @param expression the expression you want to add. + */ + public void addExpression(ResolvedIndexExpression expression) { + expressions.add(expression); + } + public void addRemoteExpressions(String original, Set remoteExpressions) { Objects.requireNonNull(original); Objects.requireNonNull(remoteExpressions); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java deleted file mode 100644 index 3c8ee569670dc..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.action.admin.indices.shrink; - -import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; - -public class ResizeAction extends ActionType { - - public static final ResizeAction INSTANCE = new ResizeAction(); - public static final String NAME = "indices:admin/resize"; - - private ResizeAction() { - super(NAME); - } - -} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java index 21013525fcc8c..8430ad210eb3e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; -import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.support.ActiveShardCount; @@ -19,11 +18,11 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xcontent.ToXContentObject; -import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; @@ -34,10 +33,11 @@ /** * Request class to shrink an index into a single shard */ -public class ResizeRequest extends AcknowledgedRequest implements IndicesRequest, ToXContentObject { +public class ResizeRequest extends AcknowledgedRequest implements IndicesRequest { public static final ObjectParser PARSER = new ObjectParser<>("resize_request"); - private static final ParseField MAX_PRIMARY_SHARD_SIZE = new ParseField("max_primary_shard_size"); + public static final ParseField MAX_PRIMARY_SHARD_SIZE = new ParseField("max_primary_shard_size"); + static { PARSER.declareField( (parser, request, context) -> request.getTargetIndexRequest().settings(parser.map()), @@ -58,8 +58,8 @@ public class ResizeRequest extends AcknowledgedRequest implements } private CreateIndexRequest targetIndexRequest; - private String sourceIndex; - private ResizeType type = ResizeType.SHRINK; + private final String sourceIndex; + private final ResizeType type; private Boolean copySettings = true; private ByteSizeValue maxPrimaryShardSize; @@ -74,14 +74,11 @@ public ResizeRequest(StreamInput in) throws IOException { } } - ResizeRequest() { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT); - } - - public ResizeRequest(String targetIndex, String sourceIndex) { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT); + public ResizeRequest(TimeValue masterNodeTimeout, TimeValue ackTimeout, ResizeType resizeType, String sourceIndex, String targetIndex) { + super(masterNodeTimeout, ackTimeout); this.targetIndexRequest = new CreateIndexRequest(targetIndex); this.sourceIndex = sourceIndex; + this.type = resizeType; } @Override @@ -106,10 +103,6 @@ public ActionRequestValidationException validate() { return validationException; } - public void setSourceIndex(String index) { - this.sourceIndex = index; - } - @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -175,13 +168,6 @@ public void setWaitForActiveShards(final int waitForActiveShards) { setWaitForActiveShards(ActiveShardCount.from(waitForActiveShards)); } - /** - * The type of the resize operation - */ - public void setResizeType(ResizeType type) { - this.type = Objects.requireNonNull(type); - } - /** * Returns the type of the resize operation */ @@ -219,30 +205,6 @@ public ByteSizeValue getMaxPrimaryShardSize() { return maxPrimaryShardSize; } - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - { - builder.startObject(CreateIndexRequest.SETTINGS.getPreferredName()); - { - targetIndexRequest.settings().toXContent(builder, params); - } - builder.endObject(); - builder.startObject(CreateIndexRequest.ALIASES.getPreferredName()); - { - for (Alias alias : targetIndexRequest.aliases()) { - alias.toXContent(builder, params); - } - } - builder.endObject(); - if (maxPrimaryShardSize != null) { - builder.field(MAX_PRIMARY_SHARD_SIZE.getPreferredName(), maxPrimaryShardSize); - } - } - builder.endObject(); - return builder; - } - public void fromXContent(XContentParser parser) throws IOException { PARSER.parse(parser, this, null); } @@ -263,4 +225,8 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(targetIndexRequest, sourceIndex, type, copySettings, maxPrimaryShardSize); } + + public void setTargetIndexSettings(Settings.Builder settings) { + targetIndexRequest.settings(settings); + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java deleted file mode 100644 index 8d14ca18c7c20..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ -package org.elasticsearch.action.admin.indices.shrink; - -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.support.ActiveShardCount; -import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; -import org.elasticsearch.client.internal.ElasticsearchClient; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; - -public class ResizeRequestBuilder extends AcknowledgedRequestBuilder { - public ResizeRequestBuilder(ElasticsearchClient client) { - super(client, ResizeAction.INSTANCE, new ResizeRequest()); - } - - public ResizeRequestBuilder setTargetIndex(CreateIndexRequest request) { - this.request.setTargetIndex(request); - return this; - } - - public ResizeRequestBuilder setSourceIndex(String index) { - this.request.setSourceIndex(index); - return this; - } - - public ResizeRequestBuilder setSettings(Settings settings) { - this.request.getTargetIndexRequest().settings(settings); - return this; - } - - /** - * Sets the number of shard copies that should be active for creation of the - * new shrunken index to return. Defaults to {@link ActiveShardCount#DEFAULT}, which will - * wait for one shard copy (the primary) to become active. Set this value to - * {@link ActiveShardCount#ALL} to wait for all shards (primary and all replicas) to be active - * before returning. Otherwise, use {@link ActiveShardCount#from(int)} to set this value to any - * non-negative integer, up to the number of copies per shard (number of replicas + 1), - * to wait for the desired amount of shard copies to become active before returning. - * Index creation will only wait up until the timeout value for the number of shard copies - * to be active before returning. Check {@link CreateIndexResponse#isShardsAcknowledged()} to - * determine if the requisite shard copies were all started before returning or timing out. - * - * @param waitForActiveShards number of active shard copies to wait on - */ - public ResizeRequestBuilder setWaitForActiveShards(ActiveShardCount waitForActiveShards) { - this.request.setWaitForActiveShards(waitForActiveShards); - return this; - } - - /** - * A shortcut for {@link #setWaitForActiveShards(ActiveShardCount)} where the numerical - * shard count is passed in, instead of having to first call {@link ActiveShardCount#from(int)} - * to get the ActiveShardCount. - */ - public ResizeRequestBuilder setWaitForActiveShards(final int waitForActiveShards) { - return setWaitForActiveShards(ActiveShardCount.from(waitForActiveShards)); - } - - public ResizeRequestBuilder setResizeType(ResizeType type) { - this.request.setResizeType(type); - return this; - } - - /** - * Sets the max primary shard size of the target index. - */ - public ResizeRequestBuilder setMaxPrimaryShardSize(ByteSizeValue maxPrimaryShardSize) { - this.request.setMaxPrimaryShardSize(maxPrimaryShardSize); - return this; - } -} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index 2f8765fc20b8c..4a781a5a761fe 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -10,6 +10,7 @@ package org.elasticsearch.action.admin.indices.shrink; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; @@ -48,6 +49,8 @@ */ public class TransportResizeAction extends TransportMasterNodeAction { + public static final ActionType TYPE = new ActionType<>("indices:admin/resize"); + private final MetadataCreateIndexService createIndexService; private final ProjectResolver projectResolver; private final Client client; @@ -62,7 +65,7 @@ public TransportResizeAction( ProjectResolver projectResolver, Client client ) { - this(ResizeAction.NAME, transportService, clusterService, threadPool, createIndexService, actionFilters, projectResolver, client); + this(TYPE.name(), transportService, clusterService, threadPool, createIndexService, actionFilters, projectResolver, client); } protected TransportResizeAction( diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java index 45b71fd63cb16..b293f195d1864 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -33,6 +34,7 @@ import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.DATA_RETENTION_FIELD; import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.DOWNSAMPLING_FIELD; +import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.DOWNSAMPLING_METHOD_FIELD; import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.ENABLED_FIELD; /** @@ -50,7 +52,8 @@ public interface Factory { Request create( @Nullable TimeValue dataRetention, @Nullable Boolean enabled, - @Nullable List downsampling + @Nullable List downsampling, + @Nullable DownsampleConfig.SamplingMethod downsamplingMethod ); } @@ -58,7 +61,12 @@ Request create( public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "put_data_stream_lifecycle_request", false, - (args, factory) -> factory.create((TimeValue) args[0], (Boolean) args[1], (List) args[2]) + (args, factory) -> factory.create( + (TimeValue) args[0], + (Boolean) args[1], + (List) args[2], + (DownsampleConfig.SamplingMethod) args[3] + ) ); static { @@ -75,6 +83,12 @@ Request create( DOWNSAMPLING_FIELD, ObjectParser.ValueType.OBJECT_ARRAY ); + PARSER.declareField( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> DownsampleConfig.SamplingMethod.fromString(p.text()), + DOWNSAMPLING_METHOD_FIELD, + ObjectParser.ValueType.STRING + ); } public static Request parseRequest(XContentParser parser, Factory factory) { @@ -120,7 +134,7 @@ public void writeTo(StreamOutput out) throws IOException { } public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names, @Nullable TimeValue dataRetention) { - this(masterNodeTimeout, ackTimeout, names, dataRetention, null, null); + this(masterNodeTimeout, ackTimeout, names, dataRetention, null); } public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names, DataStreamLifecycle lifecycle) { @@ -136,7 +150,7 @@ public Request( @Nullable TimeValue dataRetention, @Nullable Boolean enabled ) { - this(masterNodeTimeout, ackTimeout, names, dataRetention, enabled, null); + this(masterNodeTimeout, ackTimeout, names, dataRetention, enabled, null, null); } public Request( @@ -145,14 +159,16 @@ public Request( String[] names, @Nullable TimeValue dataRetention, @Nullable Boolean enabled, - @Nullable List downsampling + @Nullable List downsamplingRounds, + @Nullable DownsampleConfig.SamplingMethod downsamplingMethod ) { super(masterNodeTimeout, ackTimeout); this.names = names; this.lifecycle = DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(dataRetention) .enabled(enabled == null || enabled) - .downsampling(downsampling) + .downsamplingRounds(downsamplingRounds) + .downsamplingMethod(downsamplingMethod) .build(); } diff --git a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java index 444785d55e3d4..ae08245ceb7c9 100644 --- a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java +++ b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java @@ -101,17 +101,6 @@ public class DownsampleConfig implements NamedWriteable, ToXContentObject { ); } - /** - * Create a new {@link DownsampleConfig} using the given configuration parameters. - * @param fixedInterval the fixed interval to use for computing the date histogram for the rolled up documents (required). - * @deprecated please use {@link DownsampleConfig#DownsampleConfig(DateHistogramInterval, SamplingMethod)}, this method is being kept - * until the sampling method is completely integrated with ILM and DLM. - */ - @Deprecated - public DownsampleConfig(final DateHistogramInterval fixedInterval) { - this(fixedInterval, null); - } - /** * Create a new {@link DownsampleConfig} using the given configuration parameters. * @param fixedInterval the fixed interval to use for computing the date histogram for the rolled up documents (required). @@ -144,25 +133,28 @@ public DownsampleConfig(final StreamInput in) throws IOException { * - The target interval needs to be a multiple of the source interval * throws an IllegalArgumentException to signal that the target interval is not acceptable */ - public static void validateSourceAndTargetIntervals(DownsampleConfig source, DownsampleConfig target) { - long sourceMillis = source.fixedInterval.estimateMillis(); - long targetMillis = target.fixedInterval.estimateMillis(); + public static void validateSourceAndTargetIntervals( + DateHistogramInterval sourceFxedInterval, + DateHistogramInterval targetFixedInterval + ) { + long sourceMillis = sourceFxedInterval.estimateMillis(); + long targetMillis = targetFixedInterval.estimateMillis(); if (sourceMillis >= targetMillis) { // Downsampling interval must be greater than source interval throw new IllegalArgumentException( "Downsampling interval [" - + target.fixedInterval + + targetFixedInterval + "] must be greater than the source index interval [" - + source.fixedInterval + + sourceFxedInterval + "]." ); } else if (targetMillis % sourceMillis != 0) { // Downsampling interval must be a multiple of the source interval throw new IllegalArgumentException( "Downsampling interval [" - + target.fixedInterval + + targetFixedInterval + "] must be a multiple of the source index interval [" - + source.fixedInterval + + sourceFxedInterval + "]." ); } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java index c4c974ad6b668..c17cae1779056 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java @@ -40,6 +40,7 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implemen public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosed(); private static final TransportVersion FIELD_CAPS_ADD_CLUSTER_ALIAS = TransportVersion.fromName("field_caps_add_cluster_alias"); + static final TransportVersion RESOLVED_FIELDS_CAPS = TransportVersion.fromName("resolved_fields_caps"); private String clusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; @@ -58,6 +59,8 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implemen */ private transient boolean includeIndices = false; + private boolean includeResolvedTo = false; + /** * Controls whether all local indices should be returned if no remotes matched * See {@link org.elasticsearch.transport.RemoteClusterService#groupIndices} returnLocalAll argument. @@ -93,6 +96,11 @@ public FieldCapabilitiesRequest(StreamInput in) throws IOException { } else { clusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; } + if (in.getTransportVersion().supports(RESOLVED_FIELDS_CAPS)) { + includeResolvedTo = in.readBoolean(); + } else { + includeResolvedTo = false; + } } public FieldCapabilitiesRequest() {} @@ -145,6 +153,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().supports(FIELD_CAPS_ADD_CLUSTER_ALIAS)) { out.writeOptionalString(clusterAlias); } + if (out.getTransportVersion().supports(RESOLVED_FIELDS_CAPS)) { + out.writeBoolean(includeResolvedTo); + } } @Override @@ -223,6 +234,11 @@ public FieldCapabilitiesRequest includeIndices(boolean includeIndices) { return this; } + public FieldCapabilitiesRequest includeResolvedTo(boolean includeResolvedTo) { + this.includeResolvedTo = includeResolvedTo; + return this; + } + public FieldCapabilitiesRequest returnLocalAll(boolean returnLocalAll) { this.returnLocalAll = returnLocalAll; return this; @@ -256,6 +272,10 @@ public boolean includeIndices() { return includeIndices; } + public boolean includeResolvedTo() { + return includeResolvedTo; + } + public boolean returnLocalAll() { return returnLocalAll; } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java index 4437895c7e08d..9e79c52ce8ae1 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java @@ -63,4 +63,9 @@ public FieldCapabilitiesRequestBuilder setReturnLocalAll(boolean returnLocalAll) request().returnLocalAll(returnLocalAll); return this; } + + public FieldCapabilitiesRequestBuilder setIncludeResolvedTo(boolean resolvedTo) { + request().includeResolvedTo(resolvedTo); + return this; + } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java index daea9b1a5fc01..8372ad5909de1 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java @@ -11,6 +11,7 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ResolvedIndexExpressions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; @@ -27,6 +28,9 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; + +import static org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest.RESOLVED_FIELDS_CAPS; /** * Response for {@link FieldCapabilitiesRequest} requests. @@ -41,6 +45,19 @@ public class FieldCapabilitiesResponse extends ActionResponse implements Chunked public static final ParseField FAILURES_FIELD = new ParseField("failures"); private final String[] indices; + + // Index expressions resolved in the context of the node that creates this response. + // If created on a linked project, "local" refers to that linked project. + // If created on the coordinating node, "local" refers to the coordinator itself. + // This data is sent from linked projects to the coordinator to inform it of each remote's local resolution state. + private final ResolvedIndexExpressions resolvedLocally; + // Remotely resolved index expressions, keyed by project alias. + // This is only populated by the coordinating node with the `resolvedLocally` data structure it receives + // back from the remotes. Used in the coordinating node for error checking, it's never sent over the wire. + // Keeping this distinction (between resolvedLocally and resolvedRemotely) further prevents project chaining + // and simplifies resolution logic, because the remoteExpressions in the resolvedLocally data structure are + // used to access data in `resolvedRemotely`. + private final transient Map resolvedRemotely; private final Map> fields; private final List failures; private final List indexResponses; @@ -51,25 +68,45 @@ public FieldCapabilitiesResponse( Map> fields, List failures ) { - this(indices, fields, Collections.emptyList(), failures, null); + this(indices, null, Collections.emptyMap(), fields, Collections.emptyList(), failures, null); } public FieldCapabilitiesResponse(String[] indices, Map> fields) { - this(indices, fields, Collections.emptyList(), Collections.emptyList(), null); + this(indices, null, Collections.emptyMap(), fields, Collections.emptyList(), Collections.emptyList(), null); + } + + public static FieldCapabilitiesResponse empty() { + return new FieldCapabilitiesResponse( + Strings.EMPTY_ARRAY, + null, + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyList(), + Collections.emptyList(), + null + ); } public FieldCapabilitiesResponse(List indexResponses, List failures) { - this(Strings.EMPTY_ARRAY, Collections.emptyMap(), indexResponses, failures, null); + this(Strings.EMPTY_ARRAY, null, Collections.emptyMap(), Collections.emptyMap(), indexResponses, failures, null); + } + + public static FieldCapabilitiesResponse.Builder builder() { + return new FieldCapabilitiesResponse.Builder(); } private FieldCapabilitiesResponse( String[] indices, + ResolvedIndexExpressions resolvedLocally, + Map resolvedRemotely, Map> fields, List indexResponses, List failures, TransportVersion minTransportVersion ) { this.fields = Objects.requireNonNull(fields); + this.resolvedLocally = resolvedLocally; + this.resolvedRemotely = Objects.requireNonNull(resolvedRemotely); this.indexResponses = Objects.requireNonNull(indexResponses); this.indices = indices; this.failures = failures; @@ -84,6 +121,14 @@ public FieldCapabilitiesResponse(StreamInput in) throws IOException { this.minTransportVersion = in.getTransportVersion().supports(MIN_TRANSPORT_VERSION) ? in.readOptional(TransportVersion::readVersion) : null; + if (in.getTransportVersion().supports(RESOLVED_FIELDS_CAPS)) { + this.resolvedLocally = in.readOptionalWriteable(ResolvedIndexExpressions::new); + } else { + this.resolvedLocally = null; + } + // when receiving a response we expect the resolved remotely to be empty. + // It's only non-empty on the coordinating node if the FC requests targets remotes. + this.resolvedRemotely = Collections.emptyMap(); } /** @@ -126,6 +171,20 @@ public List getIndexResponses() { return indexResponses; } + /** + * Locally resolved index expressions + */ + public ResolvedIndexExpressions getResolvedLocally() { + return resolvedLocally; + } + + /** + * Remotely resolved index expressions, non-empty only in the FC coordinator + */ + public Map getResolvedRemotely() { + return resolvedRemotely; + } + /** * Get the field capabilities per type for the provided {@code field}. */ @@ -145,7 +204,7 @@ public TransportVersion minTransportVersion() { * Build a new response replacing the {@link #minTransportVersion()}. */ public FieldCapabilitiesResponse withMinTransportVersion(TransportVersion newMin) { - return new FieldCapabilitiesResponse(indices, fields, indexResponses, failures, newMin); + return new FieldCapabilitiesResponse(indices, resolvedLocally, resolvedRemotely, fields, indexResponses, failures, newMin); } /** @@ -172,6 +231,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().supports(MIN_TRANSPORT_VERSION)) { out.writeOptional((Writer) (o, v) -> TransportVersion.writeVersion(v, o), minTransportVersion); } + if (out.getTransportVersion().supports(RESOLVED_FIELDS_CAPS)) { + out.writeOptionalWriteable(resolvedLocally); + } } private static void writeField(StreamOutput out, Map map) throws IOException { @@ -210,6 +272,8 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; FieldCapabilitiesResponse that = (FieldCapabilitiesResponse) o; return Arrays.equals(indices, that.indices) + && Objects.equals(resolvedLocally, that.resolvedLocally) + && Objects.equals(resolvedRemotely, that.resolvedRemotely) && Objects.equals(fields, that.fields) && Objects.equals(indexResponses, that.indexResponses) && Objects.equals(failures, that.failures) @@ -218,7 +282,9 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(fields, indexResponses, failures, minTransportVersion) * 31 + Arrays.hashCode(indices); + int result = Objects.hash(resolvedLocally, resolvedRemotely, fields, indexResponses, failures, minTransportVersion); + result = 31 * result + Arrays.hashCode(indices); + return result; } @Override @@ -226,12 +292,10 @@ public String toString() { return indexResponses.isEmpty() ? Strings.toString(this) : "FieldCapabilitiesResponse{unmerged}"; } - public static Builder builder() { - return new Builder(); - } - public static class Builder { private String[] indices = Strings.EMPTY_ARRAY; + private ResolvedIndexExpressions resolvedLocally; + private Map resolvedRemotely = Collections.emptyMap(); private Map> fields = Collections.emptyMap(); private List indexResponses = Collections.emptyList(); private List failures = Collections.emptyList(); @@ -244,6 +308,24 @@ public Builder withIndices(String[] indices) { return this; } + public Builder withResolved(ResolvedIndexExpressions resolvedLocally, Map resolvedRemotely) { + this.resolvedLocally = resolvedLocally; + this.resolvedRemotely = resolvedRemotely; + return this; + } + + public Builder withResolvedRemotelyBuilder(Map resolvedRemotelyBuilder) { + this.resolvedRemotely = resolvedRemotelyBuilder.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().build())); + return this; + } + + public Builder withResolvedLocally(ResolvedIndexExpressions resolvedLocally) { + this.resolvedLocally = resolvedLocally; + return this; + } + public Builder withFields(Map> fields) { this.fields = fields; return this; @@ -265,7 +347,15 @@ public Builder withMinTransportVersion(TransportVersion minTransportVersion) { } public FieldCapabilitiesResponse build() { - return new FieldCapabilitiesResponse(indices, fields, indexResponses, failures, minTransportVersion); + return new FieldCapabilitiesResponse( + indices, + resolvedLocally, + resolvedRemotely, + fields, + indexResponses, + failures, + minTransportVersion + ); } } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index ffe14bcc6af50..87f74c0704c53 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -21,15 +21,20 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.RemoteClusterActionType; +import org.elasticsearch.action.ResolvedIndexExpression; +import org.elasticsearch.action.ResolvedIndexExpressions; import org.elasticsearch.action.support.AbstractThreadedActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ChannelActionListener; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.RefCountingRunnable; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.cluster.ProjectState; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -71,6 +76,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -80,9 +86,11 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import static org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest.RESOLVED_FIELDS_CAPS; import static org.elasticsearch.action.search.TransportSearchHelper.checkCCSVersionCompatibility; public class TransportFieldCapabilitiesAction extends HandledTransportAction { + public static final String EXCLUSION = "-"; public static final String NAME = "indices:data/read/field_caps"; public static final ActionType TYPE = new ActionType<>(NAME); public static final RemoteClusterActionType REMOTE_TYPE = new RemoteClusterActionType<>( @@ -191,21 +199,67 @@ private void doExecuteForked( final Map remoteClusterIndices = transportService.getRemoteClusterService() .groupIndices(request.indicesOptions(), request.indices(), request.returnLocalAll()); final OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); - // in the case we have one or more remote indices but no local we don't expand to all local indices and just do remote indices - final String[] concreteIndices = localIndices != null - ? indexNameExpressionResolver.concreteIndexNames(projectState.metadata(), localIndices) - : Strings.EMPTY_ARRAY; - - if (concreteIndices.length == 0 && remoteClusterIndices.isEmpty()) { - listener.onResponse( - linkedRequestExecutor.wrapPrimary( - FieldCapabilitiesResponse.builder().withMinTransportVersion(minTransportVersion.get()).build() - ) - ); + + final String[] concreteLocalIndices; + final List resolvedLocallyList; + if (request.getResolvedIndexExpressions() != null) { + // in CPS the Security Action Filter would populate resolvedExpressions for the local project + // thus we can get the concreteLocalIndices based on the resolvedLocallyList + resolvedLocallyList = request.getResolvedIndexExpressions().expressions(); + concreteLocalIndices = resolvedLocallyList.stream() + .map(r -> r.localExpressions().indices()) + .flatMap(Set::stream) + .distinct() + .toArray(String[]::new); + } else { + // In CCS/Local only search we have to populate resolvedLocallyList one by one for each localIndices.indices() + // only if the request is includeResolvedTo() + resolvedLocallyList = new ArrayList<>(); + if (localIndices == null) { + // in the case we have one or more remote indices but no local we don't expand to all local indices + // in this case resolvedLocallyList will remain empty + concreteLocalIndices = Strings.EMPTY_ARRAY; + } else { + concreteLocalIndices = indexNameExpressionResolver.concreteIndexNames(projectState.metadata(), localIndices); + if (request.includeResolvedTo()) { + ProjectMetadata projectMetadata = projectState.metadata(); + IndicesOptions indicesOptions = localIndices.indicesOptions(); + String[] localIndexNames = localIndices.indices(); + if (localIndexNames.length == 0) { + // Empty indices array means match all + String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames(projectMetadata, indicesOptions); + resolvedLocallyList.add(createResolvedIndexExpression(Metadata.ALL, concreteIndexNames)); + } else if (false == IndexNameExpressionResolver.isNoneExpression(localIndexNames)) { + // if it's neither match all nor match none, but we want to include resolutions we loop for all the indicesNames + for (String localIndexName : localIndexNames) { + if (false == localIndexName.startsWith(EXCLUSION)) { + // we populate resolvedLocally iff is not an exclusion + String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames( + projectMetadata, + indicesOptions, + localIndices.includeDataStreams(), + localIndexName + ); + resolvedLocallyList.add(createResolvedIndexExpression(localIndexName, concreteIndexNames)); + } + } + } + } + + } + } + + if (concreteLocalIndices.length == 0 && remoteClusterIndices.isEmpty()) { + FieldCapabilitiesResponse.Builder responseBuilder = FieldCapabilitiesResponse.builder(); + responseBuilder.withMinTransportVersion(minTransportVersion.get()); + if (request.includeResolvedTo()) { + responseBuilder.withResolvedLocally(new ResolvedIndexExpressions(resolvedLocallyList)); + } + listener.onResponse(linkedRequestExecutor.wrapPrimary(responseBuilder.build())); return; } - checkIndexBlocks(projectState, concreteIndices); + checkIndexBlocks(projectState, concreteLocalIndices); final FailureCollector indexFailures = new FailureCollector(); final Map indexResponses = new HashMap<>(); // This map is used to share the index response for indices which have the same index mapping hash to reduce the memory usage. @@ -216,6 +270,10 @@ private void doExecuteForked( indexResponses.clear(); indexMappingHashToResponses.clear(); }; + Map resolvedRemotely = new ConcurrentHashMap<>(); + for (String clusterAlias : remoteClusterIndices.keySet()) { + resolvedRemotely.put(clusterAlias, ResolvedIndexExpressions.builder()); + } final Consumer handleIndexResponse = resp -> { if (fieldCapTask.isCancelled()) { releaseResourcesOnCancel.run(); @@ -282,6 +340,8 @@ private void doExecuteForked( fieldCapTask, indexResponses, indexFailures, + resolvedLocallyList, + resolvedRemotely, minTransportVersion, listener.map(linkedRequestExecutor::wrapPrimary) ); @@ -297,7 +357,7 @@ private void doExecuteForked( request, localIndices, nowInMillis, - concreteIndices, + concreteLocalIndices, singleThreadedExecutor, handleIndexResponse, handleIndexFailure, @@ -312,6 +372,21 @@ private void doExecuteForked( OriginalIndices originalIndices = remoteIndices.getValue(); FieldCapabilitiesRequest remoteRequest = prepareRemoteRequest(clusterAlias, request, originalIndices, nowInMillis); ActionListener remoteListener = ActionListener.wrap(response -> { + + if (request.includeResolvedTo() && response.getResolvedLocally() != null) { + ResolvedIndexExpressions resolvedOnRemoteProject = response.getResolvedLocally(); + // for bwc we need to check that resolvedOnRemoteProject Exists in the response + if (resolvedOnRemoteProject != null) { + for (ResolvedIndexExpression remoteResolvedExpression : resolvedOnRemoteProject.expressions()) { + resolvedRemotely.computeIfPresent(clusterAlias, (k, v) -> { + v.addExpression(remoteResolvedExpression); + return v; + }); + } + } + + } + for (FieldCapabilitiesIndexResponse resp : response.getIndexResponses()) { String indexName = RemoteClusterAware.buildRemoteIndexName(clusterAlias, resp.getIndexName()); handleIndexResponse.accept( @@ -328,6 +403,21 @@ private void doExecuteForked( Exception ex = failure.getException(); for (String index : failure.getIndices()) { handleIndexFailure.accept(RemoteClusterAware.buildRemoteIndexName(clusterAlias, index), ex); + if (request.includeResolvedTo()) { + ResolvedIndexExpression err = new ResolvedIndexExpression( + index, + new ResolvedIndexExpression.LocalExpressions( + Set.of(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + null + ), + Set.of() + ); + resolvedRemotely.computeIfPresent(clusterAlias, (k, v) -> { + v.addExpression(err); + return v; + }); + } } } minTransportVersion.accumulateAndGet(response.minTransportVersion(), (lhs, rhs) -> { @@ -339,6 +429,21 @@ private void doExecuteForked( }, ex -> { for (String index : originalIndices.indices()) { handleIndexFailure.accept(RemoteClusterAware.buildRemoteIndexName(clusterAlias, index), ex); + if (request.includeResolvedTo()) { + ResolvedIndexExpression err = new ResolvedIndexExpression( + index, + new ResolvedIndexExpression.LocalExpressions( + Set.of(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + null + ), + Set.of() + ); + resolvedRemotely.computeIfPresent(clusterAlias, (k, v) -> { + v.addExpression(err); + return v; + }); + } } }); @@ -377,6 +482,20 @@ private void doExecuteForked( } } + private static ResolvedIndexExpression createResolvedIndexExpression(String original, String[] concreteIndexNames) { + boolean isWildcard = Regex.isSimpleMatchPattern(original); + // if it is a wildcard we consider it successful even if it didn't resolve to any concrete index + ResolvedIndexExpression.LocalIndexResolutionResult resolutionResult = concreteIndexNames.length > 0 || isWildcard + ? ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + : ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE; + + return new ResolvedIndexExpression( + original, + new ResolvedIndexExpression.LocalExpressions(Set.of(concreteIndexNames), resolutionResult, null), + Collections.emptySet() + ); + } + private Executor buildSingleThreadedExecutor() { final ThrottledTaskRunner throttledTaskRunner = new ThrottledTaskRunner("field_caps", 1, searchCoordinationExecutor); return r -> throttledTaskRunner.enqueueTask(new ActionListener<>() { @@ -433,19 +552,27 @@ private static void mergeIndexResponses( CancellableTask task, Map indexResponses, FailureCollector indexFailures, + List resolvedLocallyList, + Map resolvedRemotely, AtomicReference minTransportVersion, ActionListener listener ) { + ResolvedIndexExpressions resolvedLocally = new ResolvedIndexExpressions(resolvedLocallyList); List failures = indexFailures.build(indexResponses.keySet()); if (indexResponses.isEmpty() == false) { if (request.isMergeResults()) { - ActionListener.completeWith(listener, () -> merge(indexResponses, task, request, failures, minTransportVersion)); + ActionListener.completeWith( + listener, + () -> merge(indexResponses, resolvedLocally, resolvedRemotely, task, request, failures, minTransportVersion) + ); } else { listener.onResponse( FieldCapabilitiesResponse.builder() .withIndexResponses(new ArrayList<>(indexResponses.values())) - .withFailures(failures) + .withResolvedLocally(resolvedLocally) + .withResolvedRemotelyBuilder(resolvedRemotely) .withMinTransportVersion(minTransportVersion.get()) + .withFailures(failures) .build() ); } @@ -460,7 +587,12 @@ private static void mergeIndexResponses( && ise.getCause() instanceof ElasticsearchTimeoutException )) { listener.onResponse( - FieldCapabilitiesResponse.builder().withFailures(failures).withMinTransportVersion(minTransportVersion.get()).build() + FieldCapabilitiesResponse.builder() + .withFailures(failures) + .withResolvedLocally(resolvedLocally) + .withResolvedRemotelyBuilder(resolvedRemotely) + .withMinTransportVersion(minTransportVersion.get()) + .build() ); } else { // throw back the first exception @@ -489,6 +621,7 @@ private static FieldCapabilitiesRequest prepareRemoteRequest( remoteRequest.indexFilter(request.indexFilter()); remoteRequest.nowInMillis(nowInMillis); remoteRequest.includeEmptyFields(request.includeEmptyFields()); + remoteRequest.includeResolvedTo(request.includeResolvedTo()); return remoteRequest; } @@ -500,6 +633,8 @@ private static boolean hasSameMappingHash(FieldCapabilitiesIndexResponse r1, Fie private static FieldCapabilitiesResponse merge( Map indexResponsesMap, + ResolvedIndexExpressions resolvedLocally, + Map resolvedRemotely, CancellableTask task, FieldCapabilitiesRequest request, List failures, @@ -533,6 +668,22 @@ private static FieldCapabilitiesResponse merge( collectFields(fieldsBuilder, fields, request.includeIndices()); } + List failedIndices = failures.stream().flatMap(f -> Arrays.stream(f.getIndices())).toList(); + List collect = resolvedLocally.expressions().stream().map(expression -> { + if (failedIndices.contains(expression.original())) { + return new ResolvedIndexExpression( + expression.original(), + new ResolvedIndexExpression.LocalExpressions( + Set.of(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + null + ), + expression.remoteExpressions() + ); + } + return expression; + }).toList(); + // The merge method is only called on the primary coordinator for cross-cluster field caps, so we // log relevant "5xx" errors that occurred in this 2xx response to ensure they are only logged once. // These failures have already been deduplicated, before this method was called. @@ -544,12 +695,17 @@ private static FieldCapabilitiesResponse merge( ); } } - return FieldCapabilitiesResponse.builder() + + FieldCapabilitiesResponse.Builder responseBuilder = FieldCapabilitiesResponse.builder() .withIndices(indices) .withFields(Collections.unmodifiableMap(fields)) .withFailures(failures) - .withMinTransportVersion(minTransportVersion.get()) - .build(); + .withMinTransportVersion(minTransportVersion.get()); + if (request.includeResolvedTo() && minTransportVersion.get().supports(RESOLVED_FIELDS_CAPS)) { + // add resolution to response iff includeResolvedTo and all the nodes in the cluster supports it + responseBuilder.withResolvedLocally(new ResolvedIndexExpressions(collect)).withResolvedRemotelyBuilder(resolvedRemotely); + } + return responseBuilder.build(); } private static boolean shouldLogException(Exception e) { diff --git a/server/src/main/java/org/elasticsearch/client/internal/IndicesAdminClient.java b/server/src/main/java/org/elasticsearch/client/internal/IndicesAdminClient.java index 98d621955926a..b7f265b81f442 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/IndicesAdminClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/IndicesAdminClient.java @@ -94,9 +94,6 @@ import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder; -import org.elasticsearch.action.admin.indices.shrink.ResizeAction; -import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; -import org.elasticsearch.action.admin.indices.shrink.ResizeRequestBuilder; import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; @@ -466,14 +463,6 @@ public GetSettingsRequestBuilder prepareGetSettings(TimeValue masterTimeout, Str return new GetSettingsRequestBuilder(this, masterTimeout, indices); } - public ResizeRequestBuilder prepareResizeIndex(String sourceIndex, String targetIndex) { - return new ResizeRequestBuilder(this).setSourceIndex(sourceIndex).setTargetIndex(new CreateIndexRequest(targetIndex)); - } - - public void resizeIndex(ResizeRequest request, ActionListener listener) { - execute(ResizeAction.INSTANCE, request, listener); - } - public RolloverRequestBuilder prepareRolloverIndex(String alias) { return new RolloverRequestBuilder(this).setRolloverTarget(alias); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 9e7fcb1e60f16..93be5259e29f6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -1247,7 +1247,7 @@ public List getDownsamplingRoundsFor( LongSupplier nowSupplier ) { assert backingIndices.indices.contains(index) : "the provided index must be a backing index for this datastream"; - if (lifecycle == null || lifecycle.downsampling() == null) { + if (lifecycle == null || lifecycle.downsamplingRounds() == null) { return List.of(); } @@ -1260,8 +1260,8 @@ public List getDownsamplingRoundsFor( if (indexGenerationTime != null) { long nowMillis = nowSupplier.getAsLong(); long indexGenerationTimeMillis = indexGenerationTime.millis(); - List orderedRoundsForIndex = new ArrayList<>(lifecycle.downsampling().size()); - for (DownsamplingRound round : lifecycle.downsampling()) { + List orderedRoundsForIndex = new ArrayList<>(lifecycle.downsamplingRounds().size()); + for (DownsamplingRound round : lifecycle.downsamplingRounds()) { if (nowMillis >= indexGenerationTimeMillis + round.after().getMillis()) { orderedRoundsForIndex.add(round); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java index 7e60ddf0818d1..bb87ab1994ace 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java @@ -52,12 +52,13 @@ * Lifecycle supports the following configurations: * - enabled, applicable to data and failures * - data retention, applicable to data and failures - * - downsampling, applicable only to data + * - downsampling and downsampling method, applicable only to data */ public class DataStreamLifecycle implements SimpleDiffable, ToXContentObject { // Versions over the wire public static final TransportVersion ADDED_ENABLED_FLAG_VERSION = TransportVersions.V_8_10_X; + public static final TransportVersion ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM = TransportVersion.fromName("add_sample_method_downsample_dlm"); public static final String EFFECTIVE_RETENTION_REST_API_CAPABILITY = "data_stream_lifecycle_effective_retention"; public static final String DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME = "data_streams.lifecycle_only.mode"; @@ -74,6 +75,8 @@ public class DataStreamLifecycle implements SimpleDiffable, "Failure store lifecycle does not support downsampling, please remove the downsampling configuration."; private static final TransportVersion INTRODUCE_LIFECYCLE_TEMPLATE = TransportVersion.fromName("introduce_lifecycle_template"); + public static final String DOWNSAMPLING_METHOD_WITHOUT_ROUNDS_ERROR = + "Downsampling method can only be set when there is at least one downsampling round."; /** * Check if {@link #DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME} is present and set to {@code true}, indicating that @@ -94,7 +97,7 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { Setting.Property.NodeScope ); - public static final DataStreamLifecycle DEFAULT_DATA_LIFECYCLE = DataStreamLifecycle.createDataLifecycle(null, null, null); + public static final DataStreamLifecycle DEFAULT_DATA_LIFECYCLE = DataStreamLifecycle.createDataLifecycle(null, null, null, null); public static final DataStreamLifecycle DEFAULT_FAILURE_LIFECYCLE = DataStreamLifecycle.createFailuresLifecycle(null, null); public static final String DATA_STREAM_LIFECYCLE_ORIGIN = "data_stream_lifecycle"; @@ -104,13 +107,20 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { public static final ParseField EFFECTIVE_RETENTION_FIELD = new ParseField("effective_retention"); public static final ParseField RETENTION_SOURCE_FIELD = new ParseField("retention_determined_by"); public static final ParseField DOWNSAMPLING_FIELD = new ParseField("downsampling"); + public static final ParseField DOWNSAMPLING_METHOD_FIELD = new ParseField("downsampling_method"); private static final ParseField ROLLOVER_FIELD = new ParseField("rollover"); @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "lifecycle", false, - (args, lt) -> new DataStreamLifecycle(lt, (Boolean) args[0], (TimeValue) args[1], (List) args[2]) + (args, lt) -> new DataStreamLifecycle( + lt, + (Boolean) args[0], + (TimeValue) args[1], + (List) args[2], + (DownsampleConfig.SamplingMethod) args[3] + ) ); static { @@ -133,6 +143,12 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { return AbstractObjectParser.parseArray(p, null, DownsamplingRound::fromXContent); } }, DOWNSAMPLING_FIELD, ObjectParser.ValueType.OBJECT_ARRAY_OR_NULL); + PARSER.declareField( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> DownsampleConfig.SamplingMethod.fromString(p.text()), + DOWNSAMPLING_METHOD_FIELD, + ObjectParser.ValueType.STRING + ); } private static final TransportVersion INTRODUCE_FAILURES_LIFECYCLE = TransportVersion.fromName("introduce_failures_lifecycle"); @@ -142,26 +158,34 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { @Nullable private final TimeValue dataRetention; @Nullable - private final List downsampling; + private final List downsamplingRounds; + @Nullable + private final DownsampleConfig.SamplingMethod downsamplingMethod; /** - * This constructor is visible for testing, please use {@link DataStreamLifecycle#createDataLifecycle(Boolean, TimeValue, List)} or + * This constructor is visible for testing, please use + * {@link DataStreamLifecycle#createDataLifecycle(Boolean, TimeValue, List, DownsampleConfig.SamplingMethod)} or * {@link DataStreamLifecycle#createFailuresLifecycle(Boolean, TimeValue)}. */ DataStreamLifecycle( LifecycleType lifecycleType, @Nullable Boolean enabled, @Nullable TimeValue dataRetention, - @Nullable List downsampling + @Nullable List downsamplingRounds, + @Nullable DownsampleConfig.SamplingMethod downsamplingMethod ) { this.lifecycleType = lifecycleType; this.enabled = enabled == null || enabled; this.dataRetention = dataRetention; - if (lifecycleType == LifecycleType.FAILURES && downsampling != null) { + if (lifecycleType == LifecycleType.FAILURES && downsamplingRounds != null) { throw new IllegalArgumentException(DOWNSAMPLING_NOT_SUPPORTED_ERROR_MESSAGE); } - DownsamplingRound.validateRounds(downsampling); - this.downsampling = downsampling; + DownsamplingRound.validateRounds(downsamplingRounds); + this.downsamplingRounds = downsamplingRounds; + if (downsamplingMethod != null && downsamplingRounds == null) { + throw new IllegalArgumentException(DOWNSAMPLING_METHOD_WITHOUT_ROUNDS_ERROR); + } + this.downsamplingMethod = downsamplingMethod; } /** @@ -171,9 +195,10 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { public static DataStreamLifecycle createDataLifecycle( @Nullable Boolean enabled, @Nullable TimeValue dataRetention, - @Nullable List downsampling + @Nullable List downsamplingRounds, + @Nullable DownsampleConfig.SamplingMethod downsamplingMethod ) { - return new DataStreamLifecycle(LifecycleType.DATA, enabled, dataRetention, downsampling); + return new DataStreamLifecycle(LifecycleType.DATA, enabled, dataRetention, downsamplingRounds, downsamplingMethod); } /** @@ -181,7 +206,7 @@ public static DataStreamLifecycle createDataLifecycle( * means it supports only enabling and retention. */ public static DataStreamLifecycle createFailuresLifecycle(@Nullable Boolean enabled, @Nullable TimeValue dataRetention) { - return new DataStreamLifecycle(LifecycleType.FAILURES, enabled, dataRetention, null); + return new DataStreamLifecycle(LifecycleType.FAILURES, enabled, dataRetention, null, null); } /** @@ -304,11 +329,19 @@ public void addWarningHeaderIfDataRetentionNotEffective( /** * The configured downsampling rounds with the `after` and the `fixed_interval` per round. If downsampling is - * not configured then it returns null. + * not configured, then it returns null. + */ + @Nullable + public List downsamplingRounds() { + return downsamplingRounds; + } + + /** + * The configured downsampling method. If downsampling is not configured, then it returns null. */ @Nullable - public List downsampling() { - return downsampling; + public DownsampleConfig.SamplingMethod downsamplingMethod() { + return downsamplingMethod; } @Override @@ -319,13 +352,14 @@ public boolean equals(Object o) { final DataStreamLifecycle that = (DataStreamLifecycle) o; return lifecycleType == that.lifecycleType && Objects.equals(dataRetention, that.dataRetention) - && Objects.equals(downsampling, that.downsampling) + && Objects.equals(downsamplingRounds, that.downsamplingRounds) + && Objects.equals(downsamplingMethod, that.downsamplingMethod) && enabled == that.enabled; } @Override public int hashCode() { - return Objects.hash(lifecycleType, enabled, dataRetention, downsampling); + return Objects.hash(lifecycleType, enabled, dataRetention, downsamplingRounds, downsamplingMethod); } @Override @@ -340,15 +374,18 @@ public void writeTo(StreamOutput out) throws IOException { } if (out.getTransportVersion().onOrAfter(ADDED_ENABLED_FLAG_VERSION)) { if (out.getTransportVersion().supports(INTRODUCE_LIFECYCLE_TEMPLATE)) { - out.writeOptionalCollection(downsampling); + out.writeOptionalCollection(downsamplingRounds); } else { - writeLegacyOptionalValue(downsampling, out, StreamOutput::writeCollection); + writeLegacyOptionalValue(downsamplingRounds, out, StreamOutput::writeCollection); } out.writeBoolean(enabled()); } if (out.getTransportVersion().supports(INTRODUCE_FAILURES_LIFECYCLE)) { lifecycleType.writeTo(out); } + if (out.getTransportVersion().supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM)) { + out.writeOptionalWriteable(downsamplingMethod); + } } public DataStreamLifecycle(StreamInput in) throws IOException { @@ -363,16 +400,19 @@ public DataStreamLifecycle(StreamInput in) throws IOException { } if (in.getTransportVersion().onOrAfter(ADDED_ENABLED_FLAG_VERSION)) { if (in.getTransportVersion().supports(INTRODUCE_LIFECYCLE_TEMPLATE)) { - downsampling = in.readOptionalCollectionAsList(DownsamplingRound::read); + downsamplingRounds = in.readOptionalCollectionAsList(DownsamplingRound::read); } else { - downsampling = readLegacyOptionalValue(in, is -> is.readCollectionAsList(DownsamplingRound::read)); + downsamplingRounds = readLegacyOptionalValue(in, is -> is.readCollectionAsList(DownsamplingRound::read)); } enabled = in.readBoolean(); } else { - downsampling = null; + downsamplingRounds = null; enabled = true; } lifecycleType = in.getTransportVersion().supports(INTRODUCE_FAILURES_LIFECYCLE) ? LifecycleType.read(in) : LifecycleType.DATA; + downsamplingMethod = in.getTransportVersion().supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM) + ? in.readOptionalWriteable(DownsampleConfig.SamplingMethod::read) + : null; } /** @@ -425,8 +465,10 @@ public String toString() { + enabled + ", dataRetention=" + dataRetention - + ", downsampling=" - + downsampling + + ", downsamplingRounds=" + + downsamplingRounds + + ", downsamplingMethod=" + + downsamplingMethod + '}'; } @@ -465,8 +507,11 @@ public XContentBuilder toXContent( } } - if (downsampling != null) { - builder.field(DOWNSAMPLING_FIELD.getPreferredName(), downsampling); + if (downsamplingRounds != null) { + builder.field(DOWNSAMPLING_FIELD.getPreferredName(), downsamplingRounds); + } + if (downsamplingMethod != null) { + builder.field(DOWNSAMPLING_METHOD_FIELD.getPreferredName(), downsamplingMethod.toString()); } if (rolloverConfiguration != null) { builder.field(ROLLOVER_FIELD.getPreferredName()); @@ -522,9 +567,9 @@ public String displayName() { /** * A round represents the configuration for when and how elasticsearch will downsample a backing index. * @param after is a TimeValue configuring how old (based on generation age) should a backing index be before downsampling - * @param config contains the interval that the backing index is going to be downsampled. + * @param fixedInterval contains the interval that the backing index is going to be downsampled. */ - public record DownsamplingRound(TimeValue after, DownsampleConfig config) implements Writeable, ToXContentObject { + public record DownsamplingRound(TimeValue after, DateHistogramInterval fixedInterval) implements Writeable, ToXContentObject { public static final ParseField AFTER_FIELD = new ParseField("after"); public static final ParseField FIXED_INTERVAL_FIELD = new ParseField("fixed_interval"); @@ -533,7 +578,7 @@ public record DownsamplingRound(TimeValue after, DownsampleConfig config) implem private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "downsampling_round", false, - (args, unused) -> new DownsamplingRound((TimeValue) args[0], new DownsampleConfig((DateHistogramInterval) args[1])) + (args, unused) -> new DownsamplingRound((TimeValue) args[0], (DateHistogramInterval) args[1]) ); static { @@ -545,7 +590,7 @@ public record DownsamplingRound(TimeValue after, DownsampleConfig config) implem PARSER.declareField( constructorArg(), p -> new DateHistogramInterval(p.text()), - new ParseField(FIXED_INTERVAL_FIELD.getPreferredName()), + FIXED_INTERVAL_FIELD, ObjectParser.ValueType.STRING ); } @@ -576,19 +621,23 @@ static void validateRounds(List rounds) { + "." ); } - DownsampleConfig.validateSourceAndTargetIntervals(previous.config(), round.config()); + DownsampleConfig.validateSourceAndTargetIntervals(previous.fixedInterval(), round.fixedInterval()); } } } public static DownsamplingRound read(StreamInput in) throws IOException { - return new DownsamplingRound(in.readTimeValue(), new DownsampleConfig(in)); + TimeValue after = in.readTimeValue(); + DateHistogramInterval fixedInterval = in.getTransportVersion().supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM) + ? new DateHistogramInterval(in) + : new DownsampleConfig(in).getFixedInterval(); + return new DownsamplingRound(after, fixedInterval); } public DownsamplingRound { - if (config.getFixedInterval().estimateMillis() < FIVE_MINUTES_MILLIS) { + if (fixedInterval.estimateMillis() < FIVE_MINUTES_MILLIS) { throw new IllegalArgumentException( - "A downsampling round must have a fixed interval of at least five minutes but found: " + config.getFixedInterval() + "A downsampling round must have a fixed interval of at least five minutes but found: " + fixedInterval ); } } @@ -596,14 +645,18 @@ public static DownsamplingRound read(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { out.writeTimeValue(after); - out.writeWriteable(config); + if (out.getTransportVersion().supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM)) { + out.writeWriteable(fixedInterval); + } else { + out.writeWriteable(new DownsampleConfig(fixedInterval, null)); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(AFTER_FIELD.getPreferredName(), after.getStringRep()); - config.toXContentFragment(builder); + builder.field(FIXED_INTERVAL_FIELD.getPreferredName(), fixedInterval().toString()); builder.endObject(); return builder; } @@ -625,9 +678,16 @@ public String toString() { public static Template createDataLifecycleTemplate( boolean enabled, TimeValue dataRetention, - List downsampling + List downsamplingRounds, + DownsampleConfig.SamplingMethod downsamplingMethod ) { - return new Template(LifecycleType.DATA, enabled, ResettableValue.create(dataRetention), ResettableValue.create(downsampling)); + return new Template( + LifecycleType.DATA, + enabled, + ResettableValue.create(dataRetention), + ResettableValue.create(downsamplingRounds), + ResettableValue.create(downsamplingMethod) + ); } /** @@ -637,9 +697,10 @@ public static Template createDataLifecycleTemplate( public static Template createDataLifecycleTemplate( boolean enabled, ResettableValue dataRetention, - ResettableValue> downsampling + ResettableValue> downsamplingRounds, + ResettableValue downsamplingMethod ) { - return new Template(LifecycleType.DATA, enabled, dataRetention, downsampling); + return new Template(LifecycleType.DATA, enabled, dataRetention, downsamplingRounds, downsamplingMethod); } /** @@ -647,7 +708,13 @@ public static Template createDataLifecycleTemplate( * means it supports only setting the enabled and the retention. */ public static Template createFailuresLifecycleTemplate(boolean enabled, TimeValue dataRetention) { - return new Template(LifecycleType.FAILURES, enabled, ResettableValue.create(dataRetention), ResettableValue.undefined()); + return new Template( + LifecycleType.FAILURES, + enabled, + ResettableValue.create(dataRetention), + ResettableValue.undefined(), + ResettableValue.undefined() + ); } /** @@ -658,24 +725,34 @@ public record Template( LifecycleType lifecycleType, boolean enabled, ResettableValue dataRetention, - ResettableValue> downsampling + ResettableValue> downsamplingRounds, + ResettableValue downsamplingMethod ) implements ToXContentObject, Writeable { Template( LifecycleType lifecycleType, boolean enabled, TimeValue dataRetention, - List downsampling + List downsamplingRounds, + DownsampleConfig.SamplingMethod downsamplingMethod ) { - this(lifecycleType, enabled, ResettableValue.create(dataRetention), ResettableValue.create(downsampling)); + this( + lifecycleType, + enabled, + ResettableValue.create(dataRetention), + ResettableValue.create(downsamplingRounds), + ResettableValue.create(downsamplingMethod) + ); } public Template { - if (lifecycleType == LifecycleType.FAILURES && downsampling.get() != null) { + if (lifecycleType == LifecycleType.FAILURES && downsamplingRounds.get() != null) { throw new IllegalArgumentException(DOWNSAMPLING_NOT_SUPPORTED_ERROR_MESSAGE); } - if (downsampling.isDefined() && downsampling.get() != null) { - DownsamplingRound.validateRounds(downsampling.get()); + if (downsamplingRounds.isDefined() && downsamplingRounds.get() != null) { + DownsamplingRound.validateRounds(downsamplingRounds.get()); + } else if (downsamplingMethod.isDefined() && downsamplingMethod.get() != null) { + throw new IllegalArgumentException(DOWNSAMPLING_METHOD_WITHOUT_ROUNDS_ERROR); } } @@ -683,6 +760,7 @@ public record Template( LifecycleType.DATA, true, ResettableValue.undefined(), + ResettableValue.undefined(), ResettableValue.undefined() ); @@ -694,7 +772,8 @@ public record Template( lt, args[0] == null || (boolean) args[0], args[1] == null ? ResettableValue.undefined() : (ResettableValue) args[1], - args[2] == null ? ResettableValue.undefined() : (ResettableValue>) args[2] + args[2] == null ? ResettableValue.undefined() : (ResettableValue>) args[2], + args[3] == null ? ResettableValue.undefined() : (ResettableValue) args[3] ) ); @@ -713,6 +792,10 @@ public record Template( return ResettableValue.create(AbstractObjectParser.parseArray(p, null, DownsamplingRound::fromXContent)); } }, DOWNSAMPLING_FIELD, ObjectParser.ValueType.OBJECT_ARRAY_OR_NULL); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { + String value = p.textOrNull(); + return value == null ? ResettableValue.reset() : ResettableValue.create(DownsampleConfig.SamplingMethod.fromString(value)); + }, DOWNSAMPLING_METHOD_FIELD, ObjectParser.ValueType.STRING_OR_NULL); } @Override @@ -727,15 +810,18 @@ public void writeTo(StreamOutput out) throws IOException { } if (out.getTransportVersion().onOrAfter(ADDED_ENABLED_FLAG_VERSION)) { if (out.getTransportVersion().supports(INTRODUCE_LIFECYCLE_TEMPLATE)) { - ResettableValue.write(out, downsampling, StreamOutput::writeCollection); + ResettableValue.write(out, downsamplingRounds, StreamOutput::writeCollection); } else { - writeLegacyValue(out, downsampling, StreamOutput::writeCollection); + writeLegacyValue(out, downsamplingRounds, StreamOutput::writeCollection); } out.writeBoolean(enabled); } if (out.getTransportVersion().supports(INTRODUCE_FAILURES_LIFECYCLE)) { lifecycleType.writeTo(out); } + if (out.getTransportVersion().supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM)) { + ResettableValue.write(out, downsamplingMethod, StreamOutput::writeWriteable); + } } /** @@ -775,7 +861,7 @@ static ResettableValue readLegacyValues(StreamInput in, Writeable.Reader< public static Template read(StreamInput in) throws IOException { boolean enabled = true; ResettableValue dataRetention = ResettableValue.undefined(); - ResettableValue> downsampling = ResettableValue.undefined(); + ResettableValue> downsamplingRounds = ResettableValue.undefined(); // The order of the fields is like this for bwc reasons if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { @@ -787,16 +873,20 @@ public static Template read(StreamInput in) throws IOException { } if (in.getTransportVersion().onOrAfter(ADDED_ENABLED_FLAG_VERSION)) { if (in.getTransportVersion().supports(INTRODUCE_LIFECYCLE_TEMPLATE)) { - downsampling = ResettableValue.read(in, i -> i.readCollectionAsList(DownsamplingRound::read)); + downsamplingRounds = ResettableValue.read(in, i -> i.readCollectionAsList(DownsamplingRound::read)); } else { - downsampling = readLegacyValues(in, i -> i.readCollectionAsList(DownsamplingRound::read)); + downsamplingRounds = readLegacyValues(in, i -> i.readCollectionAsList(DownsamplingRound::read)); } enabled = in.readBoolean(); } var lifecycleTarget = in.getTransportVersion().supports(INTRODUCE_FAILURES_LIFECYCLE) ? LifecycleType.read(in) : LifecycleType.DATA; - return new Template(lifecycleTarget, enabled, dataRetention, downsampling); + ResettableValue downsamplingMethod = in.getTransportVersion() + .supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM) + ? ResettableValue.read(in, DownsampleConfig.SamplingMethod::read) + : ResettableValue.undefined(); + return new Template(lifecycleTarget, enabled, dataRetention, downsamplingRounds, downsamplingMethod); } public static Template dataLifecycleTemplatefromXContent(XContentParser parser) throws IOException { @@ -830,7 +920,13 @@ public XContentBuilder toXContent( builder.startObject(); builder.field(ENABLED_FIELD.getPreferredName(), enabled); dataRetention.toXContent(builder, params, DATA_RETENTION_FIELD.getPreferredName(), TimeValue::getStringRep); - downsampling.toXContent(builder, params, DOWNSAMPLING_FIELD.getPreferredName()); + downsamplingRounds.toXContent(builder, params, DOWNSAMPLING_FIELD.getPreferredName()); + downsamplingMethod.toXContent( + builder, + params, + DOWNSAMPLING_METHOD_FIELD.getPreferredName(), + DownsampleConfig.SamplingMethod::toString + ); if (rolloverConfiguration != null) { builder.field(ROLLOVER_FIELD.getPreferredName()); rolloverConfiguration.evaluateAndConvertToXContent( @@ -844,7 +940,7 @@ public XContentBuilder toXContent( } public DataStreamLifecycle toDataStreamLifecycle() { - return new DataStreamLifecycle(lifecycleType, enabled, dataRetention.get(), downsampling.get()); + return new DataStreamLifecycle(lifecycleType, enabled, dataRetention.get(), downsamplingRounds.get(), downsamplingMethod.get()); } } @@ -881,7 +977,9 @@ public static class Builder { @Nullable private TimeValue dataRetention = null; @Nullable - private List downsampling = null; + private List downsamplingRounds = null; + @Nullable + private DownsampleConfig.SamplingMethod downsamplingMethod = null; private Builder(LifecycleType lifecycleType) { this.lifecycleType = lifecycleType; @@ -891,21 +989,24 @@ private Builder(DataStreamLifecycle.Template template) { lifecycleType = template.lifecycleType(); enabled = template.enabled(); dataRetention = template.dataRetention().get(); - downsampling = template.downsampling().get(); + downsamplingRounds = template.downsamplingRounds().get(); + downsamplingMethod = template.downsamplingMethod().get(); } private Builder(DataStreamLifecycle lifecycle) { lifecycleType = lifecycle.lifecycleType; enabled = lifecycle.enabled(); dataRetention = lifecycle.dataRetention(); - downsampling = lifecycle.downsampling(); + downsamplingRounds = lifecycle.downsamplingRounds(); + downsamplingMethod = lifecycle.downsamplingMethod(); } public Builder composeTemplate(DataStreamLifecycle.Template template) { assert lifecycleType == template.lifecycleType() : "Trying to compose templates with different lifecycle types"; enabled(template.enabled()); dataRetention(template.dataRetention()); - downsampling(template.downsampling()); + downsamplingRounds(template.downsamplingRounds()); + downsamplingMethod(template.downsamplingMethod()); return this; } @@ -926,24 +1027,36 @@ public Builder dataRetention(@Nullable TimeValue dataRetention) { return this; } - public Builder downsampling(ResettableValue> downsampling) { + public Builder downsamplingRounds(ResettableValue> downsampling) { if (downsampling.isDefined()) { - this.downsampling = downsampling.get(); + this.downsamplingRounds = downsampling.get(); + } + return this; + } + + public Builder downsamplingRounds(@Nullable List downsampling) { + this.downsamplingRounds = downsampling; + return this; + } + + public Builder downsamplingMethod(ResettableValue downsamplingMethod) { + if (downsamplingMethod.isDefined()) { + this.downsamplingMethod = downsamplingMethod.get(); } return this; } - public Builder downsampling(@Nullable List downsampling) { - this.downsampling = downsampling; + public Builder downsamplingMethod(@Nullable DownsampleConfig.SamplingMethod downsamplingMethod) { + this.downsamplingMethod = downsamplingMethod; return this; } public DataStreamLifecycle build() { - return new DataStreamLifecycle(lifecycleType, enabled, dataRetention, downsampling); + return new DataStreamLifecycle(lifecycleType, enabled, dataRetention, downsamplingRounds, downsamplingMethod); } public Template buildTemplate() { - return new Template(lifecycleType, enabled, dataRetention, downsampling); + return new Template(lifecycleType, enabled, dataRetention, downsamplingRounds, downsamplingMethod); } } diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index feb7f45953aa2..9587c1db4501b 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -191,6 +191,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion UPGRADE_TO_LUCENE_10_3_1 = def(9_041_0_00, Version.LUCENE_10_3_1); public static final IndexVersion REENABLED_TIMESTAMP_DOC_VALUES_SPARSE_INDEX = def(9_042_0_00, Version.LUCENE_10_3_1); public static final IndexVersion SKIPPERS_ENABLED_BY_DEFAULT = def(9_043_0_00, Version.LUCENE_10_3_1); + public static final IndexVersion BFLOAT16_HNSW_SUPPORT = def(9_044_0_00, Version.LUCENE_10_3_1); public static final IndexVersion UPGRADE_TO_LUCENE_10_4_0 = def(9_060_00_0, Version.LUCENE_10_4_0); diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormat.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormat.java index fbdef488b8318..fb43ae176dedc 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormat.java @@ -118,7 +118,7 @@ private static boolean getOptimizedMergeEnabledDefault() { final int skipIndexIntervalSize; final int minDocsPerOrdinalForRangeEncoding; - private final boolean enableOptimizedMerge; + final boolean enableOptimizedMerge; /** Default constructor. */ public ES819TSDBDocValuesFormat() { diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/BFloat16.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/BFloat16.java index 8d25ab54d8ca1..8b50a39fe01af 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/BFloat16.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/BFloat16.java @@ -29,12 +29,15 @@ public static short floatToBFloat16(float f) { return (short) (Float.floatToIntBits(f) >>> 16); } + public static float truncateToBFloat16(float f) { + return Float.intBitsToFloat(Float.floatToIntBits(f) & 0xffff0000); + } + public static float bFloat16ToFloat(short bf) { return Float.intBitsToFloat(bf << 16); } public static void floatToBFloat16(float[] floats, ShortBuffer bFloats) { - assert bFloats.remaining() == floats.length; assert bFloats.order() == ByteOrder.LITTLE_ENDIAN; for (float v : floats) { bFloats.put(floatToBFloat16(v)); @@ -49,7 +52,6 @@ public static void bFloat16ToFloat(byte[] bfBytes, float[] floats) { } public static void bFloat16ToFloat(ShortBuffer bFloats, float[] floats) { - assert floats.length == bFloats.remaining(); assert bFloats.order() == ByteOrder.LITTLE_ENDIAN; for (int i = 0; i < floats.length; i++) { floats[i] = bFloat16ToFloat(bFloats.get()); diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/diskbbq/ES920DiskBBQVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/diskbbq/ES920DiskBBQVectorsFormat.java index 99bc9a9d7bdb2..25b711da8c18f 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/diskbbq/ES920DiskBBQVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/diskbbq/ES920DiskBBQVectorsFormat.java @@ -18,6 +18,8 @@ import org.elasticsearch.index.codec.vectors.DirectIOCapableFlatVectorsFormat; import org.elasticsearch.index.codec.vectors.OptimizedScalarQuantizer; import org.elasticsearch.index.codec.vectors.es93.DirectIOCapableLucene99FlatVectorsFormat; +import org.elasticsearch.index.codec.vectors.es93.ES93BFloat16FlatVectorsFormat; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import java.io.IOException; import java.util.Map; @@ -61,9 +63,14 @@ public class ES920DiskBBQVectorsFormat extends KnnVectorsFormat { private static final DirectIOCapableFlatVectorsFormat float32VectorFormat = new DirectIOCapableLucene99FlatVectorsFormat( FlatVectorScorerUtil.getLucene99FlatVectorsScorer() ); + private static final DirectIOCapableFlatVectorsFormat bfloat16VectorFormat = new ES93BFloat16FlatVectorsFormat( + FlatVectorScorerUtil.getLucene99FlatVectorsScorer() + ); private static final Map supportedFormats = Map.of( float32VectorFormat.getName(), - float32VectorFormat + float32VectorFormat, + bfloat16VectorFormat.getName(), + bfloat16VectorFormat ); // This dynamically sets the cluster probe based on the `k` requested and the number of clusters. @@ -78,14 +85,19 @@ public class ES920DiskBBQVectorsFormat extends KnnVectorsFormat { private final int vectorPerCluster; private final int centroidsPerParentCluster; - private final boolean useDirectIO; private final DirectIOCapableFlatVectorsFormat rawVectorFormat; + private final boolean useDirectIO; public ES920DiskBBQVectorsFormat(int vectorPerCluster, int centroidsPerParentCluster) { - this(vectorPerCluster, centroidsPerParentCluster, false); + this(vectorPerCluster, centroidsPerParentCluster, DenseVectorFieldMapper.ElementType.FLOAT, false); } - public ES920DiskBBQVectorsFormat(int vectorPerCluster, int centroidsPerParentCluster, boolean useDirectIO) { + public ES920DiskBBQVectorsFormat( + int vectorPerCluster, + int centroidsPerParentCluster, + DenseVectorFieldMapper.ElementType elementType, + boolean useDirectIO + ) { super(NAME); if (vectorPerCluster < MIN_VECTORS_PER_CLUSTER || vectorPerCluster > MAX_VECTORS_PER_CLUSTER) { throw new IllegalArgumentException( @@ -109,8 +121,12 @@ public ES920DiskBBQVectorsFormat(int vectorPerCluster, int centroidsPerParentClu } this.vectorPerCluster = vectorPerCluster; this.centroidsPerParentCluster = centroidsPerParentCluster; + this.rawVectorFormat = switch (elementType) { + case FLOAT -> float32VectorFormat; + case BFLOAT16 -> bfloat16VectorFormat; + default -> throw new IllegalArgumentException("Unsupported element type " + elementType); + }; this.useDirectIO = useDirectIO; - this.rawVectorFormat = float32VectorFormat; } /** Constructs a format using the given graph construction parameters and scalar quantization. */ diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93BinaryQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93BinaryQuantizedVectorsFormat.java index ed224c82a5aaa..290b010fef78c 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93BinaryQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93BinaryQuantizedVectorsFormat.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.codec.vectors.es818.ES818BinaryFlatVectorsScorer; import org.elasticsearch.index.codec.vectors.es818.ES818BinaryQuantizedVectorsReader; import org.elasticsearch.index.codec.vectors.es818.ES818BinaryQuantizedVectorsWriter; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import java.io.IOException; @@ -97,10 +98,10 @@ public class ES93BinaryQuantizedVectorsFormat extends AbstractFlatVectorsFormat private final ES93GenericFlatVectorsFormat rawFormat; public ES93BinaryQuantizedVectorsFormat() { - this(ES93GenericFlatVectorsFormat.ElementType.STANDARD, false); + this(DenseVectorFieldMapper.ElementType.FLOAT, false); } - public ES93BinaryQuantizedVectorsFormat(ES93GenericFlatVectorsFormat.ElementType elementType, boolean useDirectIO) { + public ES93BinaryQuantizedVectorsFormat(DenseVectorFieldMapper.ElementType elementType, boolean useDirectIO) { super(NAME); rawFormat = new ES93GenericFlatVectorsFormat(elementType, useDirectIO); } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormat.java new file mode 100644 index 0000000000000..af373d66fe827 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormat.java @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.search.AcceptDocs; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector; +import org.apache.lucene.util.hnsw.RandomVectorScorer; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; + +public class ES93FlatVectorFormat extends KnnVectorsFormat { + + static final String NAME = "ES93FlatVectorFormat"; + + private final FlatVectorsFormat format; + + /** + * Sole constructor + */ + public ES93FlatVectorFormat() { + super(NAME); + format = new ES93GenericFlatVectorsFormat(); + } + + public ES93FlatVectorFormat(DenseVectorFieldMapper.ElementType elementType) { + super(NAME); + assert elementType != DenseVectorFieldMapper.ElementType.BIT : "ES815BitFlatVectorFormat should be used for bits"; + format = new ES93GenericFlatVectorsFormat(elementType, false); + } + + @Override + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return format.fieldsWriter(state); + } + + @Override + public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { + return new ES93FlatVectorReader(format.fieldsReader(state)); + } + + @Override + public int getMaxDimensions(String fieldName) { + return MAX_DIMS_COUNT; + } + + static class ES93FlatVectorReader extends KnnVectorsReader { + + private final FlatVectorsReader reader; + + ES93FlatVectorReader(FlatVectorsReader reader) { + super(); + this.reader = reader; + } + + @Override + public void checkIntegrity() throws IOException { + reader.checkIntegrity(); + } + + @Override + public FloatVectorValues getFloatVectorValues(String field) throws IOException { + return reader.getFloatVectorValues(field); + } + + @Override + public ByteVectorValues getByteVectorValues(String field) throws IOException { + return reader.getByteVectorValues(field); + } + + @Override + public void search(String field, float[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { + collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); + } + + private void collectAllMatchingDocs(KnnCollector knnCollector, AcceptDocs acceptDocs, RandomVectorScorer scorer) + throws IOException { + OrdinalTranslatedKnnCollector collector = new OrdinalTranslatedKnnCollector(knnCollector, scorer::ordToDoc); + Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs.bits()); + for (int i = 0; i < scorer.maxOrd(); i++) { + if (acceptedOrds == null || acceptedOrds.get(i)) { + collector.collect(i, scorer.score(i)); + collector.incVisitedCount(1); + } + } + assert collector.earlyTerminated() == false; + } + + @Override + public void search(String field, byte[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { + collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); + } + + @Override + public Map getOffHeapByteSize(FieldInfo fieldInfo) { + return reader.getOffHeapByteSize(fieldInfo); + } + + @Override + public void close() throws IOException { + reader.close(); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93GenericFlatVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93GenericFlatVectorsFormat.java index d3bded4080088..0eedc3f16cdfb 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93GenericFlatVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93GenericFlatVectorsFormat.java @@ -17,19 +17,13 @@ import org.apache.lucene.index.SegmentWriteState; import org.elasticsearch.index.codec.vectors.AbstractFlatVectorsFormat; import org.elasticsearch.index.codec.vectors.DirectIOCapableFlatVectorsFormat; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import java.io.IOException; import java.util.Map; public class ES93GenericFlatVectorsFormat extends AbstractFlatVectorsFormat { - // TODO: replace with DenseVectorFieldMapper.ElementType - public enum ElementType { - STANDARD, - BIT, // only supports byte[] - BFLOAT16 // only supports float[] - } - static final String NAME = "ES93GenericFlatVectorsFormat"; static final String VECTOR_FORMAT_INFO_EXTENSION = "vfi"; static final String META_CODEC_NAME = "ES93GenericFlatVectorsFormatMeta"; @@ -44,7 +38,7 @@ public enum ElementType { VERSION_CURRENT ); - private static final DirectIOCapableFlatVectorsFormat standardVectorFormat = new DirectIOCapableLucene99FlatVectorsFormat( + private static final DirectIOCapableFlatVectorsFormat defaultVectorFormat = new DirectIOCapableLucene99FlatVectorsFormat( FlatVectorScorerUtil.getLucene99FlatVectorsScorer() ); private static final DirectIOCapableFlatVectorsFormat bitVectorFormat = new DirectIOCapableLucene99FlatVectorsFormat( @@ -61,10 +55,10 @@ public String getName() { ); private static final Map supportedFormats = Map.of( + defaultVectorFormat.getName(), + defaultVectorFormat, bitVectorFormat.getName(), bitVectorFormat, - standardVectorFormat.getName(), - standardVectorFormat, bfloat16VectorFormat.getName(), bfloat16VectorFormat ); @@ -73,13 +67,13 @@ public String getName() { private final boolean useDirectIO; public ES93GenericFlatVectorsFormat() { - this(ElementType.STANDARD, false); + this(DenseVectorFieldMapper.ElementType.FLOAT, false); } - public ES93GenericFlatVectorsFormat(ElementType elementType, boolean useDirectIO) { + public ES93GenericFlatVectorsFormat(DenseVectorFieldMapper.ElementType elementType, boolean useDirectIO) { super(NAME); writeFormat = switch (elementType) { - case STANDARD -> standardVectorFormat; + case FLOAT, BYTE -> defaultVectorFormat; case BIT -> bitVectorFormat; case BFLOAT16 -> bfloat16VectorFormat; }; diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBinaryQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBinaryQuantizedVectorsFormat.java index 8565506903566..38715ae3481f4 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBinaryQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBinaryQuantizedVectorsFormat.java @@ -27,6 +27,7 @@ import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.elasticsearch.index.codec.vectors.AbstractHnswVectorsFormat; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import java.io.IOException; import java.util.concurrent.ExecutorService; @@ -49,7 +50,7 @@ public ES93HnswBinaryQuantizedVectorsFormat() { * * @param useDirectIO whether to use direct IO when reading raw vectors */ - public ES93HnswBinaryQuantizedVectorsFormat(ES93GenericFlatVectorsFormat.ElementType elementType, boolean useDirectIO) { + public ES93HnswBinaryQuantizedVectorsFormat(DenseVectorFieldMapper.ElementType elementType, boolean useDirectIO) { super(NAME); flatVectorsFormat = new ES93BinaryQuantizedVectorsFormat(elementType, useDirectIO); } @@ -64,7 +65,7 @@ public ES93HnswBinaryQuantizedVectorsFormat(ES93GenericFlatVectorsFormat.Element public ES93HnswBinaryQuantizedVectorsFormat( int maxConn, int beamWidth, - ES93GenericFlatVectorsFormat.ElementType elementType, + DenseVectorFieldMapper.ElementType elementType, boolean useDirectIO ) { super(NAME, maxConn, beamWidth); @@ -85,7 +86,7 @@ public ES93HnswBinaryQuantizedVectorsFormat( public ES93HnswBinaryQuantizedVectorsFormat( int maxConn, int beamWidth, - ES93GenericFlatVectorsFormat.ElementType elementType, + DenseVectorFieldMapper.ElementType elementType, boolean useDirectIO, int numMergeWorkers, ExecutorService mergeExec diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java new file mode 100644 index 0000000000000..5c481db3d1260 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorScorer; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsReader; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsWriter; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.elasticsearch.index.codec.vectors.AbstractHnswVectorsFormat; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; + +import java.io.IOException; +import java.util.concurrent.ExecutorService; + +public class ES93HnswScalarQuantizedVectorsFormat extends AbstractHnswVectorsFormat { + + static final String NAME = "ES93HnswScalarQuantizedVectorsFormat"; + + static final Lucene104ScalarQuantizedVectorScorer flatVectorScorer = new Lucene104ScalarQuantizedVectorScorer( + FlatVectorScorerUtil.getLucene99FlatVectorsScorer() + ); + + private final Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding; + private final FlatVectorsFormat rawVectorFormat; + + public ES93HnswScalarQuantizedVectorsFormat() { + super(NAME); + this.encoding = Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT; + this.rawVectorFormat = new ES93GenericFlatVectorsFormat(DenseVectorFieldMapper.ElementType.FLOAT, false); + } + + public ES93HnswScalarQuantizedVectorsFormat( + int maxConn, + int beamWidth, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding, + DenseVectorFieldMapper.ElementType elementType, + boolean useDirectIO + ) { + super(NAME, maxConn, beamWidth); + this.encoding = encoding; + this.rawVectorFormat = new ES93GenericFlatVectorsFormat(elementType, useDirectIO); + } + + public ES93HnswScalarQuantizedVectorsFormat( + int maxConn, + int beamWidth, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding, + DenseVectorFieldMapper.ElementType elementType, + boolean useDirectIO, + int numMergeWorkers, + ExecutorService mergeExec + ) { + super(NAME, maxConn, beamWidth, numMergeWorkers, mergeExec); + this.encoding = encoding; + this.rawVectorFormat = new ES93GenericFlatVectorsFormat(elementType, useDirectIO); + } + + @Override + protected FlatVectorsFormat flatVectorsFormat() { + return rawVectorFormat; + } + + @Override + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return new Lucene99HnswVectorsWriter( + state, + maxConn, + beamWidth, + new Lucene104ScalarQuantizedVectorsWriter(state, encoding, rawVectorFormat.fieldsWriter(state), flatVectorScorer), + numMergeWorkers, + mergeExec, + 0 + ); + } + + @Override + public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { + return new Lucene99HnswVectorsReader( + state, + new Lucene104ScalarQuantizedVectorsReader(state, rawVectorFormat.fieldsReader(state), flatVectorScorer) + ); + } + + @Override + public String toString() { + return NAME + + "(name=" + + NAME + + ", maxConn=" + + maxConn + + ", beamWidth=" + + beamWidth + + ", bits=" + + encoding.getBits() + + ", flatVectorScorer=" + + flatVectorScorer + + ", rawVectorFormat=" + + rawVectorFormat + + ")"; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswVectorsFormat.java index ac62c8a995362..1b25be530ad6b 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswVectorsFormat.java @@ -17,6 +17,7 @@ import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.elasticsearch.index.codec.vectors.AbstractHnswVectorsFormat; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import java.io.IOException; import java.util.concurrent.ExecutorService; @@ -32,26 +33,25 @@ public ES93HnswVectorsFormat() { flatVectorsFormat = new ES93GenericFlatVectorsFormat(); } - public ES93HnswVectorsFormat(ES93GenericFlatVectorsFormat.ElementType elementType, boolean useDirectIO) { + public ES93HnswVectorsFormat(DenseVectorFieldMapper.ElementType elementType) { super(NAME); - flatVectorsFormat = new ES93GenericFlatVectorsFormat(elementType, useDirectIO); + flatVectorsFormat = new ES93GenericFlatVectorsFormat(elementType, false); } - public ES93HnswVectorsFormat(int maxConn, int beamWidth, ES93GenericFlatVectorsFormat.ElementType elementType, boolean useDirectIO) { + public ES93HnswVectorsFormat(int maxConn, int beamWidth, DenseVectorFieldMapper.ElementType elementType) { super(NAME, maxConn, beamWidth); - flatVectorsFormat = new ES93GenericFlatVectorsFormat(elementType, useDirectIO); + flatVectorsFormat = new ES93GenericFlatVectorsFormat(elementType, false); } public ES93HnswVectorsFormat( int maxConn, int beamWidth, - ES93GenericFlatVectorsFormat.ElementType elementType, - boolean useDirectIO, + DenseVectorFieldMapper.ElementType elementType, int numMergeWorkers, ExecutorService mergeExec ) { super(NAME, maxConn, beamWidth, numMergeWorkers, mergeExec); - flatVectorsFormat = new ES93GenericFlatVectorsFormat(elementType, useDirectIO); + flatVectorsFormat = new ES93GenericFlatVectorsFormat(elementType, false); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java new file mode 100644 index 0000000000000..33682fa8b22ec --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorScorer; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsReader; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsWriter; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; + +import java.io.IOException; + +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; + +public class ES93ScalarQuantizedVectorsFormat extends KnnVectorsFormat { + + static final String NAME = "ES93ScalarQuantizedVectorsFormat"; + + static final Lucene104ScalarQuantizedVectorScorer flatVectorScorer = new Lucene104ScalarQuantizedVectorScorer( + FlatVectorScorerUtil.getLucene99FlatVectorsScorer() + ); + + private final Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding; + private final FlatVectorsFormat rawVectorFormat; + + public ES93ScalarQuantizedVectorsFormat() { + this(DenseVectorFieldMapper.ElementType.FLOAT, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT); + } + + public ES93ScalarQuantizedVectorsFormat(DenseVectorFieldMapper.ElementType elementType) { + this(elementType, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT); + } + + public ES93ScalarQuantizedVectorsFormat( + DenseVectorFieldMapper.ElementType elementType, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding + ) { + super(NAME); + assert elementType != DenseVectorFieldMapper.ElementType.BIT : "BIT should not be used with scalar quantization"; + this.encoding = encoding; + this.rawVectorFormat = new ES93GenericFlatVectorsFormat(elementType, false); + } + + @Override + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return new Lucene104ScalarQuantizedVectorsWriter(state, encoding, rawVectorFormat.fieldsWriter(state), flatVectorScorer); + } + + @Override + public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { + return new Lucene104ScalarQuantizedVectorsReader(state, rawVectorFormat.fieldsReader(state), flatVectorScorer); + } + + @Override + public int getMaxDimensions(String fieldName) { + return MAX_DIMS_COUNT; + } + + @Override + public String toString() { + return NAME + + "(name=" + + NAME + + ", bits=" + + encoding.getBits() + + ", flatVectorScorer=" + + flatVectorScorer + + ", rawVectorFormat=" + + rawVectorFormat + + ")"; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java index 8544ddd0194f3..aa7b395519802 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java @@ -26,6 +26,9 @@ * {@code _source}. */ public abstract class IgnoreMalformedStoredValues { + + public static final String IGNORE_MALFORMED_FIELD_NAME_SUFFIX = "._ignore_malformed"; + /** * Creates a stored field that stores malformed data to be used in synthetic source. * Name of the stored field is original name of the field with added conventional suffix. @@ -143,6 +146,6 @@ public void reset() { } public static String name(String fieldName) { - return fieldName + "._ignore_malformed"; + return fieldName + IGNORE_MALFORMED_FIELD_NAME_SUFFIX; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 279a37fcc2061..92353bfecc66e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -58,6 +58,7 @@ public class MapperFeatures implements FeatureSpecification { "mapper.ignore_dynamic_field_names_beyond_limit" ); static final NodeFeature EXCLUDE_VECTORS_DOCVALUE_BUGFIX = new NodeFeature("mapper.exclude_vectors_docvalue_bugfix"); + public static final NodeFeature HNSW_BFLOAT16_ON_DISK_RESCORING = new NodeFeature("mapper.vectors.hnsw_bfloat16_on_disk_rescoring"); @Override public Set getTestFeatures() { @@ -99,7 +100,8 @@ public Set getTestFeatures() { DISKBBQ_ON_DISK_RESCORING, PROVIDE_INDEX_SORT_SETTING_DEFAULTS, INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_FIELD_NAME_LIMIT, - EXCLUDE_VECTORS_DOCVALUE_BUGFIX + EXCLUDE_VECTORS_DOCVALUE_BUGFIX, + HNSW_BFLOAT16_ON_DISK_RESCORING ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldType.java index efd2bafd8e2c9..de03b116f0c19 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldType.java @@ -20,6 +20,7 @@ */ public abstract class TextFamilyFieldType extends StringFieldType { + public static final String FALLBACK_FIELD_NAME_SUFFIX = "._original"; private final boolean isSyntheticSourceEnabled; private final boolean isWithinMultiField; @@ -51,7 +52,7 @@ public boolean isWithinMultiField() { * stored for whatever reason. */ public String syntheticSourceFallbackFieldName() { - return name() + "._original"; + return name() + FALLBACK_FIELD_NAME_SUFFIX; } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/DenseVectorFromBinaryBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/DenseVectorFromBinaryBlockLoader.java index f5f9e8dc88295..7e729eca933e5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/DenseVectorFromBinaryBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/DenseVectorFromBinaryBlockLoader.java @@ -50,6 +50,7 @@ public AllReader reader(LeafReaderContext context) throws IOException { } return switch (elementType) { case FLOAT -> new FloatDenseVectorFromBinary(docValues, dims, indexVersion); + case BFLOAT16 -> new BFloat16DenseVectorFromBinary(docValues, dims, indexVersion); case BYTE -> new ByteDenseVectorFromBinary(docValues, dims, indexVersion); case BIT -> new BitDenseVectorFromBinary(docValues, dims, indexVersion); }; @@ -132,6 +133,29 @@ public String toString() { } } + private static class BFloat16DenseVectorFromBinary extends AbstractDenseVectorFromBinary { + BFloat16DenseVectorFromBinary(BinaryDocValues docValues, int dims, IndexVersion indexVersion) { + super(docValues, dims, indexVersion, new float[dims]); + } + + @Override + protected void writeScratchToBuilder(float[] scratch, BlockLoader.FloatBuilder builder) { + for (float value : scratch) { + builder.appendFloat(value); + } + } + + @Override + protected void decodeDenseVector(BytesRef bytesRef, float[] scratch) { + VectorEncoderDecoder.decodeBFloat16DenseVector(bytesRef, scratch); + } + + @Override + public String toString() { + return "BFloat16DenseVectorFromBinary.Bytes"; + } + } + private static class ByteDenseVectorFromBinary extends AbstractDenseVectorFromBinary { ByteDenseVectorFromBinary(BinaryDocValues docValues, int dims, IndexVersion indexVersion) { this(docValues, dims, indexVersion, dims); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index ce920b4638468..e2753aa0c6b10 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -12,6 +12,7 @@ import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat; import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Field; @@ -47,14 +48,16 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; -import org.elasticsearch.index.codec.vectors.ES813FlatVectorFormat; -import org.elasticsearch.index.codec.vectors.ES813Int8FlatVectorFormat; -import org.elasticsearch.index.codec.vectors.ES814HnswScalarQuantizedVectorsFormat; +import org.elasticsearch.index.codec.vectors.BFloat16; import org.elasticsearch.index.codec.vectors.ES815BitFlatVectorFormat; import org.elasticsearch.index.codec.vectors.ES815HnswBitVectorsFormat; import org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat; -import org.elasticsearch.index.codec.vectors.es818.ES818BinaryQuantizedVectorsFormat; -import org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsFormat; +import org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat; +import org.elasticsearch.index.codec.vectors.es93.ES93FlatVectorFormat; +import org.elasticsearch.index.codec.vectors.es93.ES93HnswBinaryQuantizedVectorsFormat; +import org.elasticsearch.index.codec.vectors.es93.ES93HnswScalarQuantizedVectorsFormat; +import org.elasticsearch.index.codec.vectors.es93.ES93HnswVectorsFormat; +import org.elasticsearch.index.codec.vectors.es93.ES93ScalarQuantizedVectorsFormat; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.ArraySourceValueFetcher; @@ -125,15 +128,10 @@ */ public class DenseVectorFieldMapper extends FieldMapper { public static final String COSINE_MAGNITUDE_FIELD_SUFFIX = "._magnitude"; - private static final float EPS = 1e-3f; public static final int BBQ_MIN_DIMS = 64; private static final boolean DEFAULT_HNSW_EARLY_TERMINATION = false; - public static boolean isNotUnitVector(float magnitude) { - return Math.abs(magnitude - 1.0f) > EPS; - } - /** * The heuristic to utilize when executing a filtered search against vectors indexed in an HNSW graph. */ @@ -393,6 +391,7 @@ private DenseVectorIndexOptions defaultIndexOptions(boolean defaultInt8Hnsw, boo return new BBQHnswIndexOptions( Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN, Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH, + false, new RescoreVector(DEFAULT_OVERSAMPLE) ); } else if (defaultInt8Hnsw) { @@ -400,6 +399,7 @@ private DenseVectorIndexOptions defaultIndexOptions(boolean defaultInt8Hnsw, boo Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN, Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH, null, + false, null ); } @@ -462,6 +462,7 @@ public DenseVectorFieldMapper build(MapperBuilderContext context) { public enum ElementType { BYTE, FLOAT, + BFLOAT16, BIT; public static ElementType fromString(String name) { @@ -476,6 +477,7 @@ public String toString() { public static final Element BYTE_ELEMENT = new ByteElement(); public static final Element FLOAT_ELEMENT = new FloatElement(); + public static final Element BFLOAT16_ELEMENT = new BFloat16Element(); public static final Element BIT_ELEMENT = new BitElement(); public static final Map namesToElementType = Map.of( @@ -483,6 +485,8 @@ public String toString() { ElementType.BYTE, ElementType.FLOAT.toString(), ElementType.FLOAT, + ElementType.BFLOAT16.toString(), + ElementType.BFLOAT16, ElementType.BIT.toString(), ElementType.BIT ); @@ -492,6 +496,7 @@ public abstract static class Element { public static Element getElement(ElementType elementType) { return switch (elementType) { case FLOAT -> FLOAT_ELEMENT; + case BFLOAT16 -> BFLOAT16_ELEMENT; case BYTE -> BYTE_ELEMENT; case BIT -> BIT_ELEMENT; }; @@ -529,7 +534,7 @@ public static ElementType checkValidVector(float[] vector, ElementType... possib public abstract ElementType elementType(); - public abstract void writeValue(ByteBuffer byteBuffer, float value); + public abstract void writeValues(ByteBuffer byteBuffer, float[] values); public abstract void readAndWriteValue(ByteBuffer byteBuffer, XContentBuilder b) throws IOException; @@ -548,6 +553,10 @@ public abstract VectorData parseKnnVector( public abstract ByteBuffer createByteBuffer(IndexVersion indexVersion, int numBytes); + public boolean isUnitVector(float squaredMagnitude) { + return Math.abs(squaredMagnitude - 1.0f) < 1e-3f; + } + public void checkVectorBounds(float[] vector) { StringBuilder errors = checkVectorErrors(vector); if (errors != null) { @@ -628,8 +637,10 @@ public ElementType elementType() { } @Override - public void writeValue(ByteBuffer byteBuffer, float value) { - byteBuffer.put((byte) value); + public void writeValues(ByteBuffer byteBuffer, float[] values) { + for (float f : values) { + byteBuffer.put((byte) f); + } } @Override @@ -880,8 +891,9 @@ public ElementType elementType() { } @Override - public void writeValue(ByteBuffer byteBuffer, float value) { - byteBuffer.putFloat(value); + public void writeValues(ByteBuffer byteBuffer, float[] values) { + byteBuffer.asFloatBuffer().put(values); + byteBuffer.position(byteBuffer.position() + (values.length * Float.BYTES)); } @Override @@ -947,7 +959,7 @@ void checkVectorMagnitude(VectorSimilarity similarity, UnaryOperator errorElementsAppender(float[] vector) { } } + private static class BFloat16Element extends FloatElement { + + @Override + public ElementType elementType() { + return ElementType.BFLOAT16; + } + + @Override + public void writeValues(ByteBuffer byteBuffer, float[] values) { + BFloat16.floatToBFloat16(values, byteBuffer.asShortBuffer()); + byteBuffer.position(byteBuffer.position() + (values.length * BFloat16.BYTES)); + } + + @Override + public void readAndWriteValue(ByteBuffer byteBuffer, XContentBuilder b) throws IOException { + b.value(BFloat16.bFloat16ToFloat(byteBuffer.getShort())); + } + + @Override + public boolean isUnitVector(float squaredMagnitude) { + // bfloat16 needs to be more lenient + return Math.abs(squaredMagnitude - 1.0f) < 0.02f; + } + + @Override + public int getNumBytes(int dimensions) { + return dimensions * BFloat16.BYTES; + } + } + private static class BitElement extends ByteElement { @Override @@ -1124,7 +1166,7 @@ public enum VectorSimilarity { @Override float score(float similarity, ElementType elementType, int dim) { return switch (elementType) { - case BYTE, FLOAT -> 1f / (1f + similarity * similarity); + case BYTE, FLOAT, BFLOAT16 -> 1f / (1f + similarity * similarity); case BIT -> (dim - similarity) / dim; }; } @@ -1139,14 +1181,14 @@ public VectorSimilarityFunction vectorSimilarityFunction(IndexVersion indexVersi float score(float similarity, ElementType elementType, int dim) { assert elementType != ElementType.BIT; return switch (elementType) { - case BYTE, FLOAT -> (1 + similarity) / 2f; + case BYTE, FLOAT, BFLOAT16 -> (1 + similarity) / 2f; default -> throw new IllegalArgumentException("Unsupported element type [" + elementType + "]"); }; } @Override public VectorSimilarityFunction vectorSimilarityFunction(IndexVersion indexVersion, ElementType elementType) { - return indexVersion.onOrAfter(NORMALIZE_COSINE) && ElementType.FLOAT.equals(elementType) + return indexVersion.onOrAfter(NORMALIZE_COSINE) && (elementType == ElementType.FLOAT || elementType == ElementType.BFLOAT16) ? VectorSimilarityFunction.DOT_PRODUCT : VectorSimilarityFunction.COSINE; } @@ -1156,7 +1198,7 @@ public VectorSimilarityFunction vectorSimilarityFunction(IndexVersion indexVersi float score(float similarity, ElementType elementType, int dim) { return switch (elementType) { case BYTE -> 0.5f + similarity / (float) (dim * (1 << 15)); - case FLOAT -> (1 + similarity) / 2f; + case FLOAT, BFLOAT16 -> (1 + similarity) / 2f; default -> throw new IllegalArgumentException("Unsupported element type [" + elementType + "]"); }; } @@ -1170,7 +1212,7 @@ public VectorSimilarityFunction vectorSimilarityFunction(IndexVersion indexVersi @Override float score(float similarity, ElementType elementType, int dim) { return switch (elementType) { - case BYTE, FLOAT -> similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1; + case BYTE, FLOAT, BFLOAT16 -> similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1; default -> throw new IllegalArgumentException("Unsupported element type [" + elementType + "]"); }; } @@ -1285,16 +1327,11 @@ public enum VectorIndexType { public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { Object mNode = indexOptionsMap.remove("m"); Object efConstructionNode = indexOptionsMap.remove("ef_construction"); - if (mNode == null) { - mNode = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; - } - if (efConstructionNode == null) { - efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; - } - int m = XContentMapValues.nodeIntegerValue(mNode); - int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode); + int m = XContentMapValues.nodeIntegerValue(mNode, Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN); + int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode, Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH); MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); + return new HnswIndexOptions(m, efConstruction); } @@ -1314,14 +1351,11 @@ public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap, IndexVersion indexVersion) { Object mNode = indexOptionsMap.remove("m"); Object efConstructionNode = indexOptionsMap.remove("ef_construction"); - if (mNode == null) { - mNode = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; - } - if (efConstructionNode == null) { - efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; - } - int m = XContentMapValues.nodeIntegerValue(mNode); - int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode); + Object onDiskRescoreNode = indexOptionsMap.remove("on_disk_rescore"); + + int m = XContentMapValues.nodeIntegerValue(mNode, Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN); + int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode, Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH); + boolean onDiskRescore = XContentMapValues.nodeBooleanValue(onDiskRescoreNode, false); RescoreVector rescoreVector = null; if (hasRescoreIndexVersion(indexVersion)) { @@ -1471,12 +1501,12 @@ public DenseVectorIndexOptions parseIndexOptions(String fieldName, Map= this.m && confidenceInterval == int4HnswIndexOptions.confidenceInterval; + updatable = int4HnswIndexOptions.m >= this.m; } else if (update.type.equals(VectorIndexType.BBQ_HNSW)) { updatable = ((BBQHnswIndexOptions) update).m >= m; } else { @@ -1791,15 +1833,13 @@ static class Int4FlatIndexOptions extends QuantizedIndexOptions { Int4FlatIndexOptions(Float confidenceInterval, RescoreVector rescoreVector) { super(VectorIndexType.INT4_FLAT, rescoreVector); - // The default confidence interval for int4 is dynamic quantiles, this provides the best relevancy and is - // effectively required for int4 to behave well across a wide range of data. this.confidenceInterval = confidenceInterval == null ? 0f : confidenceInterval; } @Override public KnnVectorsFormat getVectorsFormat(ElementType elementType) { - assert elementType == ElementType.FLOAT; - return new ES813Int8FlatVectorFormat(confidenceInterval, 4, true); + assert elementType == ElementType.FLOAT || elementType == ElementType.BFLOAT16; + return new ES93ScalarQuantizedVectorsFormat(elementType, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.PACKED_NIBBLE); } @Override @@ -1819,12 +1859,12 @@ public boolean doEquals(DenseVectorIndexOptions o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Int4FlatIndexOptions that = (Int4FlatIndexOptions) o; - return Objects.equals(confidenceInterval, that.confidenceInterval) && Objects.equals(rescoreVector, that.rescoreVector); + return Objects.equals(rescoreVector, that.rescoreVector); } @Override public int doHashCode() { - return Objects.hash(confidenceInterval, rescoreVector); + return Objects.hash(rescoreVector); } @Override @@ -1834,7 +1874,7 @@ public boolean isFlat() { @Override public String toString() { - return "{type=" + type + ", confidence_interval=" + confidenceInterval + ", rescore_vector=" + rescoreVector + "}"; + return "{type=" + type + ", rescore_vector=" + rescoreVector + "}"; } @Override @@ -1848,25 +1888,38 @@ public boolean updatableTo(DenseVectorIndexOptions update) { || update.type.equals(VectorIndexType.BBQ_FLAT) || update.type.equals(VectorIndexType.BBQ_DISK); } - } public static class Int8HnswIndexOptions extends QuantizedIndexOptions { private final int m; private final int efConstruction; private final Float confidenceInterval; - - public Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector) { + private final boolean onDiskRescore; + + public Int8HnswIndexOptions( + int m, + int efConstruction, + Float confidenceInterval, + boolean onDiskRescore, + RescoreVector rescoreVector + ) { super(VectorIndexType.INT8_HNSW, rescoreVector); this.m = m; this.efConstruction = efConstruction; this.confidenceInterval = confidenceInterval; + this.onDiskRescore = onDiskRescore; } @Override public KnnVectorsFormat getVectorsFormat(ElementType elementType) { - assert elementType == ElementType.FLOAT; - return new ES814HnswScalarQuantizedVectorsFormat(m, efConstruction, confidenceInterval, 7, false); + assert elementType == ElementType.FLOAT || elementType == ElementType.BFLOAT16; + return new ES93HnswScalarQuantizedVectorsFormat( + m, + efConstruction, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, + elementType, + onDiskRescore + ); } @Override @@ -1878,6 +1931,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (confidenceInterval != null) { builder.field("confidence_interval", confidenceInterval); } + if (onDiskRescore) { + builder.field("on_disk_rescore", true); + } if (rescoreVector != null) { rescoreVector.toXContent(builder, params); } @@ -1892,13 +1948,13 @@ public boolean doEquals(DenseVectorIndexOptions o) { Int8HnswIndexOptions that = (Int8HnswIndexOptions) o; return m == that.m && efConstruction == that.efConstruction - && Objects.equals(confidenceInterval, that.confidenceInterval) + && onDiskRescore == that.onDiskRescore && Objects.equals(rescoreVector, that.rescoreVector); } @Override public int doHashCode() { - return Objects.hash(m, efConstruction, confidenceInterval, rescoreVector); + return Objects.hash(m, efConstruction, onDiskRescore, rescoreVector); } @Override @@ -1914,10 +1970,6 @@ public int efConstruction() { return efConstruction; } - public Float confidenceInterval() { - return confidenceInterval; - } - @Override public String toString() { return "{type=" @@ -1926,8 +1978,8 @@ public String toString() { + m + ", ef_construction=" + efConstruction - + ", confidence_interval=" - + confidenceInterval + + ", on_disk_rescore=" + + onDiskRescore + ", rescore_vector=" + (rescoreVector == null ? "none" : rescoreVector) + "}"; @@ -1941,9 +1993,6 @@ public boolean updatableTo(DenseVectorIndexOptions update) { // fewer connections would break assumptions on max number of connections (based on largest previous graph) during merge // quantization could not behave as expected with different confidence intervals (and quantiles) to be created updatable = int8HnswIndexOptions.m >= this.m; - updatable &= confidenceInterval == null - || int8HnswIndexOptions.confidenceInterval != null - && confidenceInterval.equals(int8HnswIndexOptions.confidenceInterval); } else { updatable = update.type.equals(VectorIndexType.BBQ_DISK) || update.type.equals(VectorIndexType.INT4_HNSW) && ((Int4HnswIndexOptions) update).m >= this.m @@ -1965,10 +2014,7 @@ public static class HnswIndexOptions extends DenseVectorIndexOptions { @Override public KnnVectorsFormat getVectorsFormat(ElementType elementType) { - if (elementType == ElementType.BIT) { - return new ES815HnswBitVectorsFormat(m, efConstruction); - } - return new Lucene99HnswVectorsFormat(m, efConstruction, 1, null, 0); + return new ES93HnswVectorsFormat(m, efConstruction, elementType); } @Override @@ -2031,17 +2077,19 @@ public String toString() { public static class BBQHnswIndexOptions extends QuantizedIndexOptions { private final int m; private final int efConstruction; + private final boolean onDiskRescore; - public BBQHnswIndexOptions(int m, int efConstruction, RescoreVector rescoreVector) { + public BBQHnswIndexOptions(int m, int efConstruction, boolean onDiskRescore, RescoreVector rescoreVector) { super(VectorIndexType.BBQ_HNSW, rescoreVector); this.m = m; this.efConstruction = efConstruction; + this.onDiskRescore = onDiskRescore; } @Override KnnVectorsFormat getVectorsFormat(ElementType elementType) { - assert elementType == ElementType.FLOAT; - return new ES818HnswBinaryQuantizedVectorsFormat(m, efConstruction); + assert elementType == ElementType.FLOAT || elementType == ElementType.BFLOAT16; + return new ES93HnswBinaryQuantizedVectorsFormat(m, efConstruction, elementType, onDiskRescore); } @Override @@ -2053,12 +2101,15 @@ public boolean updatableTo(DenseVectorIndexOptions update) { @Override boolean doEquals(DenseVectorIndexOptions other) { BBQHnswIndexOptions that = (BBQHnswIndexOptions) other; - return m == that.m && efConstruction == that.efConstruction && Objects.equals(rescoreVector, that.rescoreVector); + return m == that.m + && efConstruction == that.efConstruction + && onDiskRescore == that.onDiskRescore + && Objects.equals(rescoreVector, that.rescoreVector); } @Override int doHashCode() { - return Objects.hash(m, efConstruction, rescoreVector); + return Objects.hash(m, efConstruction, onDiskRescore, rescoreVector); } @Override @@ -2072,6 +2123,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("type", type); builder.field("m", m); builder.field("ef_construction", efConstruction); + if (onDiskRescore) { + builder.field("on_disk_rescore", true); + } if (rescoreVector != null) { rescoreVector.toXContent(builder, params); } @@ -2100,8 +2154,8 @@ static class BBQFlatIndexOptions extends QuantizedIndexOptions { @Override KnnVectorsFormat getVectorsFormat(ElementType elementType) { - assert elementType == ElementType.FLOAT; - return new ES818BinaryQuantizedVectorsFormat(); + assert elementType == ElementType.FLOAT || elementType == ElementType.BFLOAT16; + return new ES93BinaryQuantizedVectorsFormat(elementType, false); } @Override @@ -2164,10 +2218,10 @@ static class BBQIVFIndexOptions extends QuantizedIndexOptions { @Override KnnVectorsFormat getVectorsFormat(ElementType elementType) { - assert elementType == ElementType.FLOAT; return new ES920DiskBBQVectorsFormat( clusterSize, ES920DiskBBQVectorsFormat.DEFAULT_CENTROIDS_PER_PARENT_CLUSTER, + elementType, onDiskRescore ); } @@ -2353,7 +2407,7 @@ public Query createExactKnnQuery(VectorData queryVector, Float vectorSimilarity) } Query knnQuery = switch (element.elementType()) { case BYTE -> createExactKnnByteQuery(queryVector.asByteVector()); - case FLOAT -> createExactKnnFloatQuery(queryVector.asFloatVector()); + case FLOAT, BFLOAT16 -> createExactKnnFloatQuery(queryVector.asFloatVector()); case BIT -> createExactKnnBitQuery(queryVector.asByteVector()); }; if (vectorSimilarity != null) { @@ -2390,7 +2444,7 @@ private Query createExactKnnFloatQuery(float[] queryVector) { if (similarity == VectorSimilarity.DOT_PRODUCT || similarity == VectorSimilarity.COSINE) { float squaredMagnitude = VectorUtil.dotProduct(queryVector, queryVector); element.checkVectorMagnitude(similarity, FloatElement.errorElementsAppender(queryVector), squaredMagnitude); - if (isNormalized() && isNotUnitVector(squaredMagnitude)) { + if (isNormalized() && element.isUnitVector(squaredMagnitude) == false) { float length = (float) Math.sqrt(squaredMagnitude); queryVector = Arrays.copyOf(queryVector, queryVector.length); for (int i = 0; i < queryVector.length; i++) { @@ -2434,7 +2488,7 @@ public Query createKnnQuery( knnSearchStrategy, hnswEarlyTermination ); - case FLOAT -> createKnnFloatQuery( + case FLOAT, BFLOAT16 -> createKnnFloatQuery( queryVector.asFloatVector(), k, numCands, @@ -2586,7 +2640,7 @@ private Query createKnnFloatQuery( if (similarity == VectorSimilarity.DOT_PRODUCT || similarity == VectorSimilarity.COSINE) { float squaredMagnitude = VectorUtil.dotProduct(queryVector, queryVector); element.checkVectorMagnitude(similarity, FloatElement.errorElementsAppender(queryVector), squaredMagnitude); - if (isNormalized() && isNotUnitVector(squaredMagnitude)) { + if (isNormalized() && element.isUnitVector(squaredMagnitude) == false) { float length = (float) Math.sqrt(squaredMagnitude); queryVector = Arrays.copyOf(queryVector, queryVector.length); for (int i = 0; i < queryVector.length; i++) { @@ -2836,7 +2890,7 @@ private void parseBinaryDocValuesVectorAndIndex(DocumentParserContext context) t checkDimensionExceeded(i, context); } }, fieldType().similarity); - vectorData.addToBuffer(byteBuffer); + vectorData.addToBuffer(element, byteBuffer); if (indexCreatedVersion.onOrAfter(MAGNITUDE_STORED_INDEX_VERSION)) { // encode vector magnitude at the end double dotProduct = element.computeSquaredMagnitude(vectorData); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java index 120682d185535..8363f32f8f27f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorDVLeafFieldData.java @@ -22,6 +22,7 @@ import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; +import org.elasticsearch.script.field.vectors.BFloat16BinaryDenseVectorDocValuesField; import org.elasticsearch.script.field.vectors.BinaryDenseVectorDocValuesField; import org.elasticsearch.script.field.vectors.BitBinaryDenseVectorDocValuesField; import org.elasticsearch.script.field.vectors.BitKnnDenseVectorDocValuesField; @@ -69,7 +70,8 @@ public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { if (indexed) { return switch (elementType) { case BYTE -> new ByteKnnDenseVectorDocValuesField(reader.getByteVectorValues(field), name, dims); - case FLOAT -> new KnnDenseVectorDocValuesField(reader.getFloatVectorValues(field), name, dims); + // bfloat16 is hidden by the FloatVectorValues implementation + case FLOAT, BFLOAT16 -> new KnnDenseVectorDocValuesField(reader.getFloatVectorValues(field), name, dims); case BIT -> new BitKnnDenseVectorDocValuesField(reader.getByteVectorValues(field), name, dims); }; } else { @@ -77,6 +79,7 @@ public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { return switch (elementType) { case BYTE -> new ByteBinaryDenseVectorDocValuesField(values, name, elementType, dims); case FLOAT -> new BinaryDenseVectorDocValuesField(values, name, elementType, dims, indexVersion); + case BFLOAT16 -> new BFloat16BinaryDenseVectorDocValuesField(values, name, elementType, dims, indexVersion); case BIT -> new BitBinaryDenseVectorDocValuesField(values, name, elementType, dims); }; } @@ -85,105 +88,125 @@ public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { } } - @Override - public FormattedDocValues getFormattedValues(DocValueFormat format) { - int dims = elementType == ElementType.BIT ? this.dims / Byte.SIZE : this.dims; - return switch (elementType) { - case BYTE, BIT -> new FormattedDocValues() { - private byte[] vector = new byte[dims]; - private ByteVectorValues byteVectorValues; // use when indexed - private KnnVectorValues.DocIndexIterator iterator; // use when indexed - private BinaryDocValues binary; // use when not indexed - { - try { - if (indexed) { - byteVectorValues = reader.getByteVectorValues(field); - iterator = (byteVectorValues == null) ? null : byteVectorValues.iterator(); - } else { - binary = DocValues.getBinary(reader, field); - } - } catch (IOException e) { - throw new IllegalStateException("Cannot load doc values", e); - } - + private class ByteDocValues implements FormattedDocValues { + private final int dims; + private byte[] vector; + private ByteVectorValues byteVectorValues; // use when indexed + private KnnVectorValues.DocIndexIterator iterator; // use when indexed + private BinaryDocValues binary; // use when not indexed + + ByteDocValues(int dims) { + this.dims = dims; + this.vector = new byte[dims]; + try { + if (indexed) { + byteVectorValues = reader.getByteVectorValues(field); + iterator = (byteVectorValues == null) ? null : byteVectorValues.iterator(); + } else { + binary = DocValues.getBinary(reader, field); } + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } - @Override - public boolean advanceExact(int docId) throws IOException { - if (indexed) { - if (iteratorAdvanceExact(iterator, docId) == false) { - return false; - } - vector = byteVectorValues.vectorValue(iterator.index()); - } else { - if (binary == null || binary.advanceExact(docId) == false) { - return false; - } - BytesRef ref = binary.binaryValue(); - System.arraycopy(ref.bytes, ref.offset, vector, 0, dims); - } - return true; - } + } - @Override - public int docValueCount() { - return 1; + @Override + public boolean advanceExact(int docId) throws IOException { + if (indexed) { + if (iteratorAdvanceExact(iterator, docId) == false) { + return false; } - - public Object nextValue() { - Byte[] vectorValue = new Byte[dims]; - for (int i = 0; i < dims; i++) { - vectorValue[i] = vector[i]; - } - return vectorValue; + vector = byteVectorValues.vectorValue(iterator.index()); + } else { + if (binary == null || binary.advanceExact(docId) == false) { + return false; } - }; - case FLOAT -> new FormattedDocValues() { - float[] vector = new float[dims]; - private FloatVectorValues floatVectorValues; // use when indexed - private KnnVectorValues.DocIndexIterator iterator; // use when indexed - private BinaryDocValues binary; // use when not indexed - { - try { - if (indexed) { - floatVectorValues = reader.getFloatVectorValues(field); - iterator = (floatVectorValues == null) ? null : floatVectorValues.iterator(); - } else { - binary = DocValues.getBinary(reader, field); - } - } catch (IOException e) { - throw new IllegalStateException("Cannot load doc values", e); - } + BytesRef ref = binary.binaryValue(); + System.arraycopy(ref.bytes, ref.offset, vector, 0, dims); + } + return true; + } - } + @Override + public int docValueCount() { + return 1; + } - @Override - public boolean advanceExact(int docId) throws IOException { - if (indexed) { - if (iteratorAdvanceExact(iterator, docId) == false) { - return false; - } - vector = floatVectorValues.vectorValue(iterator.index()); - } else { - if (binary == null || binary.advanceExact(docId) == false) { - return false; - } - BytesRef ref = binary.binaryValue(); - VectorEncoderDecoder.decodeDenseVector(indexVersion, ref, vector); - } - return true; - } + public Object nextValue() { + Byte[] vectorValue = new Byte[dims]; + for (int i = 0; i < dims; i++) { + vectorValue[i] = vector[i]; + } + return vectorValue; + } + } - @Override - public int docValueCount() { - return 1; + private class FloatDocValues implements FormattedDocValues { + private float[] vector = new float[dims]; + private FloatVectorValues floatVectorValues; // use when indexed + private KnnVectorValues.DocIndexIterator iterator; // use when indexed + private BinaryDocValues binary; // use when not indexed + + FloatDocValues() { + try { + if (indexed) { + floatVectorValues = reader.getFloatVectorValues(field); + iterator = (floatVectorValues == null) ? null : floatVectorValues.iterator(); + } else { + binary = DocValues.getBinary(reader, field); } + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } - @Override - public Object nextValue() { - return Arrays.copyOf(vector, vector.length); + @Override + public boolean advanceExact(int docId) throws IOException { + if (indexed) { + if (iteratorAdvanceExact(iterator, docId) == false) { + return false; + } + vector = floatVectorValues.vectorValue(iterator.index()); + } else { + if (binary == null || binary.advanceExact(docId) == false) { + return false; } - }; + BytesRef ref = binary.binaryValue(); + decodeDenseVector(indexVersion, ref, vector); + } + return true; + } + + void decodeDenseVector(IndexVersion indexVersion, BytesRef ref, float[] vector) { + VectorEncoderDecoder.decodeDenseVector(indexVersion, ref, vector); + } + + @Override + public int docValueCount() { + return 1; + } + + @Override + public Object nextValue() { + return Arrays.copyOf(vector, vector.length); + } + } + + private class BFloat16DocValues extends FloatDocValues { + @Override + void decodeDenseVector(IndexVersion indexVersion, BytesRef vectorBR, float[] vector) { + VectorEncoderDecoder.decodeBFloat16DenseVector(vectorBR, vector); + } + } + + @Override + public FormattedDocValues getFormattedValues(DocValueFormat format) { + return switch (elementType) { + case BYTE -> new ByteDocValues(dims); + case BIT -> new ByteDocValues(dims / Byte.SIZE); + case FLOAT -> new FloatDocValues(); + case BFLOAT16 -> new BFloat16DocValues(); }; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorEncoderDecoder.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorEncoderDecoder.java index 9dec4a4f2dd61..f60e7139cd3d9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorEncoderDecoder.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorEncoderDecoder.java @@ -12,10 +12,12 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.VectorUtil; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.codec.vectors.BFloat16; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.FloatBuffer; +import java.nio.ShortBuffer; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.LITTLE_ENDIAN_FLOAT_STORED_INDEX_VERSION; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAGNITUDE_STORED_INDEX_VERSION; @@ -84,6 +86,14 @@ public static void decodeDenseVector(IndexVersion indexVersion, BytesRef vectorB } } + public static void decodeBFloat16DenseVector(BytesRef vectorBR, float[] vector) { + if (vectorBR == null) { + throw new IllegalArgumentException(DenseVectorScriptDocValues.MISSING_VECTOR_FIELD_MESSAGE); + } + ShortBuffer sb = ByteBuffer.wrap(vectorBR.bytes, vectorBR.offset, vectorBR.length).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer(); + BFloat16.bFloat16ToFloat(sb, vector); + } + /** * Decodes a BytesRef into the provided array of bytes * @param vectorBR - dense vector encoded in BytesRef diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java index aff4287ecbc58..d790b5ddb70db 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java @@ -32,10 +32,12 @@ public class RestPutComponentTemplateAction extends BaseRestHandler { public static final String SUPPORTS_FAILURE_STORE_LIFECYCLE = "data_stream_options.failure_store.lifecycle"; public static final String SUPPORTS_FAILURE_STORE = "data_stream_options.failure_store"; private static final String COMPONENT_TEMPLATE_TRACKING_INFO = "component_template_tracking_info"; + static final String SUPPORTS_DOWNSAMPLING_METHOD = "dlm.downsampling_method"; private static final Set CAPABILITIES = Set.of( SUPPORTS_FAILURE_STORE, SUPPORTS_FAILURE_STORE_LIFECYCLE, - COMPONENT_TEMPLATE_TRACKING_INFO + COMPONENT_TEMPLATE_TRACKING_INFO, + SUPPORTS_DOWNSAMPLING_METHOD ); @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComposableIndexTemplateAction.java index 28f5430775004..15f713ea583d7 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComposableIndexTemplateAction.java @@ -25,6 +25,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.RestRequest.Method.PUT; import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; +import static org.elasticsearch.rest.action.admin.indices.RestPutComponentTemplateAction.SUPPORTS_DOWNSAMPLING_METHOD; import static org.elasticsearch.rest.action.admin.indices.RestPutComponentTemplateAction.SUPPORTS_FAILURE_STORE; import static org.elasticsearch.rest.action.admin.indices.RestPutComponentTemplateAction.SUPPORTS_FAILURE_STORE_LIFECYCLE; @@ -32,10 +33,12 @@ public class RestPutComposableIndexTemplateAction extends BaseRestHandler { private static final String INDEX_TEMPLATE_TRACKING_INFO = "index_template_tracking_info"; + private static final Set CAPABILITIES = Set.of( SUPPORTS_FAILURE_STORE, SUPPORTS_FAILURE_STORE_LIFECYCLE, - INDEX_TEMPLATE_TRACKING_INFO + INDEX_TEMPLATE_TRACKING_INFO, + SUPPORTS_DOWNSAMPLING_METHOD ); @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java index 5fd4ec83c1a18..c6b8a01f78bf9 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; @@ -36,13 +37,16 @@ public abstract class RestResizeHandler extends BaseRestHandler { @Override public final RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final ResizeRequest resizeRequest = new ResizeRequest(request.param("target"), request.param("index")); - resizeRequest.setResizeType(getResizeType()); + final ResizeRequest resizeRequest = new ResizeRequest( + getMasterNodeTimeout(request), + getAckTimeout(request), + getResizeType(), + request.param("index"), + request.param("target") + ); request.applyContentParser(resizeRequest::fromXContent); - resizeRequest.ackTimeout(getAckTimeout(request)); - resizeRequest.masterNodeTimeout(getMasterNodeTimeout(request)); resizeRequest.setWaitForActiveShards(ActiveShardCount.parseString(request.param("wait_for_active_shards"))); - return channel -> client.admin().indices().resizeIndex(resizeRequest, new RestToXContentListener<>(channel)); + return channel -> client.execute(TransportResizeAction.TYPE, resizeRequest, new RestToXContentListener<>(channel)); } // no @ServerlessScope on purpose, not available diff --git a/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java b/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java index 13be089753fb7..b29a0266c220c 100644 --- a/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java +++ b/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java @@ -210,7 +210,7 @@ public L1Norm(ScoreScript scoreScript, Object queryVector, String fieldName) { } throw new IllegalArgumentException("Unsupported input object for byte vectors: " + queryVector.getClass().getName()); } - case FLOAT -> { + case FLOAT, BFLOAT16 -> { if (queryVector instanceof List) { yield new FloatL1Norm(scoreScript, field, (List) queryVector); } @@ -320,7 +320,7 @@ public L2Norm(ScoreScript scoreScript, Object queryVector, String fieldName) { } throw new IllegalArgumentException("Unsupported input object for byte vectors: " + queryVector.getClass().getName()); } - case FLOAT -> { + case FLOAT, BFLOAT16 -> { if (queryVector instanceof List) { yield new FloatL2Norm(scoreScript, field, (List) queryVector); } @@ -478,7 +478,7 @@ public DotProduct(ScoreScript scoreScript, Object queryVector, String fieldName) } throw new IllegalArgumentException("Unsupported input object for byte vectors: " + queryVector.getClass().getName()); } - case FLOAT -> { + case FLOAT, BFLOAT16 -> { if (queryVector instanceof List) { yield new FloatDotProduct(scoreScript, field, (List) queryVector); } @@ -547,7 +547,7 @@ public CosineSimilarity(ScoreScript scoreScript, Object queryVector, String fiel } throw new IllegalArgumentException("Unsupported input object for byte vectors: " + queryVector.getClass().getName()); } - case FLOAT -> { + case FLOAT, BFLOAT16 -> { if (queryVector instanceof List) { yield new FloatCosineSimilarity(scoreScript, field, (List) queryVector); } diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/BFloat16BinaryDenseVectorDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/BFloat16BinaryDenseVectorDocValuesField.java new file mode 100644 index 0000000000000..4805b77250c00 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/BFloat16BinaryDenseVectorDocValuesField.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.script.field.vectors; + +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.elasticsearch.index.mapper.vectors.VectorEncoderDecoder; + +public class BFloat16BinaryDenseVectorDocValuesField extends BinaryDenseVectorDocValuesField { + public BFloat16BinaryDenseVectorDocValuesField( + BinaryDocValues input, + String name, + DenseVectorFieldMapper.ElementType elementType, + int dims, + IndexVersion indexVersion + ) { + super(input, name, elementType, dims, indexVersion); + } + + @Override + void decodeDenseVector(IndexVersion indexVersion, BytesRef vectorBR, float[] vector) { + VectorEncoderDecoder.decodeBFloat16DenseVector(vectorBR, vector); + } +} diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/BinaryDenseVectorDocValuesField.java b/server/src/main/java/org/elasticsearch/script/field/vectors/BinaryDenseVectorDocValuesField.java index 0bb9d2a3a0b0d..376d04cbeb957 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/BinaryDenseVectorDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/BinaryDenseVectorDocValuesField.java @@ -86,8 +86,12 @@ public DenseVector getInternal() { private void decodeVectorIfNecessary() { if (decoded == false && value != null) { - VectorEncoderDecoder.decodeDenseVector(indexVersion, value, vectorValue); + decodeDenseVector(indexVersion, value, vectorValue); decoded = true; } } + + void decodeDenseVector(IndexVersion indexVersion, BytesRef value, float[] vector) { + VectorEncoderDecoder.decodeDenseVector(indexVersion, value, vectorValue); + } } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/VectorData.java b/server/src/main/java/org/elasticsearch/search/vectors/VectorData.java index a128130a71cfc..84cd638cceef2 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/VectorData.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/VectorData.java @@ -75,11 +75,9 @@ public float[] asFloatVector() { return vec; } - public void addToBuffer(ByteBuffer byteBuffer) { + public void addToBuffer(DenseVectorFieldMapper.Element element, ByteBuffer byteBuffer) { if (floatVector != null) { - for (float val : floatVector) { - byteBuffer.putFloat(val); - } + element.writeValues(byteBuffer, floatVector); } else { byteBuffer.put(byteVector); } diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat index 5370d7244df9b..0dc34ea2e808d 100644 --- a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat +++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat @@ -9,6 +9,9 @@ org.elasticsearch.index.codec.vectors.es818.ES818BinaryQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat org.elasticsearch.index.codec.vectors.diskbbq.next.ESNextDiskBBQVectorsFormat -org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat +org.elasticsearch.index.codec.vectors.es93.ES93FlatVectorFormat org.elasticsearch.index.codec.vectors.es93.ES93HnswVectorsFormat +org.elasticsearch.index.codec.vectors.es93.ES93ScalarQuantizedVectorsFormat +org.elasticsearch.index.codec.vectors.es93.ES93HnswScalarQuantizedVectorsFormat +org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.es93.ES93HnswBinaryQuantizedVectorsFormat diff --git a/server/src/main/resources/transport/definitions/referable/add_sample_method_downsample_dlm.csv b/server/src/main/resources/transport/definitions/referable/add_sample_method_downsample_dlm.csv new file mode 100644 index 0000000000000..9b32a5cf8b643 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/add_sample_method_downsample_dlm.csv @@ -0,0 +1 @@ +9211000 diff --git a/server/src/main/resources/transport/definitions/referable/resolved_fields_caps.csv b/server/src/main/resources/transport/definitions/referable/resolved_fields_caps.csv new file mode 100644 index 0000000000000..4629e277c153b --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/resolved_fields_caps.csv @@ -0,0 +1 @@ +9212000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index 88b7f13fd8f9f..ce5e1c85f99fd 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -add_sample_method_downsample_ilm,9210000 +resolved_fields_caps,9212000 diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java index e1316b40e7ce3..efd797c769f04 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.RandomCreateIndexGenerator; import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; @@ -34,6 +35,7 @@ import java.util.function.Supplier; import static org.elasticsearch.action.admin.indices.create.CreateIndexRequestTests.assertAliasesEqual; +import static org.elasticsearch.action.admin.indices.shrink.ResizeRequest.MAX_PRIMARY_SHARD_SIZE; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; import static org.hamcrest.Matchers.containsString; @@ -53,7 +55,13 @@ public void testCopySettingsValidation() { private void runTestCopySettingsValidation(final Boolean copySettings, final Consumer> consumer) { consumer.accept(() -> { - final ResizeRequest request = new ResizeRequest(); + final ResizeRequest request = new ResizeRequest( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + randomFrom(ResizeType.values()), + randomIdentifier(), + randomIdentifier() + ); request.setCopySettings(copySettings); return request; }); @@ -61,19 +69,25 @@ private void runTestCopySettingsValidation(final Boolean copySettings, final Con public void testToXContent() throws IOException { { - ResizeRequest request = new ResizeRequest("target", "source"); - String actualRequestBody = Strings.toString(request); + ResizeRequest request = new ResizeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, ResizeType.SPLIT, "source", "target"); + String actualRequestBody = Strings.toString(resizeRequestToXContent(request)); assertEquals("{\"settings\":{},\"aliases\":{}}", actualRequestBody); } { - ResizeRequest request = new ResizeRequest("target", "source"); + ResizeRequest request = new ResizeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, ResizeType.SPLIT, "source", "target"); request.setMaxPrimaryShardSize(ByteSizeValue.of(100, ByteSizeUnit.MB)); - String actualRequestBody = Strings.toString(request); + String actualRequestBody = Strings.toString(resizeRequestToXContent(request)); assertEquals(""" {"settings":{},"aliases":{},"max_primary_shard_size":"100mb"}""", actualRequestBody); } { - ResizeRequest request = new ResizeRequest(); + ResizeRequest request = new ResizeRequest( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + randomFrom(ResizeType.values()), + randomIdentifier(), + randomIdentifier() + ); CreateIndexRequest target = new CreateIndexRequest("target"); Alias alias = new Alias("test_alias"); alias.routing("1"); @@ -85,7 +99,7 @@ public void testToXContent() throws IOException { settings.put(SETTING_NUMBER_OF_SHARDS, 10); target.settings(settings); request.setTargetIndex(target); - String actualRequestBody = Strings.toString(request); + String actualRequestBody = Strings.toString(resizeRequestToXContent(request)); String expectedRequestBody = """ { "settings": { @@ -114,11 +128,19 @@ public void testToAndFromXContent() throws IOException { boolean humanReadable = randomBoolean(); final XContentType xContentType = randomFrom(XContentType.values()); - BytesReference originalBytes = toShuffledXContent(resizeRequest, xContentType, EMPTY_PARAMS, humanReadable); + BytesReference originalBytes = toShuffledXContent( + resizeRequestToXContent(resizeRequest), + xContentType, + EMPTY_PARAMS, + humanReadable + ); ResizeRequest parsedResizeRequest = new ResizeRequest( - randomValueOtherThan(resizeRequest.getTargetIndexRequest().index(), () -> randomAlphaOfLength(5)), - randomValueOtherThan(resizeRequest.getSourceIndex(), () -> randomAlphaOfLength(5)) + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + randomFrom(ResizeType.values()), + randomValueOtherThan(resizeRequest.getSourceIndex(), () -> randomAlphaOfLength(5)), + randomValueOtherThan(resizeRequest.getTargetIndexRequest().index(), () -> randomAlphaOfLength(5)) ); try (XContentParser xParser = createParser(xContentType.xContent(), originalBytes)) { parsedResizeRequest.fromXContent(xParser); @@ -135,7 +157,12 @@ public void testToAndFromXContent() throws IOException { assertEquals(resizeRequest.getTargetIndexRequest().settings(), parsedResizeRequest.getTargetIndexRequest().settings()); assertEquals(resizeRequest.getMaxPrimaryShardSize(), parsedResizeRequest.getMaxPrimaryShardSize()); - BytesReference finalBytes = toShuffledXContent(parsedResizeRequest, xContentType, EMPTY_PARAMS, humanReadable); + BytesReference finalBytes = toShuffledXContent( + resizeRequestToXContent(parsedResizeRequest), + xContentType, + EMPTY_PARAMS, + humanReadable + ); ElasticsearchAssertions.assertToXContentEquivalent(originalBytes, finalBytes, xContentType); } @@ -163,7 +190,13 @@ protected Writeable.Reader instanceReader() { @Override protected ResizeRequest createTestInstance() { - ResizeRequest resizeRequest = new ResizeRequest(randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10)); + ResizeRequest resizeRequest = new ResizeRequest( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + randomFrom(ResizeType.values()), + randomAlphaOfLengthBetween(3, 10), + randomAlphaOfLengthBetween(3, 10) + ); if (randomBoolean()) { resizeRequest.setTargetIndex(RandomCreateIndexGenerator.randomCreateIndexRequest()); } @@ -177,4 +210,29 @@ protected ResizeRequest createTestInstance() { protected ResizeRequest mutateInstance(ResizeRequest instance) { return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929 } + + private static ToXContentObject resizeRequestToXContent(ResizeRequest resizeRequest) { + return (builder, params) -> { + builder.startObject(); + { + builder.startObject(CreateIndexRequest.SETTINGS.getPreferredName()); + { + resizeRequest.getTargetIndexRequest().settings().toXContent(builder, params); + } + builder.endObject(); + builder.startObject(CreateIndexRequest.ALIASES.getPreferredName()); + { + for (Alias alias : resizeRequest.getTargetIndexRequest().aliases()) { + alias.toXContent(builder, params); + } + } + builder.endObject(); + if (resizeRequest.getMaxPrimaryShardSize() != null) { + builder.field(MAX_PRIMARY_SHARD_SIZE.getPreferredName(), resizeRequest.getMaxPrimaryShardSize()); + } + } + builder.endObject(); + return builder; + }; + } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java index 66fb8a58bba1c..dac353919d0f9 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java @@ -46,6 +46,7 @@ import java.util.Collections; import java.util.Set; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.resizeRequest; import static org.hamcrest.Matchers.equalTo; public class TransportResizeActionTests extends ESTestCase { @@ -94,7 +95,7 @@ public void testErrorCondition() { expectThrows( IllegalStateException.class, () -> TransportResizeAction.prepareCreateIndexRequest( - new ResizeRequest("target", "source"), + resizeRequest(ResizeType.SHRINK, "source", "target", Settings.builder()), projectId, state, "target", @@ -106,28 +107,28 @@ public void testErrorCondition() { ).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ") ); - assertTrue(expectThrows(IllegalStateException.class, () -> { - ResizeRequest req = new ResizeRequest("target", "source"); - req.getTargetIndexRequest().settings(Settings.builder().put("index.number_of_shards", 4)); - TransportResizeAction.prepareCreateIndexRequest( - req, - projectId, - createClusterState("source", 8, 1, Settings.builder().put("index.blocks.write", true).build()).metadata() - .getProject(projectId) - .index("source"), - "target", - new ResizeNumberOfShardsCalculator.ShrinkShardsCalculator( - new StoreStats(between(1, 100), between(0, 100), between(1, 100)), - (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE / 2, between(1, 1000), between(1, 10000)) : null + assertTrue( + expectThrows( + IllegalStateException.class, + () -> TransportResizeAction.prepareCreateIndexRequest( + resizeRequest(ResizeType.SHRINK, "source", "target", Settings.builder().put("index.number_of_shards", 4)), + projectId, + createClusterState("source", 8, 1, Settings.builder().put("index.blocks.write", true).build()).metadata() + .getProject(projectId) + .index("source"), + "target", + new ResizeNumberOfShardsCalculator.ShrinkShardsCalculator( + new StoreStats(between(1, 100), between(0, 100), between(1, 100)), + (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE / 2, between(1, 1000), between(1, 10000)) : null + ) ) - ); - }).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ")); - - IllegalArgumentException softDeletesError = expectThrows(IllegalArgumentException.class, () -> { - ResizeRequest req = new ResizeRequest("target", "source"); - req.getTargetIndexRequest().settings(Settings.builder().put("index.soft_deletes.enabled", false)); - TransportResizeAction.prepareCreateIndexRequest( - req, + ).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ") + ); + + IllegalArgumentException softDeletesError = expectThrows( + IllegalArgumentException.class, + () -> TransportResizeAction.prepareCreateIndexRequest( + resizeRequest(ResizeType.SHRINK, "source", "target", Settings.builder().put("index.soft_deletes.enabled", false)), projectId, createClusterState( "source", @@ -140,8 +141,8 @@ public void testErrorCondition() { new StoreStats(between(1, 100), between(0, 100), between(1, 100)), (i) -> new DocsStats(between(10, 1000), between(1, 10), between(1, 10000)) ) - ); - }); + ) + ); assertThat(softDeletesError.getMessage(), equalTo("Can't disable [index.soft_deletes.enabled] setting on resize")); // create one that won't fail @@ -164,7 +165,7 @@ public void testErrorCondition() { clusterState = ClusterState.builder(clusterState).putRoutingTable(projectId, routingTable).build(); TransportResizeAction.prepareCreateIndexRequest( - new ResizeRequest("target", "source"), + resizeRequest(ResizeType.SHRINK, "source", "target", Settings.builder()), projectId, clusterState.metadata().getProject(projectId).index("source"), "target", @@ -194,24 +195,22 @@ public void testPassNumRoutingShards() { routingTable = ESAllocationTestCase.startInitializingShardsAndReroute(service, clusterState, "source").routingTable(projectId); clusterState = ClusterState.builder(clusterState).putRoutingTable(projectId, routingTable).build(); - ResizeRequest resizeRequest = new ResizeRequest("target", "source"); - resizeRequest.setResizeType(ResizeType.SPLIT); - resizeRequest.getTargetIndexRequest().settings(Settings.builder().put("index.number_of_shards", 2).build()); IndexMetadata indexMetadata = clusterState.metadata().getProject(projectId).index("source"); TransportResizeAction.prepareCreateIndexRequest( - resizeRequest, + resizeRequest(ResizeType.SPLIT, "source", "target", Settings.builder().put("index.number_of_shards", 2)), projectId, indexMetadata, "target", new ResizeNumberOfShardsCalculator.SplitShardsCalculator() ); - resizeRequest.getTargetIndexRequest() - .settings( - Settings.builder().put("index.number_of_routing_shards", randomIntBetween(2, 10)).put("index.number_of_shards", 2).build() - ); TransportResizeAction.prepareCreateIndexRequest( - resizeRequest, + resizeRequest( + ResizeType.SPLIT, + "source", + "target", + Settings.builder().put("index.number_of_routing_shards", randomIntBetween(2, 10)).put("index.number_of_shards", 2) + ), projectId, indexMetadata, "target", @@ -239,26 +238,24 @@ public void testPassNumRoutingShardsAndFail() { routingTable = ESAllocationTestCase.startInitializingShardsAndReroute(service, clusterState, "source").routingTable(projectId); clusterState = ClusterState.builder(clusterState).putRoutingTable(projectId, routingTable).build(); - ResizeRequest resizeRequest = new ResizeRequest("target", "source"); - resizeRequest.setResizeType(ResizeType.SPLIT); - resizeRequest.getTargetIndexRequest().settings(Settings.builder().put("index.number_of_shards", numShards * 2).build()); TransportResizeAction.prepareCreateIndexRequest( - resizeRequest, + resizeRequest(ResizeType.SPLIT, "source", "target", Settings.builder().put("index.number_of_shards", numShards * 2)), projectId, clusterState.metadata().getProject(projectId).index("source"), "target", new ResizeNumberOfShardsCalculator.SplitShardsCalculator() ); - resizeRequest.getTargetIndexRequest() - .settings( - Settings.builder().put("index.number_of_shards", numShards * 2).put("index.number_of_routing_shards", numShards * 2).build() - ); ClusterState finalState = clusterState; IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, () -> TransportResizeAction.prepareCreateIndexRequest( - resizeRequest, + resizeRequest( + ResizeType.SPLIT, + "source", + "target", + Settings.builder().put("index.number_of_shards", numShards * 2).put("index.number_of_routing_shards", numShards * 2) + ), projectId, finalState.metadata().getProject(projectId).index("source"), "target", @@ -290,7 +287,7 @@ public void testShrinkIndexSettings() { clusterState = ClusterState.builder(clusterState).putRoutingTable(projectId, routingTable).build(); int numSourceShards = clusterState.metadata().getProject(projectId).index(indexName).getNumberOfShards(); DocsStats stats = new DocsStats(between(0, (IndexWriter.MAX_DOCS) / numSourceShards), between(1, 1000), between(1, 10000)); - ResizeRequest target = new ResizeRequest("target", indexName); + ResizeRequest target = resizeRequest(ResizeType.SHRINK, indexName, "target", Settings.builder()); final ActiveShardCount activeShardCount = randomBoolean() ? ActiveShardCount.ALL : ActiveShardCount.ONE; target.setWaitForActiveShards(activeShardCount); CreateIndexClusterStateUpdateRequest request = TransportResizeAction.prepareCreateIndexRequest( @@ -318,9 +315,13 @@ public void testShrinkWithMaxPrimaryShardSize() { randomIntBetween(0, 10), Settings.builder().put("index.blocks.write", true).build() ).metadata().getProject(projectId).index("source"); - ResizeRequest resizeRequest = new ResizeRequest("target", "source"); + ResizeRequest resizeRequest = resizeRequest( + ResizeType.SHRINK, + "source", + "target", + Settings.builder().put("index.number_of_shards", 2) + ); resizeRequest.setMaxPrimaryShardSize(ByteSizeValue.ofBytes(10)); - resizeRequest.getTargetIndexRequest().settings(Settings.builder().put("index.number_of_shards", 2).build()); assertTrue( expectThrows( IllegalArgumentException.class, @@ -359,7 +360,7 @@ public void testShrinkWithMaxPrimaryShardSize() { DocsStats stats = new DocsStats(between(0, (IndexWriter.MAX_DOCS) / numSourceShards), between(1, 1000), between(1, 10000)); // each shard's storage will not be greater than the `max_primary_shard_size` - ResizeRequest target1 = new ResizeRequest("target", "source"); + ResizeRequest target1 = resizeRequest(ResizeType.SHRINK, "source", "target", Settings.builder()); target1.setMaxPrimaryShardSize(ByteSizeValue.ofBytes(2)); StoreStats storeStats = new StoreStats(10, between(0, 100), between(1, 100)); final int targetIndexShardsNum1 = 5; @@ -381,7 +382,7 @@ public void testShrinkWithMaxPrimaryShardSize() { // if `max_primary_shard_size` is less than the single shard size of the source index, // the shards number of the target index will be equal to the source index's shards number - ResizeRequest target2 = new ResizeRequest("target2", "source"); + ResizeRequest target2 = resizeRequest(ResizeType.SHRINK, "source", "target2", Settings.builder()); target2.setMaxPrimaryShardSize(ByteSizeValue.ofBytes(1)); StoreStats storeStats2 = new StoreStats(100, between(0, 100), between(1, 100)); final int targetIndexShardsNum2 = 10; diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java index c90669650a949..c9027799a57c6 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java @@ -107,7 +107,7 @@ private static GetDataStreamAction.Response.DataStreamInfo newDataStreamInfo(boo private static DataStream newDataStreamInstance(boolean isSystem, TimeValue retention) { List indices = List.of(new Index(randomAlphaOfLength(10), randomAlphaOfLength(10))); - DataStreamLifecycle lifecycle = DataStreamLifecycle.createDataLifecycle(true, retention, null); + DataStreamLifecycle lifecycle = DataStreamLifecycle.createDataLifecycle(true, retention, null, null); Settings settings = randomSettings(); CompressedXContent mappings = randomMappings(); return DataStream.builder(randomAlphaOfLength(50), indices) diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycleTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycleTests.java index 43a39f762ce69..2fb4a1c0ced4f 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycleTests.java @@ -200,7 +200,7 @@ public void testToXContent() throws Exception { TimeValue configuredRetention = TimeValue.timeValueDays(100); TimeValue globalDefaultRetention = TimeValue.timeValueDays(10); TimeValue globalMaxRetention = TimeValue.timeValueDays(50); - DataStreamLifecycle dataStreamLifecycle = DataStreamLifecycle.createDataLifecycle(true, configuredRetention, null); + DataStreamLifecycle dataStreamLifecycle = DataStreamLifecycle.createDataLifecycle(true, configuredRetention, null, null); { boolean isSystemDataStream = true; ExplainIndexDataStreamLifecycle explainIndexDataStreamLifecycle = createManagedIndexDataStreamLifecycleExplanation( diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java index 32269d0bc5113..7c72764901505 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java @@ -96,7 +96,7 @@ public void testDataStreamLifecycleToXContent() throws Exception { TimeValue globalDefaultRetention = TimeValue.timeValueDays(10); TimeValue globalMaxRetention = TimeValue.timeValueDays(50); DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(globalDefaultRetention, globalMaxRetention); - DataStreamLifecycle lifecycle = DataStreamLifecycle.createDataLifecycle(true, configuredRetention, null); + DataStreamLifecycle lifecycle = DataStreamLifecycle.createDataLifecycle(true, configuredRetention, null, null); { boolean isInternalDataStream = true; GetDataStreamLifecycleAction.Response.DataStreamLifecycle explainIndexDataStreamLifecycle = createDataStreamLifecycle( diff --git a/server/src/test/java/org/elasticsearch/action/downsample/DownsampleConfigTests.java b/server/src/test/java/org/elasticsearch/action/downsample/DownsampleConfigTests.java index 95611f319d6ed..3469564b28729 100644 --- a/server/src/test/java/org/elasticsearch/action/downsample/DownsampleConfigTests.java +++ b/server/src/test/java/org/elasticsearch/action/downsample/DownsampleConfigTests.java @@ -49,7 +49,7 @@ public static DownsampleConfig randomConfig() { return new DownsampleConfig(randomInterval(), randomSamplingMethod()); } - private static DownsampleConfig.SamplingMethod randomSamplingMethod() { + public static DownsampleConfig.SamplingMethod randomSamplingMethod() { if (between(0, DownsampleConfig.SamplingMethod.values().length) == 0) { return null; } else { diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java index ceb84e4b2a0d9..55c8b842d3db8 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java @@ -53,7 +53,7 @@ protected FieldCapabilitiesResponse createTestInstance() { var indexMode = randomFrom(IndexMode.values()); responses.add(new FieldCapabilitiesIndexResponse("index_" + i, null, fieldCaps, randomBoolean(), indexMode)); } - randomResponse = new FieldCapabilitiesResponse(responses, Collections.emptyList()); + randomResponse = FieldCapabilitiesResponse.builder().withIndexResponses(responses).build(); return randomResponse; } @@ -88,7 +88,7 @@ protected FieldCapabilitiesResponse mutateInstance(FieldCapabilitiesResponse res ); } } - return new FieldCapabilitiesResponse(null, mutatedResponses, Collections.emptyList()); + return FieldCapabilitiesResponse.builder().withFields(mutatedResponses).build(); } public void testFailureSerialization() throws IOException { @@ -144,7 +144,7 @@ public static FieldCapabilitiesResponse createResponseWithFailures() { failures.get(failures.size() - 1).addIndex(index); } } - return new FieldCapabilitiesResponse(indices, Collections.emptyMap(), failures); + return FieldCapabilitiesResponse.builder().withIndices(indices).withFailures(failures).build(); } private static FieldCapabilitiesResponse randomCCSResponse(List indexResponses) { @@ -154,7 +154,7 @@ private static FieldCapabilitiesResponse randomCCSResponse(List failureMap = List.of( new FieldCapabilitiesFailure(new String[] { "errorindex", "errorindex2" }, new IllegalArgumentException("test")) ); - return new FieldCapabilitiesResponse(new String[] { "index1", "index2", "index3", "index4" }, responses, failureMap); + return FieldCapabilitiesResponse.builder() + .withIndices(new String[] { "index1", "index2", "index3", "index4" }) + .withFields(responses) + .withFailures(failureMap) + .build(); } public void testChunking() { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTemplateTests.java index 3b7834b67d7a1..950eece0793b7 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTemplateTests.java @@ -146,7 +146,8 @@ private static DataStreamFailureStore.Template normalise(DataStreamFailureStore. template.lifecycleType(), template.enabled(), template.dataRetention().get(), - template.downsampling().get() + template.downsamplingRounds().get(), + template.downsamplingMethod().get() ) ) ); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java index 1d1c352788346..e17d6891b71d6 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java @@ -10,6 +10,7 @@ package org.elasticsearch.cluster.metadata; import org.elasticsearch.action.downsample.DownsampleConfig; +import org.elasticsearch.action.downsample.DownsampleConfigTests; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.TimeValue; @@ -44,27 +45,38 @@ protected DataStreamLifecycle.Template mutateInstance(DataStreamLifecycle.Templa var lifecycleTarget = instance.lifecycleType(); var enabled = instance.enabled(); var retention = instance.dataRetention(); - var downsampling = instance.downsampling(); - switch (randomInt(3)) { + var downsamplingRounds = instance.downsamplingRounds(); + var downsamplingMethod = instance.downsamplingMethod(); + switch (randomInt(4)) { case 0 -> { lifecycleTarget = lifecycleTarget == DataStreamLifecycle.LifecycleType.DATA ? DataStreamLifecycle.LifecycleType.FAILURES : DataStreamLifecycle.LifecycleType.DATA; if (lifecycleTarget == DataStreamLifecycle.LifecycleType.FAILURES) { - downsampling = ResettableValue.undefined(); + downsamplingRounds = ResettableValue.undefined(); + downsamplingMethod = ResettableValue.undefined(); } } case 1 -> enabled = enabled == false; case 2 -> retention = randomValueOtherThan(retention, DataStreamLifecycleTemplateTests::randomRetention); case 3 -> { - downsampling = randomValueOtherThan(downsampling, DataStreamLifecycleTemplateTests::randomDownsampling); - if (downsampling.get() != null) { + downsamplingRounds = randomValueOtherThan(downsamplingRounds, DataStreamLifecycleTemplateTests::randomDownsamplingRounds); + if (downsamplingRounds.get() != null) { + lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; + } else { + downsamplingMethod = ResettableValue.undefined(); + } + } + case 4 -> { + downsamplingMethod = randomValueOtherThan(downsamplingMethod, DataStreamLifecycleTemplateTests::randomDownsamplingMethod); + if (downsamplingMethod.get() != null && downsamplingRounds.get() == null) { + downsamplingRounds = ResettableValue.create(DataStreamLifecycleTests.randomDownsamplingRounds()); lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; } } default -> throw new AssertionError("Illegal randomisation branch"); } - return new DataStreamLifecycle.Template(lifecycleTarget, enabled, retention, downsampling); + return new DataStreamLifecycle.Template(lifecycleTarget, enabled, retention, downsamplingRounds, downsamplingMethod); } public void testDataLifecycleXContentSerialization() throws IOException { @@ -104,16 +116,10 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2h")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(3), - new DownsampleConfig(new DateHistogramInterval("2h")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(3), new DateHistogramInterval("2h")) ) ) .buildTemplate() @@ -127,16 +133,10 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2h")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(30), - new DownsampleConfig(new DateHistogramInterval("2h")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("2h")) ) ) .buildTemplate() @@ -147,16 +147,10 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2h")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(30), - new DownsampleConfig(new DateHistogramInterval("3h")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("3h")) ) ) .buildTemplate() @@ -166,7 +160,7 @@ public void testInvalidDownsamplingConfiguration() { { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> DataStreamLifecycle.dataLifecycleBuilder().downsampling((List.of())).buildTemplate() + () -> DataStreamLifecycle.dataLifecycleBuilder().downsamplingRounds((List.of())).buildTemplate() ); assertThat(exception.getMessage(), equalTo("Downsampling configuration should have at least one round configured.")); } @@ -174,13 +168,13 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( Stream.iterate(1, i -> i * 2) .limit(12) .map( i -> new DataStreamLifecycle.DownsamplingRound( TimeValue.timeValueDays(i), - new DownsampleConfig(new DateHistogramInterval(i + "h")) + new DateHistogramInterval(i + "h") ) ) .toList() @@ -194,13 +188,8 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( - List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2m")) - ) - ) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2m"))) ) .buildTemplate() ); @@ -209,10 +198,31 @@ public void testInvalidDownsamplingConfiguration() { equalTo("A downsampling round must have a fixed interval of at least five minutes but found: 2m") ); } + + { + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingMethod(randomFrom(DownsampleConfig.SamplingMethod.values())) + .buildTemplate() + ); + assertThat( + exception.getMessage(), + equalTo("Downsampling method can only be set when there is at least one downsampling round.") + ); + } } public static DataStreamLifecycle.Template randomDataLifecycleTemplate() { - return DataStreamLifecycle.createDataLifecycleTemplate(randomBoolean(), randomRetention(), randomDownsampling()); + ResettableValue> downsamplingRounds = randomDownsamplingRounds(); + return DataStreamLifecycle.createDataLifecycleTemplate( + randomBoolean(), + randomRetention(), + downsamplingRounds, + downsamplingRounds.get() == null + ? randomBoolean() ? ResettableValue.undefined() : ResettableValue.reset() + : randomDownsamplingMethod() + ); } public void testInvalidLifecycleConfiguration() { @@ -222,13 +232,29 @@ public void testInvalidLifecycleConfiguration() { DataStreamLifecycle.LifecycleType.FAILURES, randomBoolean(), randomBoolean() ? null : DataStreamLifecycleTests.randomPositiveTimeValue(), - DataStreamLifecycleTests.randomDownsampling() + DataStreamLifecycleTests.randomDownsamplingRounds(), + null ) ); assertThat( exception.getMessage(), containsString("Failure store lifecycle does not support downsampling, please remove the downsampling configuration.") ); + + exception = expectThrows( + IllegalArgumentException.class, + () -> new DataStreamLifecycle.Template( + DataStreamLifecycle.LifecycleType.DATA, + randomBoolean(), + randomBoolean() ? null : DataStreamLifecycleTests.randomPositiveTimeValue(), + null, + randomFrom(DownsampleConfig.SamplingMethod.values()) + ) + ); + assertThat( + exception.getMessage(), + containsString("Downsampling method can only be set when there is at least one downsampling round.") + ); } /** @@ -240,6 +266,7 @@ public static DataStreamLifecycle.Template randomFailuresLifecycleTemplate() { DataStreamLifecycle.LifecycleType.FAILURES, randomBoolean(), randomRetention(), + ResettableValue.undefined(), ResettableValue.undefined() ); } @@ -253,10 +280,18 @@ private static ResettableValue randomRetention() { }; } - private static ResettableValue> randomDownsampling() { + private static ResettableValue> randomDownsamplingRounds() { + return switch (randomIntBetween(0, 1)) { + case 0 -> ResettableValue.reset(); + case 1 -> ResettableValue.create(DataStreamLifecycleTests.randomDownsamplingRounds()); + default -> throw new IllegalStateException("Unknown randomisation path"); + }; + } + + private static ResettableValue randomDownsamplingMethod() { return switch (randomIntBetween(0, 1)) { case 0 -> ResettableValue.reset(); - case 1 -> ResettableValue.create(DataStreamLifecycleTests.randomDownsampling()); + case 1 -> ResettableValue.create(DownsampleConfigTests.randomSamplingMethod()); default -> throw new IllegalStateException("Unknown randomisation path"); }; } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java index 198163645c83b..64bc13cb0dc98 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java @@ -12,7 +12,7 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverConditions; import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; import org.elasticsearch.action.admin.indices.rollover.RolloverConfigurationTests; -import org.elasticsearch.action.downsample.DownsampleConfig; +import org.elasticsearch.action.downsample.DownsampleConfigTests; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.ClusterSettings; @@ -64,14 +64,16 @@ protected DataStreamLifecycle mutateInstance(DataStreamLifecycle instance) throw : DataStreamLifecycle.LifecycleType.DATA; var enabled = instance.enabled(); var retention = instance.dataRetention(); - var downsampling = instance.downsampling(); - switch (randomInt(3)) { + var downsamplingRounds = instance.downsamplingRounds(); + var downsamplingMethod = instance.downsamplingMethod(); + switch (randomInt(4)) { case 0 -> { if (instance.targetsFailureStore()) { lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; } else { lifecycleTarget = DataStreamLifecycle.LifecycleType.FAILURES; - downsampling = null; + downsamplingRounds = null; + downsamplingMethod = null; } } case 1 -> { @@ -82,20 +84,31 @@ protected DataStreamLifecycle mutateInstance(DataStreamLifecycle instance) throw } } case 2 -> { - if (downsampling == null) { - downsampling = randomDownsampling(); + if (downsamplingRounds == null) { + downsamplingRounds = randomDownsamplingRounds(); if (lifecycleTarget == DataStreamLifecycle.LifecycleType.FAILURES) { lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; } } else { - downsampling = randomBoolean() + downsamplingRounds = randomBoolean() ? null - : randomValueOtherThan(downsampling, DataStreamLifecycleTests::randomDownsampling); + : randomValueOtherThan(downsamplingRounds, DataStreamLifecycleTests::randomDownsamplingRounds); + if (downsamplingRounds == null) { + downsamplingMethod = null; + } + } + } + case 3 -> { + // We need to enable downsampling in order to add a non-value downsampling method + downsamplingMethod = randomValueOtherThan(downsamplingMethod, DownsampleConfigTests::randomSamplingMethod); + if (downsamplingMethod != null && downsamplingRounds == null) { + downsamplingRounds = randomDownsamplingRounds(); + lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; } } default -> enabled = enabled == false; } - return new DataStreamLifecycle(lifecycleTarget, enabled, retention, downsampling); + return new DataStreamLifecycle(lifecycleTarget, enabled, retention, downsamplingRounds, downsamplingMethod); } public void testDataLifecycleXContentSerialization() throws IOException { @@ -215,16 +228,10 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2h")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(3), - new DownsampleConfig(new DateHistogramInterval("2h")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(3), new DateHistogramInterval("2h")) ) ) .build() @@ -238,16 +245,10 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2h")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(30), - new DownsampleConfig(new DateHistogramInterval("2h")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("2h")) ) ) .build() @@ -258,16 +259,10 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2h")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(30), - new DownsampleConfig(new DateHistogramInterval("3h")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("3h")) ) ) .build() @@ -277,7 +272,7 @@ public void testInvalidDownsamplingConfiguration() { { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> DataStreamLifecycle.dataLifecycleBuilder().downsampling((List.of())).build() + () -> DataStreamLifecycle.dataLifecycleBuilder().downsamplingRounds((List.of())).build() ); assertThat(exception.getMessage(), equalTo("Downsampling configuration should have at least one round configured.")); } @@ -285,13 +280,13 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( Stream.iterate(1, i -> i * 2) .limit(12) .map( i -> new DataStreamLifecycle.DownsamplingRound( TimeValue.timeValueDays(i), - new DownsampleConfig(new DateHistogramInterval(i + "h")) + new DateHistogramInterval(i + "h") ) ) .toList() @@ -305,13 +300,8 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( - List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2m")) - ) - ) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2m"))) ) .build() ); @@ -326,7 +316,7 @@ public void testEffectiveRetention() { // No retention in the data stream lifecycle { DataStreamLifecycle noDataRetentionLifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling(randomDownsampling()) + .downsamplingRounds(randomDownsamplingRounds()) .build(); DataStreamLifecycle noFailuresRetentionLifecycle = DataStreamLifecycle.failuresLifecycleBuilder().build(); TimeValue maxRetention = TimeValue.timeValueDays(randomIntBetween(50, 100)); @@ -381,7 +371,7 @@ public void testEffectiveRetention() { TimeValue dataStreamRetention = TimeValue.timeValueDays(randomIntBetween(5, 100)); DataStreamLifecycle dataLifecycleRetention = DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(dataStreamRetention) - .downsampling(randomDownsampling()) + .downsamplingRounds(randomDownsamplingRounds()) .build(); DataStreamLifecycle failuresLifecycleRetention = DataStreamLifecycle.failuresLifecycleBuilder() .dataRetention(dataStreamRetention) @@ -510,9 +500,11 @@ public void testEffectiveRetentionParams() { } public static DataStreamLifecycle randomDataLifecycle() { + List downsamplingRounds = randomBoolean() ? null : randomDownsamplingRounds(); return DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(randomBoolean() ? null : randomTimeValue(1, 365, TimeUnit.DAYS)) - .downsampling(randomBoolean() ? null : randomDownsampling()) + .downsamplingRounds(downsamplingRounds) + .downsamplingMethod(downsamplingRounds == null ? null : DownsampleConfigTests.randomSamplingMethod()) .enabled(randomBoolean()) .build(); } @@ -528,12 +520,12 @@ public static DataStreamLifecycle randomFailuresLifecycle() { .build(); } - static List randomDownsampling() { + static List randomDownsamplingRounds() { var count = randomIntBetween(0, 9); List rounds = new ArrayList<>(); var previous = new DataStreamLifecycle.DownsamplingRound( randomTimeValue(1, 365, TimeUnit.DAYS), - new DownsampleConfig(new DateHistogramInterval(randomIntBetween(1, 24) + "h")) + new DateHistogramInterval(randomIntBetween(1, 24) + "h") ); rounds.add(previous); for (int i = 0; i < count; i++) { @@ -546,9 +538,7 @@ static List randomDownsampling() { private static DataStreamLifecycle.DownsamplingRound nextRound(DataStreamLifecycle.DownsamplingRound previous) { var after = TimeValue.timeValueDays(previous.after().days() + randomIntBetween(1, 10)); - var fixedInterval = new DownsampleConfig( - new DateHistogramInterval((previous.config().getFixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms") - ); + var fixedInterval = new DateHistogramInterval((previous.fixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms"); return new DataStreamLifecycle.DownsamplingRound(after, fixedInterval); } @@ -561,7 +551,8 @@ public void testInvalidLifecycleConfiguration() { DataStreamLifecycle.LifecycleType.FAILURES, null, null, - DataStreamLifecycleTests.randomDownsampling() + DataStreamLifecycleTests.randomDownsamplingRounds(), + DownsampleConfigTests.randomSamplingMethod() ) ) ); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java index 3b7a28a894156..9e1a06126fa96 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java @@ -34,7 +34,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import static org.elasticsearch.cluster.metadata.DataStreamLifecycleTests.randomDownsampling; +import static org.elasticsearch.cluster.metadata.DataStreamLifecycleTests.randomDownsamplingRounds; import static org.elasticsearch.common.settings.Settings.builder; import static org.elasticsearch.indices.ShardLimitValidatorTests.createTestShardLimitService; import static org.hamcrest.Matchers.containsString; @@ -60,7 +60,9 @@ public void testNoHeaderWarning() { ThreadContext threadContext = new ThreadContext(Settings.EMPTY); HeaderWarning.setThreadContext(threadContext); - DataStreamLifecycle noRetentionLifecycle = DataStreamLifecycle.dataLifecycleBuilder().downsampling(randomDownsampling()).build(); + DataStreamLifecycle noRetentionLifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingRounds(randomDownsamplingRounds()) + .build(); noRetentionLifecycle.addWarningHeaderIfDataRetentionNotEffective(null, randomBoolean()); Map> responseHeaders = threadContext.getResponseHeaders(); assertThat(responseHeaders.isEmpty(), is(true)); @@ -68,7 +70,7 @@ public void testNoHeaderWarning() { TimeValue dataStreamRetention = TimeValue.timeValueDays(randomIntBetween(5, 100)); DataStreamLifecycle lifecycleWithRetention = DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(dataStreamRetention) - .downsampling(randomDownsampling()) + .downsamplingRounds(randomDownsamplingRounds()) .build(); DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention( TimeValue.timeValueDays(2), @@ -83,7 +85,9 @@ public void testDefaultRetentionHeaderWarning() { ThreadContext threadContext = new ThreadContext(Settings.EMPTY); HeaderWarning.setThreadContext(threadContext); - DataStreamLifecycle noRetentionLifecycle = DataStreamLifecycle.dataLifecycleBuilder().downsampling(randomDownsampling()).build(); + DataStreamLifecycle noRetentionLifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingRounds(randomDownsamplingRounds()) + .build(); DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention( randomTimeValue(2, 10, TimeUnit.DAYS), randomBoolean() ? null : TimeValue.timeValueDays(20) @@ -107,7 +111,7 @@ public void testMaxRetentionHeaderWarning() { TimeValue maxRetention = randomTimeValue(2, 100, TimeUnit.DAYS); DataStreamLifecycle lifecycle = DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(randomBoolean() ? null : TimeValue.timeValueDays(maxRetention.days() + 1)) - .downsampling(randomDownsampling()) + .downsamplingRounds(randomDownsamplingRounds()) .build(); DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(null, maxRetention); lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention, false); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index c822b066629f8..e3a2147991ac4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; import org.elasticsearch.action.admin.indices.rollover.RolloverConfigurationTests; import org.elasticsearch.action.admin.indices.rollover.RolloverInfo; -import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; @@ -1590,19 +1589,13 @@ public void testGetDownsampleRounds() { settings(IndexVersion.current()).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "@timestamp"), DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(2000), - new DownsampleConfig(new DateHistogramInterval("10m")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(3200), - new DownsampleConfig(new DateHistogramInterval("100m")) - ), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(2000), new DateHistogramInterval("10m")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(3200), new DateHistogramInterval("100m")), new DataStreamLifecycle.DownsamplingRound( TimeValue.timeValueMillis(3500), - new DownsampleConfig(new DateHistogramInterval("1000m")) + new DateHistogramInterval("1000m") ) ) @@ -1648,20 +1641,11 @@ public void testGetDownsampleRounds() { // no TSDB settings settings(IndexVersion.current()), DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(2000), - new DownsampleConfig(new DateHistogramInterval("10m")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(3200), - new DownsampleConfig(new DateHistogramInterval("100m")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(3500), - new DownsampleConfig(new DateHistogramInterval("1000m")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(2000), new DateHistogramInterval("10m")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(3200), new DateHistogramInterval("100m")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(3500), new DateHistogramInterval("1000m")) ) ) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index a94b92527bc53..4677ca1aed9b3 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -1104,26 +1104,40 @@ public void testResolveLifecycle() throws Exception { DataStreamLifecycle.Template lifecycle45d = DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(TimeValue.timeValueDays(45)) - .downsampling( - List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(30), - new DownsampleConfig(new DateHistogramInterval("3h")) - ) - ) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("3h"))) ) .buildTemplate(); String ct45d = "ct_45d"; project = addComponentTemplate(service, project, ct45d, lifecycle45d); + DataStreamLifecycle.Template lifecycle60d = DataStreamLifecycle.dataLifecycleBuilder() + .dataRetention(TimeValue.timeValueDays(60)) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(7), new DateHistogramInterval("3h"))) + ) + .downsamplingMethod(DownsampleConfig.SamplingMethod.LAST_VALUE) + .buildTemplate(); + String ct60d = "ct_60d"; + project = addComponentTemplate(service, project, ct60d, lifecycle60d); DataStreamLifecycle.Template lifecycleNullRetention = DataStreamLifecycle.createDataLifecycleTemplate( true, ResettableValue.reset(), + ResettableValue.undefined(), ResettableValue.undefined() ); String ctNullRetention = "ct_null_retention"; project = addComponentTemplate(service, project, ctNullRetention, lifecycleNullRetention); + DataStreamLifecycle.Template lifecycleNullDownsampling = DataStreamLifecycle.createDataLifecycleTemplate( + true, + ResettableValue.undefined(), + ResettableValue.reset(), + ResettableValue.reset() + ); + String ctNullDownsampling = "ct_null_downsampling"; + project = addComponentTemplate(service, project, ctNullDownsampling, lifecycleNullDownsampling); + String ctEmptyLifecycle = "ct_empty_lifecycle"; project = addComponentTemplate(service, project, ctEmptyLifecycle, emptyLifecycle); @@ -1167,7 +1181,7 @@ public void testResolveLifecycle() throws Exception { lifecycle30d, DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(lifecycle30d.dataRetention()) - .downsampling(lifecycle45d.downsampling()) + .downsamplingRounds(lifecycle45d.downsamplingRounds()) .buildTemplate() ); @@ -1189,7 +1203,7 @@ public void testResolveLifecycle() throws Exception { project, List.of(ctEmptyLifecycle, ct45d), lifecycleNullRetention, - DataStreamLifecycle.dataLifecycleBuilder().downsampling(lifecycle45d.downsampling()).buildTemplate() + DataStreamLifecycle.dataLifecycleBuilder().downsamplingRounds(lifecycle45d.downsamplingRounds()).buildTemplate() ); // Component A: "lifecycle": {"retention": "30d"} @@ -1203,7 +1217,7 @@ public void testResolveLifecycle() throws Exception { DataStreamLifecycle.dataLifecycleBuilder().enabled(false).buildTemplate(), DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(lifecycle45d.dataRetention()) - .downsampling(lifecycle45d.downsampling()) + .downsamplingRounds(lifecycle45d.downsamplingRounds()) .enabled(false) .buildTemplate() ); @@ -1225,6 +1239,72 @@ public void testResolveLifecycle() throws Exception { // Composable Z: "lifecycle": {"retention": "45d", "downsampling": [{"after": "30d", "fixed_interval": "3h"}]} // Result: "lifecycle": {"retention": "45d", "downsampling": [{"after": "30d", "fixed_interval": "3h"}]} assertLifecycleResolution(service, project, List.of(ct30d, ctDisabledLifecycle), lifecycle45d, lifecycle45d); + + // Component A: "lifecycle": { + // "retention": "60d", + // "downsampling_method": "last_value", + // "downsampling": [{"after": "3d", "fixed_interval": "3h"}] + // } + // Composable Z: "lifecycle": {"retention": "45d", "downsampling": [{"after": "30d", "fixed_interval": "3h"}]} + // Result: "lifecycle": { + // "retention": "45d", + // "downsampling": [{"after": "30d", "fixed_interval": "3h"}], + // "downsampling_method": "last_value" + // } + assertLifecycleResolution( + service, + project, + List.of(ct60d), + lifecycle45d, + DataStreamLifecycle.dataLifecycleBuilder() + .dataRetention(lifecycle45d.dataRetention()) + .downsamplingMethod(lifecycle60d.downsamplingMethod()) + .downsamplingRounds(lifecycle45d.downsamplingRounds()) + .buildTemplate() + ); + + // Component A: "lifecycle": { + // "retention": "60d", + // "downsampling_method": "last_value", + // "downsampling": [{"after": "3d", "fixed_interval": "3h"}] + // } + // Component B: "lifecycle": {"retention": "45d", "downsampling": [{"after": "30d", "fixed_interval": "3h"}]} + // Composable Z: "lifecycle": {"downsampling": null, "downsampling_method": null} + // Result: "lifecycle": {"retention": "45d"} + assertLifecycleResolution( + service, + project, + List.of(ct60d, ct45d), + lifecycleNullDownsampling, + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(lifecycle45d.dataRetention()).buildTemplate() + ); + + // Component A: "lifecycle": { + // "retention": "60d", + // "downsampling_method": "last_value", + // "downsampling": [{"after": "3d", "fixed_interval": "3h"}] + // } + // Composable Z: "lifecycle": {"retention": "45d", "downsampling": [{"after": "30d", "fixed_interval": "3h"}]} + // Result: "lifecycle": { + // "retention": "45d", + // "downsampling": [{"after": "30d", "fixed_interval": "3h"}], + // "downsampling_method": "last_value" + // } + assertLifecycleResolution( + service, + project, + List.of(ct60d), + DataStreamLifecycle.createDataLifecycleTemplate( + true, + ResettableValue.undefined(), + ResettableValue.undefined(), + ResettableValue.reset() + ), + DataStreamLifecycle.dataLifecycleBuilder() + .dataRetention(lifecycle60d.dataRetention()) + .downsamplingRounds(lifecycle60d.downsamplingRounds()) + .buildTemplate() + ); } public void testResolveFailureStore() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/TsdbDocValueBwcTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/TsdbDocValueBwcTests.java index 2c40b384c45ba..d880bc468e61a 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/tsdb/TsdbDocValueBwcTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/TsdbDocValueBwcTests.java @@ -52,6 +52,7 @@ import java.util.Map; import java.util.function.IntSupplier; +import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormatTests.TestES819TSDBDocValuesFormatVersion0; import static org.hamcrest.Matchers.equalTo; public class TsdbDocValueBwcTests extends ESTestCase { @@ -62,6 +63,13 @@ public void testMixedIndex() throws Exception { testMixedIndex(oldCodec, newCodec); } + // TODO update Current to Version1 once version is incremented + public void testMixedIndexDocValueVersion0ToCurrent() throws Exception { + var oldCodec = TestUtil.alwaysDocValuesFormat(new TestES819TSDBDocValuesFormatVersion0()); + var newCodec = TestUtil.alwaysDocValuesFormat(new ES819TSDBDocValuesFormat()); + testMixedIndex(oldCodec, newCodec, this::assertVersion819, this::assertVersion819); + } + public void testMixedIndex816To900Lucene101() throws Exception { var oldCodec = new Elasticsearch816Codec() { @@ -84,8 +92,37 @@ public DocValuesFormat getDocValuesFormatForField(String field) { testMixedIndex(oldCodec, newCodec); } + void assertFieldInfoDocValuesFormat(DirectoryReader reader, String expectedSuffix, String expectedFormat) throws IOException, + NoSuchFieldException, IllegalAccessException { + // Assert per field format field info attributes: + // (XPerFieldDocValuesFormat must produce the same attributes as PerFieldDocValuesFormat for BWC. + // Otherwise, doc values fields may disappear) + for (var leaf : reader.leaves()) { + for (var fieldInfo : leaf.reader().getFieldInfos()) { + assertThat(fieldInfo.attributes(), Matchers.aMapWithSize(2)); + assertThat(fieldInfo.attributes(), Matchers.hasEntry("PerFieldDocValuesFormat.suffix", expectedSuffix)); + assertThat(fieldInfo.attributes(), Matchers.hasEntry("PerFieldDocValuesFormat.format", expectedFormat)); + } + } + } + + void assertVersion87(DirectoryReader reader) throws IOException, NoSuchFieldException, IllegalAccessException { + assert87DocValuesFormatVersion(reader); + assertFieldInfoDocValuesFormat(reader, "0", "ES87TSDB"); + } + + void assertVersion819(DirectoryReader reader) throws IOException, NoSuchFieldException, ClassNotFoundException, IllegalAccessException { + assert819DocValuesFormatVersion(reader); + assertFieldInfoDocValuesFormat(reader, "0", "ES819TSDB"); + } + void testMixedIndex(Codec oldCodec, Codec newCodec) throws IOException, NoSuchFieldException, IllegalAccessException, ClassNotFoundException { + testMixedIndex(oldCodec, newCodec, this::assertVersion87, this::assertVersion819); + } + + void testMixedIndex(Codec oldCodec, Codec newCodec, VersionAssert assertOldVersion, VersionAssert assertNewVersion) throws IOException, + NoSuchFieldException, IllegalAccessException, ClassNotFoundException { String timestampField = "@timestamp"; String hostnameField = "host.name"; long baseTimestamp = 1704067200000L; @@ -136,17 +173,7 @@ void testMixedIndex(Codec oldCodec, Codec newCodec) throws IOException, NoSuchFi } // Check documents before force merge: try (var reader = DirectoryReader.open(dir)) { - assertOldDocValuesFormatVersion(reader); - // Assert per field format field info attributes: - // (XPerFieldDocValuesFormat must produce the same attributes as PerFieldDocValuesFormat for BWC. - // Otherwise, doc values fields may disappear) - for (var leaf : reader.leaves()) { - for (var fieldInfo : leaf.reader().getFieldInfos()) { - assertThat(fieldInfo.attributes(), Matchers.aMapWithSize(2)); - assertThat(fieldInfo.attributes(), Matchers.hasEntry("PerFieldDocValuesFormat.suffix", "0")); - assertThat(fieldInfo.attributes(), Matchers.hasEntry("PerFieldDocValuesFormat.format", "ES87TSDB")); - } - } + assertOldVersion.run(reader); var hostNameDV = MultiDocValues.getSortedValues(reader, hostnameField); assertNotNull(hostNameDV); @@ -205,17 +232,9 @@ void testMixedIndex(Codec oldCodec, Codec newCodec) throws IOException, NoSuchFi try (var reader = DirectoryReader.open(iw)) { assertEquals(1, reader.leaves().size()); assertEquals(numDocs, reader.maxDoc()); - assertNewDocValuesFormatVersion(reader); - var leaf = reader.leaves().get(0).reader(); - // Assert per field format field info attributes: - // (XPerFieldDocValuesFormat must produce the same attributes as PerFieldDocValuesFormat for BWC. - // Otherwise, doc values fields may disappear) - for (var fieldInfo : leaf.getFieldInfos()) { - assertThat(fieldInfo.attributes(), Matchers.aMapWithSize(2)); - assertThat(fieldInfo.attributes(), Matchers.hasEntry("PerFieldDocValuesFormat.suffix", "0")); - assertThat(fieldInfo.attributes(), Matchers.hasEntry("PerFieldDocValuesFormat.format", "ES819TSDB")); - } + assertNewVersion.run(reader); + var leaf = reader.leaves().get(0).reader(); var hostNameDV = leaf.getSortedDocValues(hostnameField); assertNotNull(hostNameDV); var timestampDV = DocValues.unwrapSingleton(leaf.getSortedNumericDocValues(timestampField)); @@ -385,7 +404,7 @@ private IndexWriterConfig getTimeSeriesIndexWriterConfig(String hostnameField, S // A hacky way to figure out whether doc values format is written in what version. Need to use reflection, because // PerFieldDocValuesFormat hides the doc values formats it wraps. - private void assertOldDocValuesFormatVersion(DirectoryReader reader) throws NoSuchFieldException, IllegalAccessException, IOException { + private void assert87DocValuesFormatVersion(DirectoryReader reader) throws NoSuchFieldException, IllegalAccessException, IOException { if (System.getSecurityManager() != null) { // With jvm version 24 entitlements are used and security manager is nog longer used. // Making this assertion work with security manager requires granting the entire test codebase privileges to use @@ -406,7 +425,7 @@ private void assertOldDocValuesFormatVersion(DirectoryReader reader) throws NoSu } } - private void assertNewDocValuesFormatVersion(DirectoryReader reader) throws NoSuchFieldException, IllegalAccessException, IOException, + private void assert819DocValuesFormatVersion(DirectoryReader reader) throws NoSuchFieldException, IllegalAccessException, IOException, ClassNotFoundException { for (var leafReaderContext : reader.leaves()) { @@ -451,4 +470,7 @@ private static Field getFormatsFieldFromPerFieldFieldsReader(Class c) throws return field; } + interface VersionAssert { + void run(DirectoryReader reader) throws IOException, NoSuchFieldException, IllegalAccessException, ClassNotFoundException; + } } diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesConsumerVersion0.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesConsumerVersion0.java new file mode 100644 index 0000000000000..df669f007db2b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesConsumerVersion0.java @@ -0,0 +1,1004 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.tsdb.es819; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.codecs.lucene90.IndexedDISI; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipIndexType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.MergeState; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.SortedSetSelector; +import org.apache.lucene.store.ByteArrayDataOutput; +import org.apache.lucene.store.ByteBuffersDataOutput; +import org.apache.lucene.store.ByteBuffersIndexOutput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.LongsRef; +import org.apache.lucene.util.StringHelper; +import org.apache.lucene.util.compress.LZ4; +import org.apache.lucene.util.packed.DirectMonotonicWriter; +import org.apache.lucene.util.packed.PackedInts; +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.index.codec.tsdb.TSDBDocValuesEncoder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.elasticsearch.index.codec.tsdb.es819.DocValuesConsumerUtil.compatibleWithOptimizedMerge; +import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; +import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.SKIP_INDEX_LEVEL_SHIFT; +import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.SKIP_INDEX_MAX_LEVEL; +import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.SORTED_SET; + +final class ES819TSDBDocValuesConsumerVersion0 extends XDocValuesConsumer { + + final Directory dir; + final IOContext context; + IndexOutput data, meta; + final int maxDoc; + private byte[] termsDictBuffer; + private final int skipIndexIntervalSize; + private final int minDocsPerOrdinalForOrdinalRangeEncoding; + final boolean enableOptimizedMerge; + private final int primarySortFieldNumber; + + ES819TSDBDocValuesConsumerVersion0( + SegmentWriteState state, + int skipIndexIntervalSize, + int minDocsPerOrdinalForOrdinalRangeEncoding, + boolean enableOptimizedMerge, + String dataCodec, + String dataExtension, + String metaCodec, + String metaExtension + ) throws IOException { + this.termsDictBuffer = new byte[1 << 14]; + this.dir = state.directory; + this.minDocsPerOrdinalForOrdinalRangeEncoding = minDocsPerOrdinalForOrdinalRangeEncoding; + this.primarySortFieldNumber = ES819TSDBDocValuesProducer.primarySortFieldNumber(state.segmentInfo, state.fieldInfos); + this.context = state.context; + boolean success = false; + try { + final String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension); + data = state.directory.createOutput(dataName, state.context); + CodecUtil.writeIndexHeader( + data, + dataCodec, + ES819TSDBDocValuesFormat.VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix + ); + String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension); + meta = state.directory.createOutput(metaName, state.context); + CodecUtil.writeIndexHeader( + meta, + metaCodec, + ES819TSDBDocValuesFormat.VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix + ); + maxDoc = state.segmentInfo.maxDoc(); + this.skipIndexIntervalSize = skipIndexIntervalSize; + this.enableOptimizedMerge = enableOptimizedMerge; + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } + } + + @Override + public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + meta.writeInt(field.number); + meta.writeByte(ES819TSDBDocValuesFormat.NUMERIC); + var producer = new TsdbDocValuesProducer(valuesProducer) { + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { + return DocValues.singleton(valuesProducer.getNumeric(field)); + } + }; + if (field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) { + writeSkipIndex(field, producer); + } + + writeField(field, producer, -1, null); + } + + private boolean shouldEncodeOrdinalRange(FieldInfo field, long maxOrd, int numDocsWithValue, long numValues) { + return maxDoc > 1 + && field.number == primarySortFieldNumber + && numDocsWithValue == numValues // Only single valued fields can be supported with range encoded ordinals format + && (numDocsWithValue / maxOrd) >= minDocsPerOrdinalForOrdinalRangeEncoding; + } + + private long[] writeField(FieldInfo field, TsdbDocValuesProducer valuesProducer, long maxOrd, OffsetsAccumulator offsetsAccumulator) + throws IOException { + int numDocsWithValue = 0; + long numValues = 0; + + SortedNumericDocValues values; + if (valuesProducer.mergeStats.supported()) { + numDocsWithValue = valuesProducer.mergeStats.sumNumDocsWithField(); + numValues = valuesProducer.mergeStats.sumNumValues(); + } else { + values = valuesProducer.getSortedNumeric(field); + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + numDocsWithValue++; + final int count = values.docValueCount(); + numValues += count; + } + } + + meta.writeLong(numValues); + meta.writeInt(numDocsWithValue); + + DISIAccumulator disiAccumulator = null; + try { + if (numValues > 0) { + assert numDocsWithValue > 0; + final ByteBuffersDataOutput indexOut = new ByteBuffersDataOutput(); + DirectMonotonicWriter indexWriter = null; + + final long valuesDataOffset = data.getFilePointer(); + if (maxOrd == 1) { + // Special case for maxOrd of 1, signal -1 that no blocks will be written + meta.writeInt(-1); + } else if (shouldEncodeOrdinalRange(field, maxOrd, numDocsWithValue, numValues)) { + assert offsetsAccumulator == null; + // When a field is sorted, use ordinal range encode for long runs of the same ordinal. + meta.writeInt(-2); + meta.writeVInt(Math.toIntExact(maxOrd)); + meta.writeByte((byte) ES819TSDBDocValuesFormat.ORDINAL_RANGE_ENCODING_BLOCK_SHIFT); + values = valuesProducer.getSortedNumeric(field); + if (valuesProducer.mergeStats.supported() && numDocsWithValue < maxDoc) { + disiAccumulator = new DISIAccumulator(dir, context, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER); + } + DirectMonotonicWriter startDocs = DirectMonotonicWriter.getInstance( + meta, + data, + maxOrd + 1, + ES819TSDBDocValuesFormat.ORDINAL_RANGE_ENCODING_BLOCK_SHIFT + ); + long lastOrd = 0; + startDocs.add(0); + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + if (disiAccumulator != null) { + disiAccumulator.addDocId(doc); + } + final long nextOrd = values.nextValue(); + if (nextOrd != lastOrd) { + lastOrd = nextOrd; + startDocs.add(doc); + } + } + startDocs.add(maxDoc); + startDocs.finish(); + } else { + indexWriter = DirectMonotonicWriter.getInstance( + meta, + new ByteBuffersIndexOutput(indexOut, "temp-dv-index", "temp-dv-index"), + 1L + ((numValues - 1) >>> ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SHIFT), + ES819TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT + ); + meta.writeInt(DIRECT_MONOTONIC_BLOCK_SHIFT); + final long[] buffer = new long[ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE]; + int bufferSize = 0; + final TSDBDocValuesEncoder encoder = new TSDBDocValuesEncoder(ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE); + values = valuesProducer.getSortedNumeric(field); + final int bitsPerOrd = maxOrd >= 0 ? PackedInts.bitsRequired(maxOrd - 1) : -1; + if (valuesProducer.mergeStats.supported() && numDocsWithValue < maxDoc) { + disiAccumulator = new DISIAccumulator(dir, context, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER); + } + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + if (disiAccumulator != null) { + disiAccumulator.addDocId(doc); + } + final int count = values.docValueCount(); + if (offsetsAccumulator != null) { + offsetsAccumulator.addDoc(count); + } + for (int i = 0; i < count; ++i) { + buffer[bufferSize++] = values.nextValue(); + if (bufferSize == ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE) { + indexWriter.add(data.getFilePointer() - valuesDataOffset); + if (maxOrd >= 0) { + encoder.encodeOrdinals(buffer, data, bitsPerOrd); + } else { + encoder.encode(buffer, data); + } + bufferSize = 0; + } + } + } + if (bufferSize > 0) { + indexWriter.add(data.getFilePointer() - valuesDataOffset); + // Fill unused slots in the block with zeroes rather than junk + Arrays.fill(buffer, bufferSize, ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE, 0L); + if (maxOrd >= 0) { + encoder.encodeOrdinals(buffer, data, bitsPerOrd); + } else { + encoder.encode(buffer, data); + } + } + } + + final long valuesDataLength = data.getFilePointer() - valuesDataOffset; + if (indexWriter != null) { + indexWriter.finish(); + } + final long indexDataOffset = data.getFilePointer(); + data.copyBytes(indexOut.toDataInput(), indexOut.size()); + meta.writeLong(indexDataOffset); + meta.writeLong(data.getFilePointer() - indexDataOffset); + + meta.writeLong(valuesDataOffset); + meta.writeLong(valuesDataLength); + } + + if (numDocsWithValue == 0) { // meta[-2, 0]: No documents with values + meta.writeLong(-2); // docsWithFieldOffset + meta.writeLong(0L); // docsWithFieldLength + meta.writeShort((short) -1); // jumpTableEntryCount + meta.writeByte((byte) -1); // denseRankPower + } else if (numDocsWithValue == maxDoc) { // meta[-1, 0]: All documents have values + meta.writeLong(-1); // docsWithFieldOffset + meta.writeLong(0L); // docsWithFieldLength + meta.writeShort((short) -1); // jumpTableEntryCount + meta.writeByte((byte) -1); // denseRankPower + } else { // meta[data.offset, data.length]: IndexedDISI structure for documents with values + long offset = data.getFilePointer(); + meta.writeLong(offset); // docsWithFieldOffset + final short jumpTableEntryCount; + if (maxOrd != 1 && disiAccumulator != null) { + jumpTableEntryCount = disiAccumulator.build(data); + } else { + values = valuesProducer.getSortedNumeric(field); + jumpTableEntryCount = IndexedDISI.writeBitSet(values, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER); + } + meta.writeLong(data.getFilePointer() - offset); // docsWithFieldLength + meta.writeShort(jumpTableEntryCount); + meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER); + } + } finally { + IOUtils.close(disiAccumulator); + } + + return new long[] { numDocsWithValue, numValues }; + } + + @Override + public void mergeNumericField(FieldInfo mergeFieldInfo, MergeState mergeState) throws IOException { + var result = compatibleWithOptimizedMerge(enableOptimizedMerge, mergeState, mergeFieldInfo); + if (result.supported()) { + mergeNumericField(result, mergeFieldInfo, mergeState); + } else { + super.mergeNumericField(mergeFieldInfo, mergeState); + } + } + + @Override + public void mergeBinaryField(FieldInfo mergeFieldInfo, MergeState mergeState) throws IOException { + var result = compatibleWithOptimizedMerge(enableOptimizedMerge, mergeState, mergeFieldInfo); + if (result.supported()) { + mergeBinaryField(result, mergeFieldInfo, mergeState); + } else { + super.mergeBinaryField(mergeFieldInfo, mergeState); + } + } + + @Override + public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + meta.writeInt(field.number); + meta.writeByte(ES819TSDBDocValuesFormat.BINARY); + + if (valuesProducer instanceof TsdbDocValuesProducer tsdbValuesProducer && tsdbValuesProducer.mergeStats.supported()) { + final int numDocsWithField = tsdbValuesProducer.mergeStats.sumNumDocsWithField(); + final int minLength = tsdbValuesProducer.mergeStats.minLength(); + final int maxLength = tsdbValuesProducer.mergeStats.maxLength(); + + assert numDocsWithField <= maxDoc; + + BinaryDocValues values = valuesProducer.getBinary(field); + long start = data.getFilePointer(); + meta.writeLong(start); // dataOffset + + OffsetsAccumulator offsetsAccumulator = null; + DISIAccumulator disiAccumulator = null; + try { + if (numDocsWithField > 0 && numDocsWithField < maxDoc) { + disiAccumulator = new DISIAccumulator(dir, context, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER); + } + + assert maxLength >= minLength; + if (maxLength > minLength) { + offsetsAccumulator = new OffsetsAccumulator(dir, context, data, numDocsWithField); + } + + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + BytesRef v = values.binaryValue(); + data.writeBytes(v.bytes, v.offset, v.length); + if (disiAccumulator != null) { + disiAccumulator.addDocId(doc); + } + if (offsetsAccumulator != null) { + offsetsAccumulator.addDoc(v.length); + } + } + meta.writeLong(data.getFilePointer() - start); // dataLength + + if (numDocsWithField == 0) { + meta.writeLong(-2); // docsWithFieldOffset + meta.writeLong(0L); // docsWithFieldLength + meta.writeShort((short) -1); // jumpTableEntryCount + meta.writeByte((byte) -1); // denseRankPower + } else if (numDocsWithField == maxDoc) { + meta.writeLong(-1); // docsWithFieldOffset + meta.writeLong(0L); // docsWithFieldLength + meta.writeShort((short) -1); // jumpTableEntryCount + meta.writeByte((byte) -1); // denseRankPower + } else { + long offset = data.getFilePointer(); + meta.writeLong(offset); // docsWithFieldOffset + final short jumpTableEntryCount = disiAccumulator.build(data); + meta.writeLong(data.getFilePointer() - offset); // docsWithFieldLength + meta.writeShort(jumpTableEntryCount); + meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER); + } + + meta.writeInt(numDocsWithField); + meta.writeInt(minLength); + meta.writeInt(maxLength); + if (offsetsAccumulator != null) { + offsetsAccumulator.build(meta, data); + } + } finally { + IOUtils.close(disiAccumulator, offsetsAccumulator); + } + } else { + BinaryDocValues values = valuesProducer.getBinary(field); + long start = data.getFilePointer(); + meta.writeLong(start); // dataOffset + int numDocsWithField = 0; + int minLength = Integer.MAX_VALUE; + int maxLength = 0; + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + numDocsWithField++; + BytesRef v = values.binaryValue(); + int length = v.length; + data.writeBytes(v.bytes, v.offset, v.length); + minLength = Math.min(length, minLength); + maxLength = Math.max(length, maxLength); + } + assert numDocsWithField <= maxDoc; + meta.writeLong(data.getFilePointer() - start); // dataLength + + if (numDocsWithField == 0) { + meta.writeLong(-2); // docsWithFieldOffset + meta.writeLong(0L); // docsWithFieldLength + meta.writeShort((short) -1); // jumpTableEntryCount + meta.writeByte((byte) -1); // denseRankPower + } else if (numDocsWithField == maxDoc) { + meta.writeLong(-1); // docsWithFieldOffset + meta.writeLong(0L); // docsWithFieldLength + meta.writeShort((short) -1); // jumpTableEntryCount + meta.writeByte((byte) -1); // denseRankPower + } else { + long offset = data.getFilePointer(); + meta.writeLong(offset); // docsWithFieldOffset + values = valuesProducer.getBinary(field); + final short jumpTableEntryCount = IndexedDISI.writeBitSet(values, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER); + meta.writeLong(data.getFilePointer() - offset); // docsWithFieldLength + meta.writeShort(jumpTableEntryCount); + meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER); + } + + meta.writeInt(numDocsWithField); + meta.writeInt(minLength); + meta.writeInt(maxLength); + if (maxLength > minLength) { + start = data.getFilePointer(); + meta.writeLong(start); + meta.writeVInt(ES819TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT); + + final DirectMonotonicWriter writer = DirectMonotonicWriter.getInstance( + meta, + data, + numDocsWithField + 1, + ES819TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT + ); + long addr = 0; + writer.add(addr); + values = valuesProducer.getBinary(field); + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + addr += values.binaryValue().length; + writer.add(addr); + } + writer.finish(); + meta.writeLong(data.getFilePointer() - start); + } + } + } + + @Override + public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + meta.writeInt(field.number); + meta.writeByte(ES819TSDBDocValuesFormat.SORTED); + doAddSortedField(field, valuesProducer, false); + } + + @Override + public void mergeSortedField(FieldInfo mergeFieldInfo, MergeState mergeState) throws IOException { + var result = compatibleWithOptimizedMerge(enableOptimizedMerge, mergeState, mergeFieldInfo); + if (result.supported()) { + mergeSortedField(result, mergeFieldInfo, mergeState); + } else { + super.mergeSortedField(mergeFieldInfo, mergeState); + } + } + + private void doAddSortedField(FieldInfo field, DocValuesProducer valuesProducer, boolean addTypeByte) throws IOException { + var producer = new TsdbDocValuesProducer(valuesProducer) { + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { + SortedDocValues sorted = valuesProducer.getSorted(field); + NumericDocValues sortedOrds = new NumericDocValues() { + @Override + public long longValue() throws IOException { + return sorted.ordValue(); + } + + @Override + public boolean advanceExact(int target) throws IOException { + return sorted.advanceExact(target); + } + + @Override + public int docID() { + return sorted.docID(); + } + + @Override + public int nextDoc() throws IOException { + return sorted.nextDoc(); + } + + @Override + public int advance(int target) throws IOException { + return sorted.advance(target); + } + + @Override + public long cost() { + return sorted.cost(); + } + }; + return DocValues.singleton(sortedOrds); + } + }; + if (field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) { + writeSkipIndex(field, producer); + } + if (addTypeByte) { + meta.writeByte((byte) 0); // multiValued (0 = singleValued) + } + SortedDocValues sorted = valuesProducer.getSorted(field); + int maxOrd = sorted.getValueCount(); + writeField(field, producer, maxOrd, null); + addTermsDict(DocValues.singleton(valuesProducer.getSorted(field))); + } + + private void addTermsDict(SortedSetDocValues values) throws IOException { + final long size = values.getValueCount(); + meta.writeVLong(size); + + int blockMask = ES819TSDBDocValuesFormat.TERMS_DICT_BLOCK_LZ4_MASK; + int shift = ES819TSDBDocValuesFormat.TERMS_DICT_BLOCK_LZ4_SHIFT; + + meta.writeInt(DIRECT_MONOTONIC_BLOCK_SHIFT); + ByteBuffersDataOutput addressBuffer = new ByteBuffersDataOutput(); + ByteBuffersIndexOutput addressOutput = new ByteBuffersIndexOutput(addressBuffer, "temp", "temp"); + long numBlocks = (size + blockMask) >>> shift; + DirectMonotonicWriter writer = DirectMonotonicWriter.getInstance(meta, addressOutput, numBlocks, DIRECT_MONOTONIC_BLOCK_SHIFT); + + BytesRefBuilder previous = new BytesRefBuilder(); + long ord = 0; + long start = data.getFilePointer(); + int maxLength = 0, maxBlockLength = 0; + TermsEnum iterator = values.termsEnum(); + + LZ4.FastCompressionHashTable ht = new LZ4.FastCompressionHashTable(); + ByteArrayDataOutput bufferedOutput = new ByteArrayDataOutput(termsDictBuffer); + int dictLength = 0; + + for (BytesRef term = iterator.next(); term != null; term = iterator.next()) { + if ((ord & blockMask) == 0) { + if (ord != 0) { + // flush the previous block + final int uncompressedLength = compressAndGetTermsDictBlockLength(bufferedOutput, dictLength, ht); + maxBlockLength = Math.max(maxBlockLength, uncompressedLength); + bufferedOutput.reset(termsDictBuffer); + } + + writer.add(data.getFilePointer() - start); + // Write the first term both to the index output, and to the buffer where we'll use it as a + // dictionary for compression + data.writeVInt(term.length); + data.writeBytes(term.bytes, term.offset, term.length); + bufferedOutput = maybeGrowBuffer(bufferedOutput, term.length); + bufferedOutput.writeBytes(term.bytes, term.offset, term.length); + dictLength = term.length; + } else { + final int prefixLength = StringHelper.bytesDifference(previous.get(), term); + final int suffixLength = term.length - prefixLength; + assert suffixLength > 0; // terms are unique + // Will write (suffixLength + 1 byte + 2 vint) bytes. Grow the buffer in need. + bufferedOutput = maybeGrowBuffer(bufferedOutput, suffixLength + 11); + bufferedOutput.writeByte((byte) (Math.min(prefixLength, 15) | (Math.min(15, suffixLength - 1) << 4))); + if (prefixLength >= 15) { + bufferedOutput.writeVInt(prefixLength - 15); + } + if (suffixLength >= 16) { + bufferedOutput.writeVInt(suffixLength - 16); + } + bufferedOutput.writeBytes(term.bytes, term.offset + prefixLength, suffixLength); + } + maxLength = Math.max(maxLength, term.length); + previous.copyBytes(term); + ++ord; + } + // Compress and write out the last block + if (bufferedOutput.getPosition() > dictLength) { + final int uncompressedLength = compressAndGetTermsDictBlockLength(bufferedOutput, dictLength, ht); + maxBlockLength = Math.max(maxBlockLength, uncompressedLength); + } + + writer.finish(); + meta.writeInt(maxLength); + // Write one more int for storing max block length. + meta.writeInt(maxBlockLength); + meta.writeLong(start); + meta.writeLong(data.getFilePointer() - start); + start = data.getFilePointer(); + addressBuffer.copyTo(data); + meta.writeLong(start); + meta.writeLong(data.getFilePointer() - start); + + // Now write the reverse terms index + writeTermsIndex(values); + } + + private int compressAndGetTermsDictBlockLength(ByteArrayDataOutput bufferedOutput, int dictLength, LZ4.FastCompressionHashTable ht) + throws IOException { + int uncompressedLength = bufferedOutput.getPosition() - dictLength; + data.writeVInt(uncompressedLength); + LZ4.compressWithDictionary(termsDictBuffer, 0, dictLength, uncompressedLength, data, ht); + return uncompressedLength; + } + + private ByteArrayDataOutput maybeGrowBuffer(ByteArrayDataOutput bufferedOutput, int termLength) { + int pos = bufferedOutput.getPosition(), originalLength = termsDictBuffer.length; + if (pos + termLength >= originalLength - 1) { + termsDictBuffer = ArrayUtil.grow(termsDictBuffer, originalLength + termLength); + bufferedOutput = new ByteArrayDataOutput(termsDictBuffer, pos, termsDictBuffer.length - pos); + } + return bufferedOutput; + } + + private void writeTermsIndex(SortedSetDocValues values) throws IOException { + final long size = values.getValueCount(); + meta.writeInt(ES819TSDBDocValuesFormat.TERMS_DICT_REVERSE_INDEX_SHIFT); + long start = data.getFilePointer(); + + long numBlocks = 1L + ((size + ES819TSDBDocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) + >>> ES819TSDBDocValuesFormat.TERMS_DICT_REVERSE_INDEX_SHIFT); + ByteBuffersDataOutput addressBuffer = new ByteBuffersDataOutput(); + DirectMonotonicWriter writer; + try (ByteBuffersIndexOutput addressOutput = new ByteBuffersIndexOutput(addressBuffer, "temp", "temp")) { + writer = DirectMonotonicWriter.getInstance(meta, addressOutput, numBlocks, DIRECT_MONOTONIC_BLOCK_SHIFT); + TermsEnum iterator = values.termsEnum(); + BytesRefBuilder previous = new BytesRefBuilder(); + long offset = 0; + long ord = 0; + for (BytesRef term = iterator.next(); term != null; term = iterator.next()) { + if ((ord & ES819TSDBDocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) == 0) { + writer.add(offset); + final int sortKeyLength; + if (ord == 0) { + // no previous term: no bytes to write + sortKeyLength = 0; + } else { + sortKeyLength = StringHelper.sortKeyLength(previous.get(), term); + } + offset += sortKeyLength; + data.writeBytes(term.bytes, term.offset, sortKeyLength); + } else if ((ord + & ES819TSDBDocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) == ES819TSDBDocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) { + previous.copyBytes(term); + } + ++ord; + } + writer.add(offset); + writer.finish(); + meta.writeLong(start); + meta.writeLong(data.getFilePointer() - start); + start = data.getFilePointer(); + addressBuffer.copyTo(data); + meta.writeLong(start); + meta.writeLong(data.getFilePointer() - start); + } + } + + @Override + public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + meta.writeInt(field.number); + meta.writeByte(ES819TSDBDocValuesFormat.SORTED_NUMERIC); + writeSortedNumericField(field, new TsdbDocValuesProducer(valuesProducer), -1); + } + + private void writeSortedNumericField(FieldInfo field, TsdbDocValuesProducer valuesProducer, long maxOrd) throws IOException { + if (field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) { + writeSkipIndex(field, valuesProducer); + } + if (maxOrd > -1) { + meta.writeByte((byte) 1); // multiValued (1 = multiValued) + } + + if (valuesProducer.mergeStats.supported()) { + int numDocsWithField = valuesProducer.mergeStats.sumNumDocsWithField(); + long numValues = valuesProducer.mergeStats.sumNumValues(); + if (numDocsWithField == numValues) { + writeField(field, valuesProducer, maxOrd, null); + } else { + assert numValues > numDocsWithField; + try (var accumulator = new OffsetsAccumulator(dir, context, data, numDocsWithField)) { + writeField(field, valuesProducer, maxOrd, accumulator); + accumulator.build(meta, data); + } + } + } else { + long[] stats = writeField(field, valuesProducer, maxOrd, null); + int numDocsWithField = Math.toIntExact(stats[0]); + long numValues = stats[1]; + assert numValues >= numDocsWithField; + + if (numValues > numDocsWithField) { + long start = data.getFilePointer(); + meta.writeLong(start); + meta.writeVInt(ES819TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT); + + final DirectMonotonicWriter addressesWriter = DirectMonotonicWriter.getInstance( + meta, + data, + numDocsWithField + 1L, + ES819TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT + ); + long addr = 0; + addressesWriter.add(addr); + SortedNumericDocValues values = valuesProducer.getSortedNumeric(field); + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + addr += values.docValueCount(); + addressesWriter.add(addr); + } + addressesWriter.finish(); + meta.writeLong(data.getFilePointer() - start); + } + } + } + + @Override + public void mergeSortedNumericField(FieldInfo mergeFieldInfo, MergeState mergeState) throws IOException { + var result = compatibleWithOptimizedMerge(enableOptimizedMerge, mergeState, mergeFieldInfo); + if (result.supported()) { + mergeSortedNumericField(result, mergeFieldInfo, mergeState); + } else { + super.mergeSortedNumericField(mergeFieldInfo, mergeState); + } + } + + private static boolean isSingleValued(FieldInfo field, TsdbDocValuesProducer producer) throws IOException { + if (producer.mergeStats.supported()) { + return producer.mergeStats.sumNumValues() == producer.mergeStats.sumNumDocsWithField(); + } + + var values = producer.getSortedSet(field); + if (DocValues.unwrapSingleton(values) != null) { + return true; + } + + assert values.docID() == -1; + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + int docValueCount = values.docValueCount(); + assert docValueCount > 0; + if (docValueCount > 1) { + return false; + } + } + return true; + } + + @Override + public void mergeSortedSetField(FieldInfo mergeFieldInfo, MergeState mergeState) throws IOException { + var result = compatibleWithOptimizedMerge(enableOptimizedMerge, mergeState, mergeFieldInfo); + if (result.supported()) { + mergeSortedSetField(result, mergeFieldInfo, mergeState); + } else { + super.mergeSortedSetField(mergeFieldInfo, mergeState); + } + } + + @Override + public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + meta.writeInt(field.number); + meta.writeByte(SORTED_SET); + + if (isSingleValued(field, new TsdbDocValuesProducer(valuesProducer))) { + doAddSortedField(field, new TsdbDocValuesProducer(valuesProducer) { + @Override + public SortedDocValues getSorted(FieldInfo field) throws IOException { + return SortedSetSelector.wrap(valuesProducer.getSortedSet(field), SortedSetSelector.Type.MIN); + } + }, true); + return; + } + + SortedSetDocValues values = valuesProducer.getSortedSet(field); + long maxOrd = values.getValueCount(); + writeSortedNumericField(field, new TsdbDocValuesProducer(valuesProducer) { + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { + SortedSetDocValues values = valuesProducer.getSortedSet(field); + return new SortedNumericDocValues() { + + long[] ords = LongsRef.EMPTY_LONGS; + int i, docValueCount; + + @Override + public long nextValue() { + return ords[i++]; + } + + @Override + public int docValueCount() { + return docValueCount; + } + + @Override + public boolean advanceExact(int target) { + throw new UnsupportedOperationException(); + } + + @Override + public int docID() { + return values.docID(); + } + + @Override + public int nextDoc() throws IOException { + int doc = values.nextDoc(); + if (doc != NO_MORE_DOCS) { + docValueCount = values.docValueCount(); + ords = ArrayUtil.grow(ords, docValueCount); + for (int j = 0; j < docValueCount; j++) { + ords[j] = values.nextOrd(); + } + i = 0; + } + return doc; + } + + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + return values.cost(); + } + }; + } + }, maxOrd); + + addTermsDict(valuesProducer.getSortedSet(field)); + } + + @Override + public void close() throws IOException { + boolean success = false; + try { + if (meta != null) { + meta.writeInt(-1); // write EOF marker + CodecUtil.writeFooter(meta); // write checksum + } + if (data != null) { + CodecUtil.writeFooter(data); // write checksum + } + success = true; + } finally { + if (success) { + IOUtils.close(data, meta); + } else { + IOUtils.closeWhileHandlingException(data, meta); + } + meta = data = null; + } + } + + private static class SkipAccumulator { + int minDocID; + int maxDocID; + int docCount; + long minValue; + long maxValue; + + SkipAccumulator(int docID) { + minDocID = docID; + minValue = Long.MAX_VALUE; + maxValue = Long.MIN_VALUE; + docCount = 0; + } + + boolean isDone(int skipIndexIntervalSize, int valueCount, long nextValue, int nextDoc) { + if (docCount < skipIndexIntervalSize) { + return false; + } + // Once we reach the interval size, we will keep accepting documents if + // - next doc value is not a multi-value + // - current accumulator only contains a single value and next value is the same value + // - the accumulator is dense and the next doc keeps the density (no gaps) + return valueCount > 1 || minValue != maxValue || minValue != nextValue || docCount != nextDoc - minDocID; + } + + void accumulate(long value) { + minValue = Math.min(minValue, value); + maxValue = Math.max(maxValue, value); + } + + void accumulate(SkipAccumulator other) { + assert minDocID <= other.minDocID && maxDocID < other.maxDocID; + maxDocID = other.maxDocID; + minValue = Math.min(minValue, other.minValue); + maxValue = Math.max(maxValue, other.maxValue); + docCount += other.docCount; + } + + void nextDoc(int docID) { + maxDocID = docID; + ++docCount; + } + + public static SkipAccumulator merge(List list, int index, int length) { + SkipAccumulator acc = new SkipAccumulator(list.get(index).minDocID); + for (int i = 0; i < length; i++) { + acc.accumulate(list.get(index + i)); + } + return acc; + } + } + + private void writeSkipIndex(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + assert field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE; + final long start = data.getFilePointer(); + final SortedNumericDocValues values = valuesProducer.getSortedNumeric(field); + long globalMaxValue = Long.MIN_VALUE; + long globalMinValue = Long.MAX_VALUE; + int globalDocCount = 0; + int maxDocId = -1; + final List accumulators = new ArrayList<>(); + SkipAccumulator accumulator = null; + final int maxAccumulators = 1 << (SKIP_INDEX_LEVEL_SHIFT * (SKIP_INDEX_MAX_LEVEL - 1)); + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + final long firstValue = values.nextValue(); + if (accumulator != null && accumulator.isDone(skipIndexIntervalSize, values.docValueCount(), firstValue, doc)) { + globalMaxValue = Math.max(globalMaxValue, accumulator.maxValue); + globalMinValue = Math.min(globalMinValue, accumulator.minValue); + globalDocCount += accumulator.docCount; + maxDocId = accumulator.maxDocID; + accumulator = null; + if (accumulators.size() == maxAccumulators) { + writeLevels(accumulators); + accumulators.clear(); + } + } + if (accumulator == null) { + accumulator = new SkipAccumulator(doc); + accumulators.add(accumulator); + } + accumulator.nextDoc(doc); + accumulator.accumulate(firstValue); + for (int i = 1, end = values.docValueCount(); i < end; ++i) { + accumulator.accumulate(values.nextValue()); + } + } + + if (accumulators.isEmpty() == false) { + globalMaxValue = Math.max(globalMaxValue, accumulator.maxValue); + globalMinValue = Math.min(globalMinValue, accumulator.minValue); + globalDocCount += accumulator.docCount; + maxDocId = accumulator.maxDocID; + writeLevels(accumulators); + } + meta.writeLong(start); // record the start in meta + meta.writeLong(data.getFilePointer() - start); // record the length + assert globalDocCount == 0 || globalMaxValue >= globalMinValue; + meta.writeLong(globalMaxValue); + meta.writeLong(globalMinValue); + assert globalDocCount <= maxDocId + 1; + meta.writeInt(globalDocCount); + meta.writeInt(maxDocId); + } + + private void writeLevels(List accumulators) throws IOException { + final List> accumulatorsLevels = new ArrayList<>(SKIP_INDEX_MAX_LEVEL); + accumulatorsLevels.add(accumulators); + for (int i = 0; i < SKIP_INDEX_MAX_LEVEL - 1; i++) { + accumulatorsLevels.add(buildLevel(accumulatorsLevels.get(i))); + } + int totalAccumulators = accumulators.size(); + for (int index = 0; index < totalAccumulators; index++) { + // compute how many levels we need to write for the current accumulator + final int levels = getLevels(index, totalAccumulators); + // write the number of levels + data.writeByte((byte) levels); + // write intervals in reverse order. This is done so we don't + // need to read all of them in case of slipping + for (int level = levels - 1; level >= 0; level--) { + final SkipAccumulator accumulator = accumulatorsLevels.get(level).get(index >> (SKIP_INDEX_LEVEL_SHIFT * level)); + data.writeInt(accumulator.maxDocID); + data.writeInt(accumulator.minDocID); + data.writeLong(accumulator.maxValue); + data.writeLong(accumulator.minValue); + data.writeInt(accumulator.docCount); + } + } + } + + private static List buildLevel(List accumulators) { + final int levelSize = 1 << SKIP_INDEX_LEVEL_SHIFT; + final List collector = new ArrayList<>(); + for (int i = 0; i < accumulators.size() - levelSize + 1; i += levelSize) { + collector.add(SkipAccumulator.merge(accumulators, i, levelSize)); + } + return collector; + } + + private static int getLevels(int index, int size) { + if (Integer.numberOfTrailingZeros(index) >= SKIP_INDEX_LEVEL_SHIFT) { + // TODO: can we do it in constant time rather than linearly with SKIP_INDEX_MAX_LEVEL? + final int left = size - index; + for (int level = SKIP_INDEX_MAX_LEVEL - 1; level > 0; level--) { + final int numberIntervals = 1 << (SKIP_INDEX_LEVEL_SHIFT * level); + if (left >= numberIntervals && index % numberIntervals == 0) { + return level + 1; + } + } + } + return 1; + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormatTests.java index 1838c2c9e0ecb..ea1042cfa72df 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormatTests.java @@ -10,6 +10,7 @@ package org.elasticsearch.index.codec.tsdb.es819; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; import org.apache.lucene.document.BinaryDocValuesField; @@ -27,6 +28,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.LogByteSizeMergePolicy; import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; @@ -83,6 +85,27 @@ public DocValuesFormat getDocValuesFormatForField(String field) { } }; + public static class TestES819TSDBDocValuesFormatVersion0 extends ES819TSDBDocValuesFormat { + + public TestES819TSDBDocValuesFormatVersion0() { + super(); + } + + @Override + public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + return new ES819TSDBDocValuesConsumerVersion0( + state, + skipIndexIntervalSize, + minDocsPerOrdinalForRangeEncoding, + enableOptimizedMerge, + DATA_CODEC, + DATA_EXTENSION, + META_CODEC, + META_EXTENSION + ); + } + } + @Override protected Codec getCodec() { return codec; diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/diskbbq/ES920DiskBBQBFloat16VectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/diskbbq/ES920DiskBBQBFloat16VectorsFormatTests.java new file mode 100644 index 0000000000000..509d96b5159e1 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/diskbbq/ES920DiskBBQBFloat16VectorsFormatTests.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.diskbbq; + +import com.carrotsearch.randomizedtesting.generators.RandomPicks; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.apache.lucene.index.CodecReader; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.tests.util.TestUtil; +import org.elasticsearch.common.logging.LogConfigurator; +import org.elasticsearch.index.codec.vectors.BaseBFloat16KnnVectorsFormatTestCase; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.junit.AssumptionViolatedException; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat.MIN_CENTROIDS_PER_PARENT_CLUSTER; +import static org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat.MIN_VECTORS_PER_CLUSTER; +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.equalTo; + +public class ES920DiskBBQBFloat16VectorsFormatTests extends BaseBFloat16KnnVectorsFormatTestCase { + + static { + LogConfigurator.loadLog4jPlugins(); + LogConfigurator.configureESLogging(); // native access requires logging to be initialized + } + + private KnnVectorsFormat format; + + @Before + @Override + public void setUp() throws Exception { + if (rarely()) { + format = new ES920DiskBBQVectorsFormat( + random().nextInt(2 * MIN_VECTORS_PER_CLUSTER, ES920DiskBBQVectorsFormat.MAX_VECTORS_PER_CLUSTER), + random().nextInt(8, ES920DiskBBQVectorsFormat.MAX_CENTROIDS_PER_PARENT_CLUSTER), + DenseVectorFieldMapper.ElementType.FLOAT, + random().nextBoolean() + ); + } else { + // run with low numbers to force many clusters with parents + format = new ES920DiskBBQVectorsFormat( + random().nextInt(MIN_VECTORS_PER_CLUSTER, 2 * MIN_VECTORS_PER_CLUSTER), + random().nextInt(MIN_CENTROIDS_PER_PARENT_CLUSTER, 8), + DenseVectorFieldMapper.ElementType.FLOAT, + random().nextBoolean() + ); + } + super.setUp(); + } + + @Override + protected Codec getCodec() { + return TestUtil.alwaysKnnVectorsFormat(format); + } + + @Override + protected VectorSimilarityFunction randomSimilarity() { + return RandomPicks.randomFrom( + random(), + List.of( + VectorSimilarityFunction.DOT_PRODUCT, + VectorSimilarityFunction.EUCLIDEAN, + VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT + ) + ); + } + + @Override + public void testSearchWithVisitedLimit() { + throw new AssumptionViolatedException("ivf doesn't enforce visitation limit"); + } + + @Override + public void testAdvance() throws Exception { + // TODO re-enable with hierarchical IVF, clustering as it is is flaky + } + + @Override + protected void assertOffHeapByteSize(LeafReader r, String fieldName) throws IOException { + var fieldInfo = r.getFieldInfos().fieldInfo(fieldName); + + if (r instanceof CodecReader codecReader) { + KnnVectorsReader knnVectorsReader = codecReader.getVectorReader(); + if (knnVectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader fieldsReader) { + knnVectorsReader = fieldsReader.getFieldReader(fieldName); + } + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); + long totalByteSize = offHeap.values().stream().mapToLong(Long::longValue).sum(); + // IVF doesn't report stats at the moment + assertThat(offHeap, anEmptyMap()); + assertThat(totalByteSize, equalTo(0L)); + } else { + throw new AssertionError("unexpected:" + r.getClass()); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/diskbbq/ES920DiskBBQVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/diskbbq/ES920DiskBBQVectorsFormatTests.java index 29e6c59d995be..9b452fe4fb9cb 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/diskbbq/ES920DiskBBQVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/diskbbq/ES920DiskBBQVectorsFormatTests.java @@ -11,7 +11,6 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.KnnVectorsWriter; @@ -36,11 +35,12 @@ import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; import org.apache.lucene.tests.util.TestUtil; import org.elasticsearch.common.logging.LogConfigurator; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.junit.AssumptionViolatedException; import org.junit.Before; import java.io.IOException; import java.util.List; -import java.util.Locale; import java.util.concurrent.atomic.AtomicBoolean; import static java.lang.String.format; @@ -51,8 +51,7 @@ import static org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat.MIN_VECTORS_PER_CLUSTER; import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.oneOf; +import static org.hamcrest.Matchers.hasToString; public class ES920DiskBBQVectorsFormatTests extends BaseKnnVectorsFormatTestCase { @@ -70,6 +69,7 @@ public void setUp() throws Exception { format = new ES920DiskBBQVectorsFormat( random().nextInt(2 * MIN_VECTORS_PER_CLUSTER, ES920DiskBBQVectorsFormat.MAX_VECTORS_PER_CLUSTER), random().nextInt(8, ES920DiskBBQVectorsFormat.MAX_CENTROIDS_PER_PARENT_CLUSTER), + DenseVectorFieldMapper.ElementType.FLOAT, random().nextBoolean() ); } else { @@ -77,6 +77,7 @@ public void setUp() throws Exception { format = new ES920DiskBBQVectorsFormat( random().nextInt(MIN_VECTORS_PER_CLUSTER, 2 * MIN_VECTORS_PER_CLUSTER), random().nextInt(MIN_CENTROIDS_PER_PARENT_CLUSTER, 8), + DenseVectorFieldMapper.ElementType.FLOAT, random().nextBoolean() ); } @@ -102,7 +103,7 @@ protected VectorEncoding randomVectorEncoding() { @Override public void testSearchWithVisitedLimit() { - // ivf doesn't enforce visitation limit + throw new AssumptionViolatedException("ivf doesn't enforce visitation limit"); } @Override @@ -135,17 +136,9 @@ public void testAdvance() throws Exception { } public void testToString() { - FilterCodec customCodec = new FilterCodec("foo", Codec.getDefault()) { - @Override - public KnnVectorsFormat knnVectorsFormat() { - return new ES920DiskBBQVectorsFormat(128, 4); - } - }; - String expectedPattern = "ES920DiskBBQVectorsFormat(vectorPerCluster=128)"; + KnnVectorsFormat format = new ES920DiskBBQVectorsFormat(128, 4); - var defaultScorer = format(Locale.ROOT, expectedPattern, "DefaultFlatVectorScorer"); - var memSegScorer = format(Locale.ROOT, expectedPattern, "Lucene99MemorySegmentFlatVectorsScorer"); - assertThat(customCodec.knnVectorsFormat().toString(), is(oneOf(defaultScorer, memSegScorer))); + assertThat(format, hasToString("ES920DiskBBQVectorsFormat(vectorPerCluster=128)")); } public void testLimits() { diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93BinaryQuantizedBFloat16VectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93BinaryQuantizedBFloat16VectorsFormatTests.java index 6cb1f7e61af5f..fb287e39b37d1 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93BinaryQuantizedBFloat16VectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93BinaryQuantizedBFloat16VectorsFormatTests.java @@ -46,6 +46,7 @@ import org.elasticsearch.common.logging.LogConfigurator; import org.elasticsearch.index.codec.vectors.BFloat16; import org.elasticsearch.index.codec.vectors.BaseBFloat16KnnVectorsFormatTestCase; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.junit.AssumptionViolatedException; import java.io.IOException; @@ -75,7 +76,7 @@ public class ES93BinaryQuantizedBFloat16VectorsFormatTests extends BaseBFloat16K @Override public void setUp() throws Exception { - format = new ES93BinaryQuantizedVectorsFormat(ES93GenericFlatVectorsFormat.ElementType.BFLOAT16, random().nextBoolean()); + format = new ES93BinaryQuantizedVectorsFormat(DenseVectorFieldMapper.ElementType.BFLOAT16, random().nextBoolean()); super.setUp(); } @@ -196,7 +197,7 @@ public void testToString() { var defaultScorer = expected.replaceAll("\\{}", "DefaultFlatVectorScorer"); var memSegScorer = expected.replaceAll("\\{}", "Lucene99MemorySegmentFlatVectorsScorer"); - KnnVectorsFormat format = new ES93BinaryQuantizedVectorsFormat(ES93GenericFlatVectorsFormat.ElementType.BFLOAT16, false); + KnnVectorsFormat format = new ES93BinaryQuantizedVectorsFormat(DenseVectorFieldMapper.ElementType.BFLOAT16, false); assertThat(format, hasToString(oneOf(defaultScorer, memSegScorer))); } diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93BinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93BinaryQuantizedVectorsFormatTests.java index b9d53944ac654..39439bac02a8c 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93BinaryQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93BinaryQuantizedVectorsFormatTests.java @@ -55,6 +55,7 @@ import org.apache.lucene.tests.store.MockDirectoryWrapper; import org.apache.lucene.tests.util.TestUtil; import org.elasticsearch.common.logging.LogConfigurator; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.junit.AssumptionViolatedException; import java.io.IOException; @@ -84,7 +85,7 @@ public class ES93BinaryQuantizedVectorsFormatTests extends BaseKnnVectorsFormatT @Override public void setUp() throws Exception { - format = new ES93BinaryQuantizedVectorsFormat(ES93GenericFlatVectorsFormat.ElementType.STANDARD, random().nextBoolean()); + format = new ES93BinaryQuantizedVectorsFormat(DenseVectorFieldMapper.ElementType.FLOAT, random().nextBoolean()); super.setUp(); } @@ -201,7 +202,7 @@ public void testToString() { var defaultScorer = expected.replaceAll("\\{}", "DefaultFlatVectorScorer"); var memSegScorer = expected.replaceAll("\\{}", "Lucene99MemorySegmentFlatVectorsScorer"); - KnnVectorsFormat format = new ES93BinaryQuantizedVectorsFormat(ES93GenericFlatVectorsFormat.ElementType.STANDARD, false); + KnnVectorsFormat format = new ES93BinaryQuantizedVectorsFormat(DenseVectorFieldMapper.ElementType.FLOAT, false); assertThat(format, hasToString(oneOf(defaultScorer, memSegScorer))); } diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatBFloat16VectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatBFloat16VectorFormatTests.java new file mode 100644 index 0000000000000..66ee2c038d8e0 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatBFloat16VectorFormatTests.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.CodecReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.util.TestUtil; +import org.elasticsearch.common.logging.LogConfigurator; +import org.elasticsearch.index.codec.vectors.BFloat16; +import org.elasticsearch.index.codec.vectors.BaseBFloat16KnnVectorsFormatTestCase; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.junit.AssumptionViolatedException; + +import java.io.IOException; + +import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.hasEntry; + +public class ES93FlatBFloat16VectorFormatTests extends BaseBFloat16KnnVectorsFormatTestCase { + + static { + LogConfigurator.loadLog4jPlugins(); + LogConfigurator.configureESLogging(); // native access requires logging to be initialized + } + + @Override + protected Codec getCodec() { + return TestUtil.alwaysKnnVectorsFormat(new ES93FlatVectorFormat(DenseVectorFieldMapper.ElementType.BFLOAT16)); + } + + public void testSearchWithVisitedLimit() { + throw new AssumptionViolatedException("requires graph-based vector codec"); + } + + public void testSimpleOffHeapSize() throws IOException { + float[] vector = randomVector(random().nextInt(12, 500)); + try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + Document doc = new Document(); + doc.add(new KnnFloatVectorField("f", vector, DOT_PRODUCT)); + w.addDocument(doc); + w.commit(); + try (IndexReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + if (r instanceof CodecReader codecReader) { + KnnVectorsReader knnVectorsReader = codecReader.getVectorReader(); + if (knnVectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader fieldsReader) { + knnVectorsReader = fieldsReader.getFieldReader("f"); + } + var fieldInfo = r.getFieldInfos().fieldInfo("f"); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); + assertThat(offHeap, aMapWithSize(1)); + assertThat(offHeap, hasEntry("vec", (long) vector.length * BFloat16.BYTES)); + } + } + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormatTests.java new file mode 100644 index 0000000000000..7cc4ddb949832 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormatTests.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.CodecReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; +import org.apache.lucene.tests.util.TestUtil; +import org.elasticsearch.common.logging.LogConfigurator; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.junit.AssumptionViolatedException; + +import java.io.IOException; + +import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.hasEntry; + +public class ES93FlatVectorFormatTests extends BaseKnnVectorsFormatTestCase { + + static { + LogConfigurator.loadLog4jPlugins(); + LogConfigurator.configureESLogging(); // native access requires logging to be initialized + } + + @Override + protected Codec getCodec() { + return TestUtil.alwaysKnnVectorsFormat(new ES93FlatVectorFormat(DenseVectorFieldMapper.ElementType.FLOAT)); + } + + public void testSearchWithVisitedLimit() { + throw new AssumptionViolatedException("requires graph-based vector codec"); + } + + public void testSimpleOffHeapSize() throws IOException { + float[] vector = randomVector(random().nextInt(12, 500)); + try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + Document doc = new Document(); + doc.add(new KnnFloatVectorField("f", vector, DOT_PRODUCT)); + w.addDocument(doc); + w.commit(); + try (IndexReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + if (r instanceof CodecReader codecReader) { + KnnVectorsReader knnVectorsReader = codecReader.getVectorReader(); + if (knnVectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader fieldsReader) { + knnVectorsReader = fieldsReader.getFieldReader("f"); + } + var fieldInfo = r.getFieldInfos().fieldInfo("f"); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); + assertThat(offHeap, aMapWithSize(1)); + assertThat(offHeap, hasEntry("vec", (long) vector.length * Float.BYTES)); + } + } + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBFloat16VectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBFloat16VectorsFormatTests.java index e6c7ab2f256d4..0220ba831e75d 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBFloat16VectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBFloat16VectorsFormatTests.java @@ -13,6 +13,7 @@ import org.apache.lucene.store.Directory; import org.elasticsearch.index.codec.vectors.BFloat16; import org.elasticsearch.index.codec.vectors.BaseHnswBFloat16VectorsFormatTestCase; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import java.io.IOException; import java.util.Locale; @@ -30,24 +31,17 @@ public class ES93HnswBFloat16VectorsFormatTests extends BaseHnswBFloat16VectorsF @Override protected KnnVectorsFormat createFormat() { - return new ES93HnswVectorsFormat(ES93GenericFlatVectorsFormat.ElementType.BFLOAT16, random().nextBoolean()); + return new ES93HnswVectorsFormat(DenseVectorFieldMapper.ElementType.BFLOAT16); } @Override protected KnnVectorsFormat createFormat(int maxConn, int beamWidth) { - return new ES93HnswVectorsFormat(maxConn, beamWidth, ES93GenericFlatVectorsFormat.ElementType.BFLOAT16, random().nextBoolean()); + return new ES93HnswVectorsFormat(maxConn, beamWidth, DenseVectorFieldMapper.ElementType.BFLOAT16); } @Override protected KnnVectorsFormat createFormat(int maxConn, int beamWidth, int numMergeWorkers, ExecutorService service) { - return new ES93HnswVectorsFormat( - maxConn, - beamWidth, - ES93GenericFlatVectorsFormat.ElementType.BFLOAT16, - random().nextBoolean(), - numMergeWorkers, - service - ); + return new ES93HnswVectorsFormat(maxConn, beamWidth, DenseVectorFieldMapper.ElementType.BFLOAT16, numMergeWorkers, service); } public void testToString() { diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBinaryQuantizedBFloat16VectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBinaryQuantizedBFloat16VectorsFormatTests.java index 49dd1ba7c64e4..8d09eafb81ca2 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBinaryQuantizedBFloat16VectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBinaryQuantizedBFloat16VectorsFormatTests.java @@ -16,6 +16,7 @@ import org.apache.lucene.tests.store.MockDirectoryWrapper; import org.elasticsearch.index.codec.vectors.BFloat16; import org.elasticsearch.index.codec.vectors.BaseHnswBFloat16VectorsFormatTestCase; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import java.io.IOException; import java.util.Locale; @@ -34,7 +35,7 @@ public class ES93HnswBinaryQuantizedBFloat16VectorsFormatTests extends BaseHnswB @Override protected KnnVectorsFormat createFormat() { - return new ES93HnswBinaryQuantizedVectorsFormat(ES93GenericFlatVectorsFormat.ElementType.BFLOAT16, random().nextBoolean()); + return new ES93HnswBinaryQuantizedVectorsFormat(DenseVectorFieldMapper.ElementType.BFLOAT16, random().nextBoolean()); } @Override @@ -42,7 +43,7 @@ protected KnnVectorsFormat createFormat(int maxConn, int beamWidth) { return new ES93HnswBinaryQuantizedVectorsFormat( maxConn, beamWidth, - ES93GenericFlatVectorsFormat.ElementType.BFLOAT16, + DenseVectorFieldMapper.ElementType.BFLOAT16, random().nextBoolean() ); } @@ -52,7 +53,7 @@ protected KnnVectorsFormat createFormat(int maxConn, int beamWidth, int numMerge return new ES93HnswBinaryQuantizedVectorsFormat( maxConn, beamWidth, - ES93GenericFlatVectorsFormat.ElementType.BFLOAT16, + DenseVectorFieldMapper.ElementType.BFLOAT16, random().nextBoolean(), numMergeWorkers, service diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBinaryQuantizedVectorsFormatTests.java index 8afc974cfca7d..75235d894bb74 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBinaryQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBinaryQuantizedVectorsFormatTests.java @@ -25,6 +25,7 @@ import org.apache.lucene.store.MMapDirectory; import org.apache.lucene.tests.store.MockDirectoryWrapper; import org.elasticsearch.index.codec.vectors.BaseHnswVectorsFormatTestCase; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import java.io.IOException; import java.util.Locale; @@ -43,7 +44,7 @@ public class ES93HnswBinaryQuantizedVectorsFormatTests extends BaseHnswVectorsFo @Override protected KnnVectorsFormat createFormat() { - return new ES93HnswBinaryQuantizedVectorsFormat(ES93GenericFlatVectorsFormat.ElementType.STANDARD, random().nextBoolean()); + return new ES93HnswBinaryQuantizedVectorsFormat(DenseVectorFieldMapper.ElementType.FLOAT, random().nextBoolean()); } @Override @@ -51,7 +52,7 @@ protected KnnVectorsFormat createFormat(int maxConn, int beamWidth) { return new ES93HnswBinaryQuantizedVectorsFormat( maxConn, beamWidth, - ES93GenericFlatVectorsFormat.ElementType.STANDARD, + DenseVectorFieldMapper.ElementType.FLOAT, random().nextBoolean() ); } @@ -61,7 +62,7 @@ protected KnnVectorsFormat createFormat(int maxConn, int beamWidth, int numMerge return new ES93HnswBinaryQuantizedVectorsFormat( maxConn, beamWidth, - ES93GenericFlatVectorsFormat.ElementType.STANDARD, + DenseVectorFieldMapper.ElementType.FLOAT, random().nextBoolean(), numMergeWorkers, service diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBitVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBitVectorsFormatTests.java index f5f15bb1f06df..b54db35d77273 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBitVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswBitVectorsFormatTests.java @@ -23,6 +23,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.tests.util.TestUtil; import org.elasticsearch.index.codec.vectors.BaseKnnBitVectorsFormatTestCase; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.junit.Before; import java.io.IOException; @@ -36,9 +37,7 @@ public class ES93HnswBitVectorsFormatTests extends BaseKnnBitVectorsFormatTestCa @Override protected Codec getCodec() { - return TestUtil.alwaysKnnVectorsFormat( - new ES93HnswVectorsFormat(ES93GenericFlatVectorsFormat.ElementType.BIT, random().nextBoolean()) - ); + return TestUtil.alwaysKnnVectorsFormat(new ES93HnswVectorsFormat(DenseVectorFieldMapper.ElementType.BIT)); } @Before diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedBFloat16VectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedBFloat16VectorsFormatTests.java new file mode 100644 index 0000000000000..961edfaf3bf90 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedBFloat16VectorsFormatTests.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat; +import org.apache.lucene.store.Directory; +import org.elasticsearch.index.codec.vectors.BFloat16; +import org.elasticsearch.index.codec.vectors.BaseHnswBFloat16VectorsFormatTestCase; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; + +import java.io.IOException; +import java.util.concurrent.ExecutorService; + +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasEntry; + +public class ES93HnswScalarQuantizedBFloat16VectorsFormatTests extends BaseHnswBFloat16VectorsFormatTestCase { + + @Override + protected KnnVectorsFormat createFormat() { + return new ES93HnswScalarQuantizedVectorsFormat( + DEFAULT_MAX_CONN, + DEFAULT_BEAM_WIDTH, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, + DenseVectorFieldMapper.ElementType.BFLOAT16, + random().nextBoolean() + ); + } + + @Override + protected KnnVectorsFormat createFormat(int maxConn, int beamWidth) { + return new ES93HnswScalarQuantizedVectorsFormat( + maxConn, + beamWidth, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, + DenseVectorFieldMapper.ElementType.BFLOAT16, + random().nextBoolean() + ); + } + + @Override + protected KnnVectorsFormat createFormat(int maxConn, int beamWidth, int numMergeWorkers, ExecutorService service) { + return new ES93HnswScalarQuantizedVectorsFormat( + maxConn, + beamWidth, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, + DenseVectorFieldMapper.ElementType.BFLOAT16, + random().nextBoolean(), + numMergeWorkers, + service + ); + } + + public void testSimpleOffHeapSize() throws IOException { + float[] vector = randomVector(random().nextInt(12, 500)); + try (Directory dir = newDirectory()) { + testSimpleOffHeapSize( + dir, + newIndexWriterConfig(), + vector, + allOf( + aMapWithSize(3), + hasEntry("vec", (long) vector.length * BFloat16.BYTES), + hasEntry("vex", 1L), + hasEntry(equalTo("veq"), greaterThan(0L)) + ) + ); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormatTests.java new file mode 100644 index 0000000000000..242e53fa64c96 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormatTests.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat; +import org.apache.lucene.store.Directory; +import org.elasticsearch.index.codec.vectors.BaseHnswVectorsFormatTestCase; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; + +import java.io.IOException; +import java.util.concurrent.ExecutorService; + +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasEntry; + +public class ES93HnswScalarQuantizedVectorsFormatTests extends BaseHnswVectorsFormatTestCase { + + @Override + protected KnnVectorsFormat createFormat() { + return new ES93HnswScalarQuantizedVectorsFormat( + DEFAULT_MAX_CONN, + DEFAULT_BEAM_WIDTH, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, + DenseVectorFieldMapper.ElementType.FLOAT, + random().nextBoolean() + ); + } + + @Override + protected KnnVectorsFormat createFormat(int maxConn, int beamWidth) { + return new ES93HnswScalarQuantizedVectorsFormat( + maxConn, + beamWidth, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, + DenseVectorFieldMapper.ElementType.FLOAT, + random().nextBoolean() + ); + } + + @Override + protected KnnVectorsFormat createFormat(int maxConn, int beamWidth, int numMergeWorkers, ExecutorService service) { + return new ES93HnswScalarQuantizedVectorsFormat( + maxConn, + beamWidth, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, + DenseVectorFieldMapper.ElementType.FLOAT, + random().nextBoolean(), + numMergeWorkers, + service + ); + } + + public void testSimpleOffHeapSize() throws IOException { + float[] vector = randomVector(random().nextInt(12, 500)); + try (Directory dir = newDirectory()) { + testSimpleOffHeapSize( + dir, + newIndexWriterConfig(), + vector, + allOf( + aMapWithSize(3), + hasEntry("vec", (long) vector.length * Float.BYTES), + hasEntry("vex", 1L), + hasEntry(equalTo("veq"), greaterThan(0L)) + ) + ); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswVectorsFormatTests.java index 5fa507c23d756..84057c7709063 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswVectorsFormatTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.store.Directory; import org.elasticsearch.index.codec.vectors.BaseHnswVectorsFormatTestCase; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import java.io.IOException; import java.util.Locale; @@ -29,24 +30,17 @@ public class ES93HnswVectorsFormatTests extends BaseHnswVectorsFormatTestCase { @Override protected KnnVectorsFormat createFormat() { - return new ES93HnswVectorsFormat(ES93GenericFlatVectorsFormat.ElementType.STANDARD, random().nextBoolean()); + return new ES93HnswVectorsFormat(DenseVectorFieldMapper.ElementType.FLOAT); } @Override protected KnnVectorsFormat createFormat(int maxConn, int beamWidth) { - return new ES93HnswVectorsFormat(maxConn, beamWidth, ES93GenericFlatVectorsFormat.ElementType.STANDARD, random().nextBoolean()); + return new ES93HnswVectorsFormat(maxConn, beamWidth, DenseVectorFieldMapper.ElementType.FLOAT); } @Override protected KnnVectorsFormat createFormat(int maxConn, int beamWidth, int numMergeWorkers, ExecutorService service) { - return new ES93HnswVectorsFormat( - maxConn, - beamWidth, - ES93GenericFlatVectorsFormat.ElementType.STANDARD, - random().nextBoolean(), - numMergeWorkers, - service - ); + return new ES93HnswVectorsFormat(maxConn, beamWidth, DenseVectorFieldMapper.ElementType.FLOAT, numMergeWorkers, service); } public void testToString() { diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedBFloat16VectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedBFloat16VectorFormatTests.java new file mode 100644 index 0000000000000..8e4bc24ab6403 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedBFloat16VectorFormatTests.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.CodecReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.util.TestUtil; +import org.elasticsearch.common.logging.LogConfigurator; +import org.elasticsearch.index.codec.vectors.BFloat16; +import org.elasticsearch.index.codec.vectors.BaseBFloat16KnnVectorsFormatTestCase; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.junit.AssumptionViolatedException; + +import java.io.IOException; + +import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasEntry; + +public class ES93ScalarQuantizedBFloat16VectorFormatTests extends BaseBFloat16KnnVectorsFormatTestCase { + + static { + LogConfigurator.loadLog4jPlugins(); + LogConfigurator.configureESLogging(); // native access requires logging to be initialized + } + + private KnnVectorsFormat format; + + @Override + public void setUp() throws Exception { + format = new ES93ScalarQuantizedVectorsFormat(DenseVectorFieldMapper.ElementType.BFLOAT16); + super.setUp(); + } + + @Override + protected Codec getCodec() { + return TestUtil.alwaysKnnVectorsFormat(format); + } + + public void testSearchWithVisitedLimit() { + throw new AssumptionViolatedException("requires graph vector codec"); + } + + public void testSimpleOffHeapSize() throws IOException { + float[] vector = randomVector(random().nextInt(12, 500)); + try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + Document doc = new Document(); + doc.add(new KnnFloatVectorField("f", vector, DOT_PRODUCT)); + w.addDocument(doc); + w.commit(); + try (IndexReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + if (r instanceof CodecReader codecReader) { + KnnVectorsReader knnVectorsReader = codecReader.getVectorReader(); + if (knnVectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader fieldsReader) { + knnVectorsReader = fieldsReader.getFieldReader("f"); + } + var fieldInfo = r.getFieldInfos().fieldInfo("f"); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); + assertThat(offHeap, aMapWithSize(2)); + assertThat(offHeap, hasEntry("vec", (long) vector.length * BFloat16.BYTES)); + assertThat(offHeap, hasEntry(equalTo("veq"), greaterThan(0L))); + } + } + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormatTests.java new file mode 100644 index 0000000000000..5b6b381945c41 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormatTests.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.CodecReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; +import org.apache.lucene.tests.util.TestUtil; +import org.elasticsearch.common.logging.LogConfigurator; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; +import org.junit.AssumptionViolatedException; + +import java.io.IOException; + +import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasEntry; + +public class ES93ScalarQuantizedVectorsFormatTests extends BaseKnnVectorsFormatTestCase { + + static { + LogConfigurator.loadLog4jPlugins(); + LogConfigurator.configureESLogging(); // native access requires logging to be initialized + } + + @Override + protected Codec getCodec() { + return TestUtil.alwaysKnnVectorsFormat(new ES93ScalarQuantizedVectorsFormat(DenseVectorFieldMapper.ElementType.FLOAT)); + } + + public void testSearchWithVisitedLimit() { + throw new AssumptionViolatedException("requires graph vector codec"); + } + + public void testSimpleOffHeapSize() throws IOException { + float[] vector = randomVector(random().nextInt(12, 500)); + try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + Document doc = new Document(); + doc.add(new KnnFloatVectorField("f", vector, DOT_PRODUCT)); + w.addDocument(doc); + w.commit(); + try (IndexReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + if (r instanceof CodecReader codecReader) { + KnnVectorsReader knnVectorsReader = codecReader.getVectorReader(); + if (knnVectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader fieldsReader) { + knnVectorsReader = fieldsReader.getFieldReader("f"); + } + var fieldInfo = r.getFieldInfos().fieldInfo("f"); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); + assertThat(offHeap, aMapWithSize(2)); + assertThat(offHeap, hasEntry("vec", (long) vector.length * Float.BYTES)); + assertThat(offHeap, hasEntry(equalTo("veq"), greaterThan(0L))); + } + } + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java index 7f257172638c2..8c3800bdf6132 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java @@ -263,4 +263,9 @@ public int compareTo(BytesCompareUnsigned o) { protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 47161239a363d..76df9b6303109 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -409,4 +409,13 @@ public void execute() { } }; } + + @Override + protected List getSortShortcutSupport() { + return List.of( + // TODO: boolean field mapper uses a numeric comparator but is indexed with Terms + // so skipping doesn't work here. + new SortShortcutSupport(this::minimalMapping, this::writeField, false) + ); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index fa108a2bec780..7c4fee4744088 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -985,4 +985,9 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index 4d6368b712f98..f3e2633717a4b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -914,4 +914,24 @@ public void testSingletonLongBulkBlockReadingManyValues() throws Exception { } } } + + @Override + protected List getSortShortcutSupport() { + return List.of( + new SortShortcutSupport(b -> b.field("type", "date"), b -> b.field("field", "2025-10-30T00:00:00"), true), + new SortShortcutSupport(b -> b.field("type", "date_nanos"), b -> b.field("field", "2025-10-30T00:00:00"), true), + new SortShortcutSupport( + IndexVersion.fromId(5000099), + b -> b.field("type", "date"), + b -> b.field("field", "2025-10-30T00:00:00"), + false + ), + new SortShortcutSupport( + IndexVersion.fromId(5000099), + b -> b.field("type", "date_nanos"), + b -> b.field("field", "2025-10-30T00:00:00"), + false + ) + ); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java index 9ffe1ba394b0b..c560448f6bb33 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java @@ -719,4 +719,9 @@ public void testSyntheticSourceKeepArrays() { protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 2ca2a7f8eb62d..e0308b5fd8bbf 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -438,4 +438,13 @@ public void execute() { protected String randomSyntheticSourceKeep() { return "all"; } + + @Override + protected List getSortShortcutSupport() { + return List.of( + // TODO - shortcuts are disabled here, can we enable them? + new SortShortcutSupport(this::minimalMapping, this::writeField, false), + new SortShortcutSupport(IndexVersion.fromId(5000099), this::minimalMapping, this::writeField, false) + ); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 520ec8a8c3400..641ddeb170f77 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -1219,4 +1219,9 @@ protected String randomSyntheticSourceKeep() { // Only option all keeps array source in ignored source. return randomFrom("all"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(new SortShortcutSupport(this::minimalMapping, this::writeField, true)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index 7555d1cd7fb48..6dd85a3f8c5e6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -426,4 +426,9 @@ protected Object generateRandomInputValue(MappedFieldType ft) { assumeFalse("DocValuesFetcher doesn't work", true); return null; } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 970dd8886b93f..4947af5eeba11 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1776,4 +1776,8 @@ public void testNormsEnabledWhenIndexModeIsTsdb_bwcCheck() throws IOException { assertThat(fieldType.omitNorms(), is(false)); } + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index d9223a157b2ff..3939fcaa6e0e8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -1016,4 +1016,9 @@ public void assertStoredFieldsEquals(String info, IndexReader leftReader, IndexR assertFalse(info, rightIterator.hasNext()); } } + + @Override + protected List getSortShortcutSupport() { + return List.of(new SortShortcutSupport(this::minimalMapping, this::writeField, true)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTestUtils.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTestUtils.java index 9478508da88d0..c3e3ed0c93c98 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTestUtils.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTestUtils.java @@ -22,14 +22,14 @@ private DenseVectorFieldMapperTestUtils() {} public static List getSupportedSimilarities(DenseVectorFieldMapper.ElementType elementType) { return switch (elementType) { - case FLOAT, BYTE -> List.of(SimilarityMeasure.values()); + case FLOAT, BFLOAT16, BYTE -> List.of(SimilarityMeasure.values()); case BIT -> List.of(SimilarityMeasure.L2_NORM); }; } public static int getEmbeddingLength(DenseVectorFieldMapper.ElementType elementType, int dimensions) { return switch (elementType) { - case FLOAT, BYTE -> dimensions; + case FLOAT, BFLOAT16, BYTE -> dimensions; case BIT -> { assert dimensions % Byte.SIZE == 0; yield dimensions / Byte.SIZE; @@ -43,7 +43,7 @@ public static int randomCompatibleDimensions(DenseVectorFieldMapper.ElementType } return switch (elementType) { - case FLOAT, BYTE -> RandomNumbers.randomIntBetween(random(), 1, max); + case FLOAT, BFLOAT16, BYTE -> RandomNumbers.randomIntBetween(random(), 1, max); case BIT -> { if (max < 8) { throw new IllegalArgumentException("max must be at least 8 for bit vectors"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 58334f95b7160..753175465c340 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -27,11 +27,13 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Strings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.codec.LegacyPerFieldMapperCodec; import org.elasticsearch.index.codec.PerFieldMapperCodec; +import org.elasticsearch.index.codec.vectors.BFloat16; import org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentParsingException; @@ -74,6 +76,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.startsWith; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -86,11 +89,14 @@ public class DenseVectorFieldMapperTests extends SyntheticVectorsMapperTestCase private final int dims; public DenseVectorFieldMapperTests() { - this.elementType = randomFrom(ElementType.BYTE, ElementType.FLOAT, ElementType.BIT); + this.elementType = randomFrom(ElementType.BYTE, ElementType.FLOAT, ElementType.BFLOAT16, ElementType.BIT); this.indexed = usually(); this.indexOptionsSet = this.indexed && randomBoolean(); int baseDims = ElementType.BIT == elementType ? 4 * Byte.SIZE : 4; - int randomMultiplier = ElementType.FLOAT == elementType ? randomIntBetween(1, 64) : 1; + int randomMultiplier = switch (elementType) { + case FLOAT, BFLOAT16 -> randomIntBetween(1, 64); + case BYTE, BIT -> 1; + }; this.dims = baseDims * randomMultiplier; } @@ -150,9 +156,12 @@ private void indexMapping(XContentBuilder b, IndexVersion indexVersion) throws I @Override protected Object getSampleValueForDocument() { - return elementType == ElementType.FLOAT - ? convertToList(randomNormalizedVector(this.dims)) - : convertToList(randomByteArrayOfLength(elementType == ElementType.BIT ? this.dims / Byte.SIZE : dims)); + return switch (elementType) { + case FLOAT -> convertToList(randomNormalizedVector(this.dims)); + case BFLOAT16 -> convertToBFloat16List(randomNormalizedVector(this.dims)); + case BYTE -> convertToList(randomByteArrayOfLength(dims)); + case BIT -> convertToList(randomByteArrayOfLength(this.dims / Byte.SIZE)); + }; } public static List convertToList(float[] vector) { @@ -163,6 +172,14 @@ public static List convertToList(float[] vector) { return list; } + public static List convertToBFloat16List(float[] vector) { + List list = new ArrayList<>(vector.length); + for (float v : vector) { + list.add(BFloat16.truncateToBFloat16(v)); + } + return list; + } + public static List convertToList(byte[] vector) { List list = new ArrayList<>(vector.length); for (byte v : vector) { @@ -264,18 +281,16 @@ protected void registerParameters(ParameterChecker checker) throws IOException { registerConflict( checker, "element_type", - b -> b.field("type", "dense_vector").field("dims", dims).field("index", true).field("similarity", "l2_norm"), - "element_type", - "float", - "bit" + b -> b.field("type", "dense_vector").field("index", true).field("similarity", "l2_norm"), + b -> b.field("dims", dims).field("element_type", "float"), + b -> b.field("dims", dims * 8).field("element_type", "bit") ); registerConflict( checker, "element_type", - b -> b.field("type", "dense_vector").field("dims", dims).field("index", true).field("similarity", "l2_norm"), - "element_type", - "byte", - "bit" + b -> b.field("type", "dense_vector").field("index", true).field("similarity", "l2_norm"), + b -> b.field("dims", dims).field("element_type", "float"), + b -> b.field("dims", dims * 8).field("element_type", "bit") ); // update for flat @@ -1571,7 +1586,7 @@ protected Object generateRandomInputValue(MappedFieldType ft) { DenseVectorFieldType vectorFieldType = (DenseVectorFieldType) ft; return switch (vectorFieldType.getElementType()) { case BYTE -> randomByteArrayOfLength(vectorFieldType.getVectorDimensions()); - case FLOAT -> randomNormalizedVector(vectorFieldType.getVectorDimensions()); + case FLOAT, BFLOAT16 -> randomNormalizedVector(vectorFieldType.getVectorDimensions()); case BIT -> randomByteArrayOfLength(vectorFieldType.getVectorDimensions() / 8); }; } @@ -1885,19 +1900,16 @@ public void testKnnVectorsFormat() throws IOException { assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class)); knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); } - String expectedString = "Lucene99HnswVectorsFormat(name=Lucene99HnswVectorsFormat, maxConn=" + String expectedString = "ES93HnswVectorsFormat(name=ES93HnswVectorsFormat, maxConn=" + (setM ? m : DEFAULT_MAX_CONN) + ", beamWidth=" + (setEfConstruction ? efConstruction : DEFAULT_BEAM_WIDTH) - + ", tinySegmentsThreshold=0" - + ", flatVectorFormat=Lucene99FlatVectorsFormat(vectorsScorer=DefaultFlatVectorScorer())" - + ")"; + + ", flatVectorFormat=ES93GenericFlatVectorsFormat(name=ES93GenericFlatVectorsFormat" + + ", format=Lucene99FlatVectorsFormat(name=Lucene99FlatVectorsFormat, flatVectorScorer=DefaultFlatVectorScorer())))"; assertEquals(expectedString, knnVectorsFormat.toString()); } public void testKnnQuantizedFlatVectorsFormat() throws IOException { - boolean setConfidenceInterval = randomBoolean(); - float confidenceInterval = (float) randomDoubleBetween(0.90f, 1.0f, true); for (String quantizedFlatFormat : new String[] { "int8_flat", "int4_flat" }) { MapperService mapperService = createMapperService(fieldMapping(b -> { b.field("type", "dense_vector"); @@ -1906,9 +1918,6 @@ public void testKnnQuantizedFlatVectorsFormat() throws IOException { b.field("similarity", "dot_product"); b.startObject("index_options"); b.field("type", quantizedFlatFormat); - if (setConfidenceInterval) { - b.field("confidence_interval", confidenceInterval); - } b.endObject(); })); CodecService codecService = new CodecService(mapperService, BigArrays.NON_RECYCLING_INSTANCE); @@ -1925,29 +1934,25 @@ public void testKnnQuantizedFlatVectorsFormat() throws IOException { knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); } VectorScorerFactory factory = VectorScorerFactory.instance().orElse(null); - String expectedString = "ES813Int8FlatVectorFormat(name=ES813Int8FlatVectorFormat, innerFormat=" - + "ES814ScalarQuantizedVectorsFormat(name=ES814ScalarQuantizedVectorsFormat," - + " confidenceInterval=" - + (setConfidenceInterval ? Float.toString(confidenceInterval) : (quantizedFlatFormat.equals("int4_flat") ? "0.0" : null)) - + ", bits=" - + (quantizedFlatFormat.equals("int4_flat") ? 4 : 7) - + ", compressed=" - + quantizedFlatFormat.equals("int4_flat") - + ", flatVectorScorer=ESFlatVectorsScorer(" - + "delegate=ScalarQuantizedVectorScorer(nonQuantizedDelegate=DefaultFlatVectorScorer())" - + ", factory=" - + (factory != null ? factory : "null") - + "), " - + "rawVectorFormat=Lucene99FlatVectorsFormat(vectorsScorer=DefaultFlatVectorScorer())))"; - assertEquals(expectedString, knnVectorsFormat.toString()); + String expectedString = Strings.format( + "ES93ScalarQuantizedVectorsFormat(name=ES93ScalarQuantizedVectorsFormat" + + ", bits=%s" + + ", flatVectorScorer=Lucene104ScalarQuantizedVectorScorer(nonQuantizedDelegate=DefaultFlatVectorScorer())" + + ", rawVectorFormat=ES93GenericFlatVectorsFormat(name=ES93GenericFlatVectorsFormat" + + ", format=Lucene99FlatVectorsFormat(name=Lucene99FlatVectorsFormat, flatVectorScorer=DefaultFlatVectorScorer())))", + switch (quantizedFlatFormat) { + case "int8_flat" -> "7"; + case "int4_flat" -> "4"; + default -> throw new AssertionError(); + } + ); + assertThat(knnVectorsFormat, hasToString(expectedString)); } } public void testKnnQuantizedHNSWVectorsFormat() throws IOException { final int m = randomIntBetween(1, DEFAULT_MAX_CONN + 10); final int efConstruction = randomIntBetween(1, DEFAULT_BEAM_WIDTH + 10); - boolean setConfidenceInterval = randomBoolean(); - float confidenceInterval = (float) randomDoubleBetween(0.90f, 1.0f, true); MapperService mapperService = createMapperService(fieldMapping(b -> { b.field("type", "dense_vector"); b.field("dims", dims); @@ -1957,9 +1962,6 @@ public void testKnnQuantizedHNSWVectorsFormat() throws IOException { b.field("type", "int8_hnsw"); b.field("m", m); b.field("ef_construction", efConstruction); - if (setConfidenceInterval) { - b.field("confidence_interval", confidenceInterval); - } b.endObject(); })); CodecService codecService = new CodecService(mapperService, BigArrays.NON_RECYCLING_INSTANCE); @@ -1975,22 +1977,16 @@ public void testKnnQuantizedHNSWVectorsFormat() throws IOException { assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class)); knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); } - VectorScorerFactory factory = VectorScorerFactory.instance().orElse(null); - String expectedString = "ES814HnswScalarQuantizedVectorsFormat(name=ES814HnswScalarQuantizedVectorsFormat, maxConn=" - + m - + ", beamWidth=" - + efConstruction - + ", flatVectorFormat=ES814ScalarQuantizedVectorsFormat(" - + "name=ES814ScalarQuantizedVectorsFormat, confidenceInterval=" - + (setConfidenceInterval ? confidenceInterval : null) - + ", bits=7, compressed=false, " - + "flatVectorScorer=ESFlatVectorsScorer(delegate=ScalarQuantizedVectorScorer(nonQuantizedDelegate=DefaultFlatVectorScorer()), " - + "factory=" - + (factory != null ? factory : "null") - + "), " - + "rawVectorFormat=Lucene99FlatVectorsFormat(vectorsScorer=DefaultFlatVectorScorer())" - + "))"; - assertEquals(expectedString, knnVectorsFormat.toString()); + String expectedString = Strings.format( + "ES93HnswScalarQuantizedVectorsFormat(name=ES93HnswScalarQuantizedVectorsFormat" + + ", maxConn=%s, beamWidth=%s, bits=7" + + ", flatVectorScorer=Lucene104ScalarQuantizedVectorScorer(nonQuantizedDelegate=DefaultFlatVectorScorer())" + + ", rawVectorFormat=ES93GenericFlatVectorsFormat(name=ES93GenericFlatVectorsFormat" + + ", format=Lucene99FlatVectorsFormat(name=Lucene99FlatVectorsFormat, flatVectorScorer=DefaultFlatVectorScorer())))", + m, + efConstruction + ); + assertThat(knnVectorsFormat, hasToString(expectedString)); } public void testKnnBBQHNSWVectorsFormat() throws IOException { @@ -2021,14 +2017,15 @@ public void testKnnBBQHNSWVectorsFormat() throws IOException { assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class)); knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); } - String expectedString = "ES818HnswBinaryQuantizedVectorsFormat(name=ES818HnswBinaryQuantizedVectorsFormat, maxConn=" + String expectedString = "ES93HnswBinaryQuantizedVectorsFormat(name=ES93HnswBinaryQuantizedVectorsFormat, maxConn=" + m + ", beamWidth=" + efConstruction - + ", flatVectorFormat=ES818BinaryQuantizedVectorsFormat(" - + "name=ES818BinaryQuantizedVectorsFormat, " - + "flatVectorScorer=ES818BinaryFlatVectorsScorer(nonQuantizedDelegate=DefaultFlatVectorScorer())))"; - assertEquals(expectedString, knnVectorsFormat.toString()); + + ", flatVectorFormat=ES93BinaryQuantizedVectorsFormat(" + + "name=ES93BinaryQuantizedVectorsFormat, " + + "rawVectorFormat=ES93GenericFlatVectorsFormat(name=ES93GenericFlatVectorsFormat," + + " format=Lucene99FlatVectorsFormat"; + assertThat(knnVectorsFormat, hasToString(startsWith(expectedString))); } public void testKnnBBQIVFVectorsFormat() throws IOException { @@ -2078,8 +2075,6 @@ public void testInvalidVectorDimensionsBBQ() { public void testKnnHalfByteQuantizedHNSWVectorsFormat() throws IOException { final int m = randomIntBetween(1, DEFAULT_MAX_CONN + 10); final int efConstruction = randomIntBetween(1, DEFAULT_BEAM_WIDTH + 10); - boolean setConfidenceInterval = randomBoolean(); - float confidenceInterval = (float) randomDoubleBetween(0.90f, 1.0f, true); MapperService mapperService = createMapperService(fieldMapping(b -> { b.field("type", "dense_vector"); b.field("dims", dims); @@ -2089,9 +2084,6 @@ public void testKnnHalfByteQuantizedHNSWVectorsFormat() throws IOException { b.field("type", "int4_hnsw"); b.field("m", m); b.field("ef_construction", efConstruction); - if (setConfidenceInterval) { - b.field("confidence_interval", confidenceInterval); - } b.endObject(); })); CodecService codecService = new CodecService(mapperService, BigArrays.NON_RECYCLING_INSTANCE); @@ -2107,22 +2099,16 @@ public void testKnnHalfByteQuantizedHNSWVectorsFormat() throws IOException { assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class)); knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); } - VectorScorerFactory factory = VectorScorerFactory.instance().orElse(null); - String expectedString = "ES814HnswScalarQuantizedVectorsFormat(name=ES814HnswScalarQuantizedVectorsFormat, maxConn=" - + m - + ", beamWidth=" - + efConstruction - + ", flatVectorFormat=ES814ScalarQuantizedVectorsFormat(" - + "name=ES814ScalarQuantizedVectorsFormat, confidenceInterval=" - + (setConfidenceInterval ? confidenceInterval : 0.0f) - + ", bits=4, compressed=true, " - + "flatVectorScorer=ESFlatVectorsScorer(delegate=ScalarQuantizedVectorScorer(nonQuantizedDelegate=DefaultFlatVectorScorer()), " - + "factory=" - + (factory != null ? factory : "null") - + "), " - + "rawVectorFormat=Lucene99FlatVectorsFormat(vectorsScorer=DefaultFlatVectorScorer())" - + "))"; - assertEquals(expectedString, knnVectorsFormat.toString()); + String expectedString = Strings.format( + "ES93HnswScalarQuantizedVectorsFormat(name=ES93HnswScalarQuantizedVectorsFormat" + + ", maxConn=%s, beamWidth=%s, bits=4" + + ", flatVectorScorer=Lucene104ScalarQuantizedVectorScorer(nonQuantizedDelegate=DefaultFlatVectorScorer())" + + ", rawVectorFormat=ES93GenericFlatVectorsFormat(name=ES93GenericFlatVectorsFormat" + + ", format=Lucene99FlatVectorsFormat(name=Lucene99FlatVectorsFormat, flatVectorScorer=DefaultFlatVectorScorer())))", + m, + efConstruction + ); + assertThat(knnVectorsFormat, hasToString(expectedString)); } public void testInvalidVectorDimensions() { @@ -2158,24 +2144,23 @@ protected boolean supportsEmptyInputArray() { private static class DenseVectorSyntheticSourceSupport implements SyntheticSourceSupport { private final int dims = between(5, 1000); - private final ElementType elementType = randomFrom(ElementType.BYTE, ElementType.FLOAT, ElementType.BIT); + private final ElementType elementType = randomFrom(ElementType.BYTE, ElementType.FLOAT, ElementType.BFLOAT16, ElementType.BIT); private final boolean indexed = randomBoolean(); private final boolean indexOptionsSet = indexed && randomBoolean(); @Override public SyntheticSourceExample example(int maxValues) throws IOException { Object value = switch (elementType) { - case BYTE, BIT: - yield randomList(dims, dims, ESTestCase::randomByte); - case FLOAT: - yield randomList(dims, dims, ESTestCase::randomFloat); + case BYTE, BIT -> randomList(dims, dims, ESTestCase::randomByte); + case FLOAT -> randomList(dims, dims, ESTestCase::randomFloat); + case BFLOAT16 -> randomList(dims, dims, () -> BFloat16.truncateToBFloat16(randomFloat())); }; return new SyntheticSourceExample(value, value, this::mapping); } private void mapping(XContentBuilder b) throws IOException { b.field("type", "dense_vector"); - if (elementType == ElementType.BYTE || elementType == ElementType.BIT || randomBoolean()) { + if (elementType != ElementType.FLOAT || randomBoolean()) { b.field("element_type", elementType.toString()); } b.field("dims", elementType == ElementType.BIT ? dims * Byte.SIZE : dims); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 24db52cb10c0c..b48fd3cde3837 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -43,6 +43,7 @@ import static org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat.MAX_VECTORS_PER_CLUSTER; import static org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat.MIN_VECTORS_PER_CLUSTER; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.BBQ_MIN_DIMS; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType.BFLOAT16; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType.BIT; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType.BYTE; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.ElementType.FLOAT; @@ -74,11 +75,11 @@ public static DenseVectorFieldMapper.DenseVectorIndexOptions randomFlatIndexOpti return randomFrom( new DenseVectorFieldMapper.FlatIndexOptions(), new DenseVectorFieldMapper.Int8FlatIndexOptions( - randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), + null, randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) ), new DenseVectorFieldMapper.Int4FlatIndexOptions( - randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), + null, randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) ) ); @@ -90,7 +91,8 @@ public static DenseVectorFieldMapper.DenseVectorIndexOptions randomGpuSupportedI new DenseVectorFieldMapper.Int8HnswIndexOptions( randomIntBetween(1, 100), randomIntBetween(1, 3199), - randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), + null, + randomBoolean(), randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) ) ); @@ -112,27 +114,30 @@ public static DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsA new DenseVectorFieldMapper.Int8HnswIndexOptions( randomIntBetween(1, 100), randomIntBetween(1, 10_000), - randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), + null, + randomBoolean(), randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) ), new DenseVectorFieldMapper.Int4HnswIndexOptions( randomIntBetween(1, 100), randomIntBetween(1, 10_000), - randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), + null, + randomBoolean(), randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) ), new DenseVectorFieldMapper.FlatIndexOptions(), new DenseVectorFieldMapper.Int8FlatIndexOptions( - randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), + null, randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) ), new DenseVectorFieldMapper.Int4FlatIndexOptions( - randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), + null, randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) ), new DenseVectorFieldMapper.BBQHnswIndexOptions( randomIntBetween(1, 100), randomIntBetween(1, 10_000), + randomBoolean(), randomFrom((DenseVectorFieldMapper.RescoreVector) null, randomRescoreVector()) ), new DenseVectorFieldMapper.BBQFlatIndexOptions( @@ -164,16 +169,23 @@ private DenseVectorFieldMapper.DenseVectorIndexOptions randomIndexOptionsHnswQua new DenseVectorFieldMapper.Int8HnswIndexOptions( randomIntBetween(1, 100), randomIntBetween(1, 10_000), - randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), + null, + randomBoolean(), rescoreVector ), new DenseVectorFieldMapper.Int4HnswIndexOptions( randomIntBetween(1, 100), randomIntBetween(1, 10_000), - randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)), + null, + randomBoolean(), rescoreVector ), - new DenseVectorFieldMapper.BBQHnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000), rescoreVector) + new DenseVectorFieldMapper.BBQHnswIndexOptions( + randomIntBetween(1, 100), + randomIntBetween(1, 10_000), + randomBoolean(), + rescoreVector + ) ); } @@ -782,6 +794,7 @@ public void testRescoreOversampleQueryOverrides() { public void testFilterSearchThreshold() { List>> cases = List.of( Tuple.tuple(FLOAT, q -> ((ESKnnFloatVectorQuery) q).getStrategy()), + Tuple.tuple(BFLOAT16, q -> ((ESKnnFloatVectorQuery) q).getStrategy()), Tuple.tuple(BYTE, q -> ((ESKnnByteVectorQuery) q).getStrategy()), Tuple.tuple(BIT, q -> ((ESKnnByteVectorQuery) q).getStrategy()) ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SyntheticVectorsMapperTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SyntheticVectorsMapperTestCase.java index f7a23383f4e92..dbf4bd9846165 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SyntheticVectorsMapperTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SyntheticVectorsMapperTestCase.java @@ -25,6 +25,7 @@ import org.elasticsearch.xcontent.XContentType; import java.io.IOException; +import java.util.List; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; @@ -262,4 +263,9 @@ private void assertSyntheticVectors(String mapping, BytesReference source, XCont } } } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java index 04731f193fb14..65644e0dc7f50 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/AbstractKnnVectorQueryBuilderTestCase.java @@ -247,7 +247,7 @@ protected void doAssertLuceneQuery(KnnVectorQueryBuilder queryBuilder, Query que approxFilterQuery, expectedStrategy ); - case FLOAT -> new ESKnnFloatVectorQuery( + case FLOAT, BFLOAT16 -> new ESKnnFloatVectorQuery( VECTOR_FIELD, queryBuilder.queryVector().asFloatVector(), k, @@ -268,7 +268,7 @@ protected void doAssertLuceneQuery(KnnVectorQueryBuilder queryBuilder, Query que yield new DenseVectorQuery.Bytes(queryBuilder.queryVector().asByteVector(), VECTOR_FIELD); } } - case FLOAT -> { + case FLOAT, BFLOAT16 -> { if (filterQuery != null) { yield new BooleanQuery.Builder().add( new DenseVectorQuery.Floats(queryBuilder.queryVector().asFloatVector(), VECTOR_FIELD), diff --git a/server/src/test/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilderTests.java index 5871c1d8f18c5..27728aaa550b1 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/ExactKnnQueryBuilderTests.java @@ -101,7 +101,7 @@ protected void doAssertLuceneQuery(ExactKnnQueryBuilder queryBuilder, Query quer float[] expected = Arrays.copyOf(queryBuilder.getQuery().asFloatVector(), queryBuilder.getQuery().asFloatVector().length); float magnitude = VectorUtil.dotProduct(expected, expected); if (context.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.NORMALIZED_VECTOR_COSINE) - && DenseVectorFieldMapper.isNotUnitVector(magnitude)) { + && DenseVectorFieldMapper.FLOAT_ELEMENT.isUnitVector(magnitude) == false) { VectorUtil.l2normalize(expected); assertArrayEquals(expected, denseVectorQuery.getQuery(), 0.0f); } else { diff --git a/server/src/test/java/org/elasticsearch/search/vectors/RescoreKnnVectorQueryTests.java b/server/src/test/java/org/elasticsearch/search/vectors/RescoreKnnVectorQueryTests.java index 1fad497944026..286bfccca1f00 100644 --- a/server/src/test/java/org/elasticsearch/search/vectors/RescoreKnnVectorQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/vectors/RescoreKnnVectorQueryTests.java @@ -43,10 +43,10 @@ import org.elasticsearch.index.codec.vectors.ES813Int8FlatVectorFormat; import org.elasticsearch.index.codec.vectors.ES814HnswScalarQuantizedVectorsFormat; import org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat; -import org.elasticsearch.index.codec.vectors.es818.ES818BinaryQuantizedVectorsFormat; -import org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsFormat; +import org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat; import org.elasticsearch.index.codec.vectors.es93.ES93HnswBinaryQuantizedVectorsFormat; import org.elasticsearch.index.codec.zstd.Zstd814StoredFieldsFormat; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.search.profile.query.QueryProfiler; import org.elasticsearch.test.ESTestCase; @@ -283,10 +283,20 @@ private static void addRandomDocuments(int numDocs, Directory d, int numDims) th IndexWriterConfig iwc = new IndexWriterConfig(); // Pick codec from quantized vector formats to ensure scores use real scores when using knn rescore KnnVectorsFormat format = randomFrom( - new ES920DiskBBQVectorsFormat(DEFAULT_VECTORS_PER_CLUSTER, DEFAULT_CENTROIDS_PER_PARENT_CLUSTER, randomBoolean()), - new ES818BinaryQuantizedVectorsFormat(), - new ES818HnswBinaryQuantizedVectorsFormat(), - new ES93HnswBinaryQuantizedVectorsFormat(), + new ES920DiskBBQVectorsFormat( + DEFAULT_VECTORS_PER_CLUSTER, + DEFAULT_CENTROIDS_PER_PARENT_CLUSTER, + randomFrom(DenseVectorFieldMapper.ElementType.FLOAT, DenseVectorFieldMapper.ElementType.BFLOAT16), + randomBoolean() + ), + new ES93BinaryQuantizedVectorsFormat( + randomFrom(DenseVectorFieldMapper.ElementType.FLOAT, DenseVectorFieldMapper.ElementType.BFLOAT16), + randomBoolean() + ), + new ES93HnswBinaryQuantizedVectorsFormat( + randomFrom(DenseVectorFieldMapper.ElementType.FLOAT, DenseVectorFieldMapper.ElementType.BFLOAT16), + randomBoolean() + ), new ES813Int8FlatVectorFormat(), new ES813Int8FlatVectorFormat(), new ES814HnswScalarQuantizedVectorsFormat() diff --git a/test/framework/src/main/java/org/elasticsearch/action/admin/indices/ResizeIndexTestUtils.java b/test/framework/src/main/java/org/elasticsearch/action/admin/indices/ResizeIndexTestUtils.java new file mode 100644 index 0000000000000..7073c124e23d9 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/action/admin/indices/ResizeIndexTestUtils.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.admin.indices; + +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; +import org.elasticsearch.common.settings.Settings; + +import static org.elasticsearch.test.ESIntegTestCase.client; +import static org.elasticsearch.test.ESTestCase.TEST_REQUEST_TIMEOUT; + +public enum ResizeIndexTestUtils { + ; + + public static ResizeRequest resizeRequest(ResizeType resizeType, String sourceIndex, String targetIndex, Settings.Builder settings) { + final var resizeRequest = new ResizeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, resizeType, sourceIndex, targetIndex); + resizeRequest.setTargetIndexSettings(settings); + return resizeRequest; + } + + public static ActionFuture executeResize( + ResizeType resizeType, + String sourceIndex, + String targetIndex, + Settings.Builder settings + ) { + return client().execute(TransportResizeAction.TYPE, resizeRequest(resizeType, sourceIndex, targetIndex, settings)); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapsUtils.java b/test/framework/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapsUtils.java index 84c057d3b6a81..4a181d816451b 100644 --- a/test/framework/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapsUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapsUtils.java @@ -61,7 +61,12 @@ public static FieldCapabilitiesFailure parseFailure(XContentParser parser) throw .collect(Collectors.toMap(Tuple::v1, Tuple::v2)); List indices = a[1] == null ? Collections.emptyList() : (List) a[1]; List failures = a[2] == null ? Collections.emptyList() : (List) a[2]; - return new FieldCapabilitiesResponse(indices.toArray(String[]::new), responseMap, failures); + + return FieldCapabilitiesResponse.builder() + .withIndices(indices.toArray(String[]::new)) + .withFields(responseMap) + .withFailures(failures) + .build(); } ); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 3ef66cebc1f14..f7ee3dcdbda43 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -22,6 +22,7 @@ import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.Query; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; @@ -64,6 +65,7 @@ import org.elasticsearch.script.ScriptFactory; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.lookup.LeafStoredFieldsLookup; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.Source; @@ -1697,4 +1699,68 @@ protected T compileOtherScript(Script script, ScriptContext context) { */ abstract ScriptFactory nonEmptyFieldScript(); } + + /** + * Return a list of scenarios where sorts on this field should or should not produce + * competitive iterators. Field types that do not support sorting should return an + * empty List. + */ + protected abstract List getSortShortcutSupport(); + + public record SortShortcutSupport( + IndexVersion indexVersion, + Settings settings, + CheckedConsumer mappings, + CheckedConsumer document, + boolean supportsShortcut + ) { + public SortShortcutSupport( + CheckedConsumer mappings, + CheckedConsumer document, + boolean supportsShortcut + ) { + this(IndexVersion.current(), SETTINGS, mappings, document, supportsShortcut); + } + + public SortShortcutSupport( + IndexVersion indexVersion, + CheckedConsumer mappings, + CheckedConsumer document, + boolean supportsShortcut + ) { + this(indexVersion, SETTINGS, mappings, document, supportsShortcut); + } + } + + public final void testSortShortcuts() throws IOException { + List tests = getSortShortcutSupport(); + assumeTrue("Sort shortcuts not supported", tests != null && tests.isEmpty() == false); + + for (SortShortcutSupport sortShortcutSupport : tests) { + MapperService mapperService = createMapperService(sortShortcutSupport.indexVersion(), sortShortcutSupport.settings, () -> true); + merge(mapperService, fieldMapping(sortShortcutSupport.mappings)); + withLuceneIndex(mapperService, iw -> { + iw.addDocument( + mapperService.documentParser() + .parseDocument(source(sortShortcutSupport.document()), mapperService.mappingLookup()) + .rootDoc() + ); + }, reader -> { + IndexSearcher searcher = newSearcher(reader); + MappedFieldType ft = mapperService.fieldType("field"); + SortField sortField = ft.fielddataBuilder(new FieldDataContext("", mapperService.getIndexSettings(), () -> { + throw new UnsupportedOperationException(); + }, Set::of, MappedFieldType.FielddataOperation.SEARCH)) + .build(null, null) + .sortField(getVersion(), null, MultiValueMode.MIN, null, false); + var comparator = sortField.getComparator(10, Pruning.GREATER_THAN_OR_EQUAL_TO); + var leafComparator = comparator.getLeafComparator(searcher.getLeafContexts().getFirst()); + if (sortShortcutSupport.supportsShortcut) { + assertNotNull(leafComparator.competitiveIterator()); + } else { + assertNull(leafComparator.competitiveIterator()); + } + }); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 18e60c008db96..2f16534908b2e 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.script.DoubleFieldScript; import org.elasticsearch.script.LongFieldScript; import org.elasticsearch.script.Script; @@ -496,4 +497,12 @@ public List invalidExample() throws IOException { return List.of(); } } + + @Override + protected List getSortShortcutSupport() { + return List.of( + new SortShortcutSupport(this::minimalMapping, this::writeField, true), + new SortShortcutSupport(IndexVersion.fromId(5000099), this::minimalMapping, this::writeField, false) + ); + } } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java index ef096f344f352..b26b33cc01544 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java @@ -523,4 +523,9 @@ public List invalidExample() throws IOException { public void testSyntheticSourceKeepArrays() { // The mapper expects to parse an array of values by default, it's not compatible with array of arrays. } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java index bca057b4d3416..0b18d4b152888 100644 --- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java +++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils; import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; @@ -41,6 +42,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.resizeRequest; import static org.elasticsearch.index.store.Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.xpack.autoscaling.storage.ReactiveStorageDeciderService.AllocationState.MAX_AMOUNT_OF_SHARD_DECISIONS; @@ -347,16 +349,9 @@ public void testScaleWhileShrinking() throws Exception { }); String shrinkName = "shrink-" + indexName; - assertAcked( - indicesAdmin().prepareResizeIndex(indexName, shrinkName) - .setSettings( - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .build() - ) - .setWaitForActiveShards(ActiveShardCount.NONE) - ); + final var shrinkRequest = resizeRequest(ResizeType.SHRINK, indexName, shrinkName, indexSettings(1, 0)); + shrinkRequest.setWaitForActiveShards(ActiveShardCount.NONE); + assertAcked(client().execute(TransportResizeAction.TYPE, shrinkRequest)); // * 2 since worst case is no hard links, see DiskThresholdDecider.getExpectedShardSize. long requiredSpaceForShrink = used * 2 + LOW_WATERMARK_BYTES; @@ -455,17 +450,9 @@ public void testScaleDuringSplit() throws Exception { updateIndexSettings(Settings.builder().put("index.blocks.write", true), indexName); String cloneName = "clone-" + indexName; int resizedShardCount = between(2, 10); - assertAcked( - indicesAdmin().prepareResizeIndex(indexName, cloneName) - .setSettings( - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, resizedShardCount) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .build() - ) - .setWaitForActiveShards(ActiveShardCount.NONE) - .setResizeType(ResizeType.SPLIT) - ); + final var splitRequest = resizeRequest(ResizeType.SPLIT, indexName, cloneName, indexSettings(resizedShardCount, 0)); + splitRequest.setWaitForActiveShards(ActiveShardCount.NONE); + assertAcked(client().execute(TransportResizeAction.TYPE, splitRequest)); // * 2 since worst case is no hard links, see DiskThresholdDecider.getExpectedShardSize. long requiredSpaceForClone = used * 2 + LOW_WATERMARK_BYTES; diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java index 041be4ea40e6d..421a010daec50 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanationUtils; import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesAction; import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesRequest; +import org.elasticsearch.action.admin.indices.ResizeIndexTestUtils; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.cluster.health.ClusterHealthStatus; @@ -324,12 +325,7 @@ public void testShrinkStaysOnTier() { .get(); indicesAdmin().prepareAddBlock(IndexMetadata.APIBlock.READ_ONLY, index).get(); - indicesAdmin().prepareResizeIndex(index, index + "-shrunk") - .setResizeType(ResizeType.SHRINK) - .setSettings( - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build() - ) - .get(); + ResizeIndexTestUtils.executeResize(ResizeType.SHRINK, index, index + "-shrunk", indexSettings(1, 0)).actionGet(); ensureGreen(index + "-shrunk"); diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java index f9c326c1ac854..8330f7c92a8f9 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.core.action; -import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; @@ -164,7 +163,7 @@ public void testAction() throws Exception { timeSeriesDataStreamCount.incrementAndGet(); if (downsamplingConfiguredBy == DownsampledBy.DLM) { dlmDownsampledDataStreamCount.incrementAndGet(); - updateRounds(lifecycle.downsampling().size(), dlmRoundsCount, dlmRoundsSum, dlmRoundsMin, dlmRoundsMax); + updateRounds(lifecycle.downsamplingRounds().size(), dlmRoundsCount, dlmRoundsSum, dlmRoundsMin, dlmRoundsMax); } else if (downsamplingConfiguredBy == DownsampledBy.ILM) { ilmDownsampledDataStreamCount.incrementAndGet(); } @@ -410,7 +409,7 @@ private DataStreamLifecycle maybeCreateLifecycle(boolean isDownsampled, boolean } var builder = DataStreamLifecycle.dataLifecycleBuilder(); if (isDownsampled) { - builder.downsampling(randomDownsamplingRounds()); + builder.downsamplingRounds(randomDownsamplingRounds()); } return builder.build(); } @@ -471,12 +470,7 @@ private List randomDownsamplingRounds() { int minutes = 5; int days = 1; for (int i = 0; i < randomIntBetween(1, 10); i++) { - rounds.add( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(days), - new DownsampleConfig(new DateHistogramInterval(minutes + "m")) - ) - ); + rounds.add(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(days), new DateHistogramInterval(minutes + "m"))); minutes *= randomIntBetween(2, 5); days += randomIntBetween(1, 5); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java index 77245a396e5db..5e980387bc4f8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java @@ -89,9 +89,9 @@ protected void localClusterStateOperation( continue; } tsDataStreamCount++; - Integer dlmRounds = ds.getDataLifecycle() == null || ds.getDataLifecycle().downsampling() == null + Integer dlmRounds = ds.getDataLifecycle() == null || ds.getDataLifecycle().downsamplingRounds() == null ? null - : ds.getDataLifecycle().downsampling().size(); + : ds.getDataLifecycle().downsamplingRounds().size(); for (Index backingIndex : ds.getIndices()) { IndexMetadata indexMetadata = projectMetadata.index(backingIndex); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ResizeIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ResizeIndexStep.java index 847647b0cd5d3..81315944498cf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ResizeIndexStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ResizeIndexStep.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.ProjectState; @@ -95,10 +96,13 @@ public void performAction( .put(LifecycleSettings.LIFECYCLE_SKIP, true) .build(); - ResizeRequest resizeRequest = new ResizeRequest(targetIndexName, indexMetadata.getIndex().getName()).masterNodeTimeout( - TimeValue.MAX_VALUE + ResizeRequest resizeRequest = new ResizeRequest( + TimeValue.MAX_VALUE, + TimeValue.MAX_VALUE, + resizeType, + indexMetadata.getIndex().getName(), + targetIndexName ); - resizeRequest.setResizeType(resizeType); resizeRequest.getTargetIndexRequest().settings(relevantTargetSettings); if (resizeType == ResizeType.SHRINK) { resizeRequest.setMaxPrimaryShardSize(maxPrimaryShardSize); @@ -106,9 +110,7 @@ public void performAction( // This request does not wait for (successful) completion of the resize operation - it fires-and-forgets. // It's up to a subsequent step to check for the existence of the target index and wait for it to be green. - getClient(currentState.projectId()).admin() - .indices() - .resizeIndex(resizeRequest, listener.delegateFailureAndWrap((l, response) -> l.onResponse(null))); + getClient(currentState.projectId()).execute(TransportResizeAction.TYPE, resizeRequest, listener.map(response -> null)); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java index 1b25b36e1d2db..548db7e08f598 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java @@ -41,8 +41,10 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.IgnoreMalformedStoredValues; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.TextFamilyFieldType; import org.elasticsearch.transport.Transports; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -55,6 +57,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; /** * A {@link FilterLeafReader} that exposes only a subset @@ -69,15 +72,18 @@ public final class FieldSubsetReader extends SequentialStoredFieldsLeafReader { * Note that for convenience, the returned reader * can be used normally (e.g. passed to {@link DirectoryReader#openIfChanged(DirectoryReader)}) * and so on. - * @param in reader to filter - * @param filter fields to filter. + * + * @param in reader to filter + * @param filter fields to filter. + * @param isMapped whether a field is mapped or not. */ public static DirectoryReader wrap( DirectoryReader in, CharacterRunAutomaton filter, - IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat, + Function isMapped ) throws IOException { - return new FieldSubsetDirectoryReader(in, filter, ignoredSourceFormat); + return new FieldSubsetDirectoryReader(in, filter, ignoredSourceFormat, isMapped); } // wraps subreaders with fieldsubsetreaders. @@ -85,17 +91,19 @@ static class FieldSubsetDirectoryReader extends FilterDirectoryReader { private final CharacterRunAutomaton filter; private final IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat; + private final Function isMapped; FieldSubsetDirectoryReader( DirectoryReader in, final CharacterRunAutomaton filter, - final IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat + final IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat, + Function isMapped ) throws IOException { super(in, new FilterDirectoryReader.SubReaderWrapper() { @Override public LeafReader wrap(LeafReader reader) { try { - return new FieldSubsetReader(reader, filter, ignoredSourceFormat); + return new FieldSubsetReader(reader, filter, ignoredSourceFormat, isMapped); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -103,12 +111,13 @@ public LeafReader wrap(LeafReader reader) { }); this.filter = filter; this.ignoredSourceFormat = ignoredSourceFormat; + this.isMapped = isMapped; verifyNoOtherFieldSubsetDirectoryReaderIsWrapped(in); } @Override protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { - return new FieldSubsetDirectoryReader(in, filter, ignoredSourceFormat); + return new FieldSubsetDirectoryReader(in, filter, ignoredSourceFormat, isMapped); } /** Return the automaton that is used to filter fields. */ @@ -145,12 +154,25 @@ public CacheHelper getReaderCacheHelper() { /** * Wrap a single segment, exposing a subset of its fields. */ - FieldSubsetReader(LeafReader in, CharacterRunAutomaton filter, IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat) - throws IOException { + FieldSubsetReader( + LeafReader in, + CharacterRunAutomaton filter, + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat, + Function isMapped + ) throws IOException { super(in); ArrayList filteredInfos = new ArrayList<>(); for (FieldInfo fi : in.getFieldInfos()) { - if (filter.run(fi.name)) { + String name = fi.name; + if (fi.getName().endsWith(TextFamilyFieldType.FALLBACK_FIELD_NAME_SUFFIX) && isMapped.apply(fi.getName()) == false) { + name = fi.getName().substring(0, fi.getName().length() - TextFamilyFieldType.FALLBACK_FIELD_NAME_SUFFIX.length()); + } + if (fi.getName().endsWith(IgnoreMalformedStoredValues.IGNORE_MALFORMED_FIELD_NAME_SUFFIX) + && isMapped.apply(fi.getName()) == false) { + name = fi.getName() + .substring(0, fi.getName().length() - IgnoreMalformedStoredValues.IGNORE_MALFORMED_FIELD_NAME_SUFFIX.length()); + } + if (filter.run(name)) { filteredInfos.add(fi); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java index abb13702afb86..db8e71295f155 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java @@ -96,9 +96,11 @@ public DirectoryReader apply(final DirectoryReader reader) { } } - var indexVersionCreated = searchExecutionContextProvider.apply(shardId).indexVersionCreated(); + var searchContext = searchExecutionContextProvider.apply(shardId); + var indexVersionCreated = searchContext.indexVersionCreated(); + Function isMapped = searchContext::isFieldMapped; - return permissions.getFieldPermissions().filter(wrappedReader, indexVersionCreated); + return permissions.getFieldPermissions().filter(wrappedReader, indexVersionCreated, isMapped); } catch (IOException e) { logger.error("Unable to apply field level security"); throw ExceptionsHelper.convertToElastic(e); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java index 3369774ad5c5a..828485066fca4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java @@ -34,6 +34,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -245,11 +246,17 @@ public boolean hasFieldLevelSecurity() { } /** Return a wrapped reader that only exposes allowed fields. */ - public DirectoryReader filter(DirectoryReader reader, IndexVersion indexVersionCreated) throws IOException { + public DirectoryReader filter(DirectoryReader reader, IndexVersion indexVersionCreated, Function isMapped) + throws IOException { if (hasFieldLevelSecurity() == false) { return reader; } - return FieldSubsetReader.wrap(reader, permittedFieldsAutomaton, IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersionCreated)); + return FieldSubsetReader.wrap( + reader, + permittedFieldsAutomaton, + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersionCreated), + isMapped + ); } Automaton getIncludeAutomaton() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datastreams/DataStreamLifecycleFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datastreams/DataStreamLifecycleFeatureSetUsageTests.java index fa4e7ead7eaf9..1516353731b56 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datastreams/DataStreamLifecycleFeatureSetUsageTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datastreams/DataStreamLifecycleFeatureSetUsageTests.java @@ -117,7 +117,7 @@ public void testLifecycleStats() { 1L, null, false, - DataStreamLifecycle.createDataLifecycle(true, TimeValue.timeValueSeconds(50), null) + DataStreamLifecycle.createDataLifecycle(true, TimeValue.timeValueSeconds(50), null, null) ), DataStreamTestHelper.newInstance( randomAlphaOfLength(10), @@ -125,7 +125,7 @@ public void testLifecycleStats() { 1L, null, false, - DataStreamLifecycle.createDataLifecycle(true, TimeValue.timeValueMillis(150), null) + DataStreamLifecycle.createDataLifecycle(true, TimeValue.timeValueMillis(150), null, null) ), DataStreamTestHelper.newInstance( randomAlphaOfLength(10), @@ -133,7 +133,7 @@ public void testLifecycleStats() { 1L, null, false, - DataStreamLifecycle.createDataLifecycle(false, TimeValue.timeValueSeconds(5), null) + DataStreamLifecycle.createDataLifecycle(false, TimeValue.timeValueSeconds(5), null, null) ), DataStreamTestHelper.newInstance( randomAlphaOfLength(10), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ResizeIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ResizeIndexStepTests.java index 9fb230885a4b9..8316c587384af 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ResizeIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ResizeIndexStepTests.java @@ -8,9 +8,9 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.cluster.ProjectState; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -119,9 +119,8 @@ public void testPerformAction() throws Exception { .build(); Mockito.doAnswer(invocation -> { - ResizeRequest request = (ResizeRequest) invocation.getArguments()[0]; - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; + final ResizeRequest request = invocation.getArgument(1); + final ActionListener listener = invocation.getArgument(2); assertThat(request.getSourceIndex(), equalTo(sourceIndexMetadata.getIndex().getName())); assertThat(request.getTargetIndexRequest().aliases(), equalTo(Set.of())); @@ -133,16 +132,14 @@ public void testPerformAction() throws Exception { assertThat(request.getMaxPrimaryShardSize(), equalTo(step.getMaxPrimaryShardSize())); listener.onResponse(new CreateIndexResponse(true, true, sourceIndexMetadata.getIndex().getName())); return null; - }).when(indicesClient).resizeIndex(Mockito.any(), Mockito.any()); + }).when(projectClient).execute(Mockito.same(TransportResizeAction.TYPE), Mockito.any(), Mockito.any()); final var state = projectStateWithEmptyProject(); performActionAndWait(step, sourceIndexMetadata, state, null); Mockito.verify(client).projectClient(state.projectId()); - Mockito.verify(projectClient).admin(); - Mockito.verifyNoMoreInteractions(client); - Mockito.verify(adminClient, Mockito.only()).indices(); - Mockito.verify(indicesClient, Mockito.only()).resizeIndex(Mockito.any(), Mockito.any()); + Mockito.verify(projectClient).execute(Mockito.same(TransportResizeAction.TYPE), Mockito.any(), Mockito.any()); + Mockito.verifyNoMoreInteractions(client, projectClient); } public void testPerformActionShrunkenIndexExists() throws Exception { @@ -195,23 +192,20 @@ public void testPerformActionIsCompleteForUnAckedRequests() throws Exception { ResizeIndexStep step = createRandomInstance(); Mockito.doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; + final ActionListener listener = invocation.getArgument(2); listener.onResponse(new CreateIndexResponse(false, false, indexMetadata.getIndex().getName())); return null; - }).when(indicesClient).resizeIndex(Mockito.any(), Mockito.any()); + }).when(projectClient).execute(Mockito.same(TransportResizeAction.TYPE), Mockito.any(), Mockito.any()); final var state = projectStateWithEmptyProject(); performActionAndWait(step, indexMetadata, state, null); Mockito.verify(client).projectClient(state.projectId()); - Mockito.verify(projectClient).admin(); - Mockito.verifyNoMoreInteractions(client); - Mockito.verify(adminClient, Mockito.only()).indices(); - Mockito.verify(indicesClient, Mockito.only()).resizeIndex(Mockito.any(), Mockito.any()); + Mockito.verify(projectClient).execute(Mockito.same(TransportResizeAction.TYPE), Mockito.any(), Mockito.any()); + Mockito.verifyNoMoreInteractions(client, projectClient); } - public void testPerformActionFailure() throws Exception { + public void testPerformActionFailure() { LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(); lifecycleState.setIndexCreationDate(randomNonNegativeLong()); IndexMetadata indexMetadata = IndexMetadata.builder(randomAlphaOfLength(10)) @@ -224,20 +218,17 @@ public void testPerformActionFailure() throws Exception { ResizeIndexStep step = createRandomInstance(); Mockito.doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; + final ActionListener listener = invocation.getArgument(2); listener.onFailure(exception); return null; - }).when(indicesClient).resizeIndex(Mockito.any(), Mockito.any()); + }).when(projectClient).execute(Mockito.same(TransportResizeAction.TYPE), Mockito.any(), Mockito.any()); final var state = projectStateWithEmptyProject(); assertSame(exception, expectThrows(Exception.class, () -> performActionAndWait(step, indexMetadata, state, null))); Mockito.verify(client).projectClient(state.projectId()); - Mockito.verify(projectClient).admin(); - Mockito.verifyNoMoreInteractions(client); - Mockito.verify(adminClient, Mockito.only()).indices(); - Mockito.verify(indicesClient, Mockito.only()).resizeIndex(Mockito.any(), Mockito.any()); + Mockito.verify(projectClient).execute(Mockito.same(TransportResizeAction.TYPE), Mockito.any(), Mockito.any()); + Mockito.verifyNoMoreInteractions(client, projectClient); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java index d14bd8c8c8196..549a5f5714b8b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java @@ -427,7 +427,7 @@ public void testFieldCardinalityLimitsIsNonEmpty() { } public void testGetResultMappings_DependentVariableMappingIsAbsent() { - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse(new String[0], Collections.emptyMap()); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.empty(); expectThrows( ElasticsearchStatusException.class, () -> new Classification("foo").getResultMappings("results", fieldCapabilitiesResponse) @@ -435,10 +435,9 @@ public void testGetResultMappings_DependentVariableMappingIsAbsent() { } public void testGetResultMappings_DependentVariableMappingHasNoTypes() { - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse( - new String[0], - Collections.singletonMap("foo", Collections.emptyMap()) - ); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.builder() + .withFields(Collections.singletonMap("foo", Collections.emptyMap())) + .build(); expectThrows( ElasticsearchStatusException.class, () -> new Classification("foo").getResultMappings("results", fieldCapabilitiesResponse) @@ -459,10 +458,9 @@ public void testGetResultMappings_DependentVariableMappingIsPresent() { Map.of("type", "double") ) ); - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse( - new String[0], - Collections.singletonMap("foo", Collections.singletonMap("dummy", createFieldCapabilities("foo", "dummy"))) - ); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.builder() + .withFields(Collections.singletonMap("foo", Collections.singletonMap("dummy", createFieldCapabilities("foo", "dummy")))) + .build(); Map resultMappings = new Classification("foo").getResultMappings("results", fieldCapabilitiesResponse); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java index 23a2792663ec0..01d7175e9ad52 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java @@ -115,7 +115,8 @@ public void testIndexed() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -150,7 +151,8 @@ public void testPoints() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -213,7 +215,8 @@ public void testKnnVectors() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); LeafReader leafReader = ir.leaves().get(0).reader(); @@ -264,7 +267,8 @@ public void testKnnByteVectors() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); LeafReader leafReader = ir.leaves().get(0).reader(); @@ -320,7 +324,8 @@ public void testStoredFieldsString() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -351,7 +356,8 @@ public void testStoredFieldsBinary() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -382,7 +388,8 @@ public void testStoredFieldsInt() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -413,7 +420,8 @@ public void testStoredFieldsLong() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -444,7 +452,8 @@ public void testStoredFieldsFloat() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -475,7 +484,8 @@ public void testStoredFieldsDouble() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -508,7 +518,8 @@ public void testVectors() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -541,7 +552,8 @@ public void testNorms() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -571,7 +583,8 @@ public void testNumericDocValues() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -604,7 +617,8 @@ public void testBinaryDocValues() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -637,7 +651,8 @@ public void testSortedDocValues() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -670,7 +685,8 @@ public void testSortedSetDocValues() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -704,7 +720,8 @@ public void testSortedNumericDocValues() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -738,7 +755,8 @@ public void testFieldInfos() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -773,7 +791,8 @@ public void testSourceFilteringIntegration() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -819,7 +838,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion), + (fieldName) -> true ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -837,7 +857,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion), + (fieldName) -> true ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -852,7 +873,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion), + (fieldName) -> true ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -871,7 +893,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion), + (fieldName) -> true ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -886,7 +909,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion), + (fieldName) -> true ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -905,7 +929,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion), + (fieldName) -> true ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -916,6 +941,82 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { } } + public void testVisibilityOriginalFieldNames() throws Exception { + try (Directory dir = newDirectory()) { + try (IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(null))) { + Document doc = new Document(); + doc.add(new StoredField("a._original", new BytesRef("a"))); + doc.add(new StoredField("b._ignore_malformed", new BytesRef("b"))); + doc.add(new StoredField("c", new BytesRef("c"))); + iw.addDocument(doc); + + // Field a is mapped: + + var filter = new CharacterRunAutomaton(Automatons.patterns(List.of("a", "c"))); + try ( + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + filter, + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false + ) + ) { + + Document fields = ir.storedFields().document(0); + assertEquals(2, fields.getFields().size()); + assertEquals(new BytesRef("a"), fields.getBinaryValue("a._original")); + assertEquals(new BytesRef("c"), fields.getBinaryValue("c")); + } + // Field a is not mapped: + try ( + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + filter, + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true + ) + ) { + + Document fields = ir.storedFields().document(0); + assertEquals(1, fields.getFields().size()); + assertNull(fields.getBinaryValue("a._original")); + assertEquals(new BytesRef("c"), fields.getBinaryValue("c")); + } + // Field b is mapped: + filter = new CharacterRunAutomaton(Automatons.patterns(List.of("b", "c"))); + try ( + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + filter, + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false + ) + ) { + + Document fields = ir.storedFields().document(0); + assertEquals(2, fields.getFields().size()); + assertEquals(new BytesRef("b"), fields.getBinaryValue("b._ignore_malformed")); + assertEquals(new BytesRef("c"), fields.getBinaryValue("c")); + } + // Field b is not mapped: + try ( + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + filter, + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true + ) + ) { + + Document fields = ir.storedFields().document(0); + assertEquals(1, fields.getFields().size()); + assertNull(fields.getBinaryValue("b._ignore_malformed")); + assertEquals(new BytesRef("c"), fields.getBinaryValue("c")); + } + } + } + } + public void testSourceFiltering() { // include on top-level value Map map = new HashMap<>(); @@ -1081,7 +1182,8 @@ public void testFieldNames() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); // see only one field @@ -1143,7 +1245,8 @@ public void testFieldNamesThreeFields() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); // see only two fields @@ -1192,7 +1295,8 @@ public void testFieldNamesMissing() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); // see only one field @@ -1230,7 +1334,8 @@ public void testFieldNamesOldIndex() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); // see only one field @@ -1262,7 +1367,8 @@ public void testCoreCacheKey() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("id")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); assertEquals(2, ir.numDocs()); assertEquals(1, ir.leaves().size()); @@ -1300,7 +1406,8 @@ public void testFilterAwayAllVectors() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldB")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); // sees no fields @@ -1323,7 +1430,8 @@ public void testEmpty() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); // see no fields @@ -1356,14 +1464,16 @@ public void testWrapTwice() throws Exception { final DirectoryReader directoryReader = FieldSubsetReader.wrap( DirectoryReader.open(dir), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, () -> FieldSubsetReader.wrap( directoryReader, new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ) ); assertThat( @@ -1553,7 +1663,8 @@ public void testProducesStoredFieldsReader() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); TestUtil.checkReader(ir); diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java index b07a00c13903c..8fff5bb52a75f 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.rollover.RolloverAction; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; -import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; @@ -52,13 +51,8 @@ public void testDataStreamLifecycleDownsampleRollingRestart() throws Exception { final String dataStreamName = "metrics-foo"; DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( - List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ) - ) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m"))) ) .buildTemplate(); setupTSDBDataStreamAndIngestDocs( diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java index 114bdeeff7e98..7cfbdef90a2bd 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java @@ -9,13 +9,19 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverAction; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction; import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.junit.annotations.TestLogging; import java.util.HashSet; @@ -26,6 +32,7 @@ import static org.elasticsearch.cluster.metadata.ClusterChangedEventUtils.indicesCreated; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; public class DataStreamLifecycleDownsampleIT extends DownsamplingIntegTestCase { @@ -42,17 +49,13 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { public void testDownsampling() throws Exception { String dataStreamName = "metrics-foo"; + DownsampleConfig.SamplingMethod downsamplingMethod = randomSamplingMethod(); DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingMethod(downsamplingMethod) + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueSeconds(10), - new DownsampleConfig(new DateHistogramInterval("10m")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueSeconds(10), new DateHistogramInterval("10m")) ) ) .buildTemplate(); @@ -68,34 +71,34 @@ public void testDownsampling() throws Exception { List backingIndices = getDataStreamBackingIndexNames(dataStreamName); String firstGenerationBackingIndex = backingIndices.get(0); - String oneSecondDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex; - String tenSecondsDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex; + String fiveMinuteDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex; + String tenMinuteDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex; Set witnessedDownsamplingIndices = new HashSet<>(); clusterService().addListener(event -> { - if (indicesCreated(event).contains(oneSecondDownsampleIndex) - || event.indicesDeleted().stream().anyMatch(index -> index.getName().equals(oneSecondDownsampleIndex))) { - witnessedDownsamplingIndices.add(oneSecondDownsampleIndex); + if (indicesCreated(event).contains(fiveMinuteDownsampleIndex) + || event.indicesDeleted().stream().anyMatch(index -> index.getName().equals(fiveMinuteDownsampleIndex))) { + witnessedDownsamplingIndices.add(fiveMinuteDownsampleIndex); } - if (indicesCreated(event).contains(tenSecondsDownsampleIndex)) { - witnessedDownsamplingIndices.add(tenSecondsDownsampleIndex); + if (indicesCreated(event).contains(tenMinuteDownsampleIndex)) { + witnessedDownsamplingIndices.add(tenMinuteDownsampleIndex); } }); // before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with - // testing so DSL doesn't have to wait for the end_time to lapse) + // testing, so DSL doesn't have to wait for the end_time to lapse) putTSDBIndexTemplate(dataStreamName, null, null, lifecycle); client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet(); assertBusy(() -> { // first downsampling round - assertThat(witnessedDownsamplingIndices.contains(oneSecondDownsampleIndex), is(true)); + assertThat(witnessedDownsamplingIndices.contains(fiveMinuteDownsampleIndex), is(true)); }, 30, TimeUnit.SECONDS); assertBusy(() -> { assertThat(witnessedDownsamplingIndices.size(), is(2)); - assertThat(witnessedDownsamplingIndices.contains(oneSecondDownsampleIndex), is(true)); - assertThat(witnessedDownsamplingIndices.contains(tenSecondsDownsampleIndex), is(true)); + assertThat(witnessedDownsamplingIndices.contains(fiveMinuteDownsampleIndex), is(true)); + assertThat(witnessedDownsamplingIndices.contains(tenMinuteDownsampleIndex), is(true)); }, 30, TimeUnit.SECONDS); assertBusy(() -> { @@ -105,28 +108,25 @@ public void testDownsampling() throws Exception { String writeIndex = dsBackingIndices.get(1); assertThat(writeIndex, backingIndexEqualTo(dataStreamName, 2)); // the last downsampling round must remain in the data stream - assertThat(dsBackingIndices.get(0), is(tenSecondsDownsampleIndex)); - assertThat(indexExists(oneSecondDownsampleIndex), is(false)); + assertThat(dsBackingIndices.get(0), is(tenMinuteDownsampleIndex)); + assertThat(indexExists(fiveMinuteDownsampleIndex), is(false)); }, 30, TimeUnit.SECONDS); + assertDownsamplingMethod(downsamplingMethod, tenMinuteDownsampleIndex); } @TestLogging(value = "org.elasticsearch.datastreams.lifecycle:TRACE", reason = "debugging") public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception { String dataStreamName = "metrics-bar"; + DownsampleConfig.SamplingMethod downsamplingMethod = randomSamplingMethod(); DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingMethod(downsamplingMethod) + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), // data stream lifecycle runs every 1 second, so by the time we forcemerge the backing index it would've been at // least 2 seconds since rollover. only the 10 seconds round should be executed. - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(10), - new DownsampleConfig(new DateHistogramInterval("10m")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(10), new DateHistogramInterval("10m")) ) ) .buildTemplate(); @@ -141,28 +141,28 @@ public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception List backingIndices = getDataStreamBackingIndexNames(dataStreamName); String firstGenerationBackingIndex = backingIndices.get(0); - String oneSecondDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex; - String tenSecondsDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex; + String fiveMinuteDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex; + String tenMinuteDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex; Set witnessedDownsamplingIndices = new HashSet<>(); clusterService().addListener(event -> { - if (indicesCreated(event).contains(oneSecondDownsampleIndex) - || event.indicesDeleted().stream().anyMatch(index -> index.getName().equals(oneSecondDownsampleIndex))) { - witnessedDownsamplingIndices.add(oneSecondDownsampleIndex); + if (indicesCreated(event).contains(fiveMinuteDownsampleIndex) + || event.indicesDeleted().stream().anyMatch(index -> index.getName().equals(fiveMinuteDownsampleIndex))) { + witnessedDownsamplingIndices.add(fiveMinuteDownsampleIndex); } - if (indicesCreated(event).contains(tenSecondsDownsampleIndex)) { - witnessedDownsamplingIndices.add(tenSecondsDownsampleIndex); + if (indicesCreated(event).contains(tenMinuteDownsampleIndex)) { + witnessedDownsamplingIndices.add(tenMinuteDownsampleIndex); } }); // before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with - // testing so DSL doesn't have to wait for the end_time to lapse) + // testing, so DSL doesn't have to wait for the end_time to lapse) putTSDBIndexTemplate(dataStreamName, null, null, lifecycle); client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet(); assertBusy(() -> { assertThat(witnessedDownsamplingIndices.size(), is(1)); // only the ten seconds downsample round should've been executed - assertThat(witnessedDownsamplingIndices.contains(tenSecondsDownsampleIndex), is(true)); + assertThat(witnessedDownsamplingIndices.contains(tenMinuteDownsampleIndex), is(true)); }, 30, TimeUnit.SECONDS); assertBusy(() -> { @@ -171,8 +171,9 @@ public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception assertThat(dsBackingIndices.size(), is(2)); String writeIndex = dsBackingIndices.get(1); assertThat(writeIndex, backingIndexEqualTo(dataStreamName, 2)); - assertThat(dsBackingIndices.get(0), is(tenSecondsDownsampleIndex)); + assertThat(dsBackingIndices.get(0), is(tenMinuteDownsampleIndex)); }, 30, TimeUnit.SECONDS); + assertDownsamplingMethod(downsamplingMethod, tenMinuteDownsampleIndex); } @TestLogging(value = "org.elasticsearch.datastreams.lifecycle:TRACE", reason = "debugging") @@ -181,19 +182,15 @@ public void testUpdateDownsampleRound() throws Exception { // we expect the earlier round to be ignored String dataStreamName = "metrics-baz"; + DownsampleConfig.SamplingMethod downsamplingMethod = randomSamplingMethod(); DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingMethod(downsamplingMethod) + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), // data stream lifecycle runs every 1 second, so by the time we forcemerge the backing index it would've been at - // least 2 seconds since rollover. only the 10 seconds round should be executed. - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(10), - new DownsampleConfig(new DateHistogramInterval("10m")) - ) + // least 2 seconds since rollover. Only the 10 seconds round should be executed. + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(10), new DateHistogramInterval("10m")) ) ) .buildTemplate(); @@ -243,15 +240,10 @@ public void testUpdateDownsampleRound() throws Exception { // update the lifecycle so that it only has one round, for the same `after` parameter as before, but a different interval // the different interval should yield a different downsample index name so we expect the data stream lifecycle to get the previous - // `10s` interval downsample index, downsample it to `20m` and replace it in the data stream instead of the `10s` one. + // `10m` interval downsample index, downsample it to `20m` and replace it in the data stream instead of the `10m` one. DataStreamLifecycle updatedLifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( - List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(10), - new DownsampleConfig(new DateHistogramInterval("20m")) - ) - ) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(10), new DateHistogramInterval("20m"))) ) .build(); assertAcked( @@ -277,5 +269,111 @@ public void testUpdateDownsampleRound() throws Exception { assertThat(writeIndex, backingIndexEqualTo(dataStreamName, 2)); assertThat(dsBackingIndices.get(0), is(thirtySecondsDownsampleIndex)); }, 30, TimeUnit.SECONDS); + assertDownsamplingMethod(downsamplingMethod, thirtySecondsDownsampleIndex); + } + + /** + * This test ensures that when we change the sampling method, the already downsampled indices will use the original sampling method, + * while the raw data ones will be downsampled with the most recent configuration. + * To achieve that, we set the following test: + * 1. Create a data stream that is downsampled with a sampling method. + * 2. Rollover and wait for the downsampling to occur + * 3. Double the downsample interval (so it can downsample the first index as well) and change the sampling method. + * 4. Rollover and wait for both indices to be downsampled with the new interval + * 5. Check that the indices have been downsampled with the correct method. + */ + public void testUpdateDownsampleSamplingMode() throws Exception { + String dataStreamName = "metrics-baz"; + DownsampleConfig.SamplingMethod initialSamplingMethod = randomBoolean() + ? null + : randomFrom(DownsampleConfig.SamplingMethod.values()); + DownsampleConfig.SamplingMethod updatedSamplingMethod = initialSamplingMethod == DownsampleConfig.SamplingMethod.LAST_VALUE + ? (randomBoolean() ? null : DownsampleConfig.SamplingMethod.AGGREGATE) + : DownsampleConfig.SamplingMethod.LAST_VALUE; + + DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingMethod(initialSamplingMethod) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(10), new DateHistogramInterval("5m"))) + ) + .buildTemplate(); + + // Start and end time there just to ease with testing, so DLM doesn't have to wait for the end_time to lapse + // Creating the first backing index. + setupTSDBDataStreamAndIngestDocs( + dataStreamName, + "1986-01-08T23:40:53.384Z", + "2022-01-08T23:40:53.384Z", + lifecycle, + DOC_COUNT, + "1990-09-09T18:00:00" + ); + + // before we roll over, we update the index template to have new start/end time boundaries + // Creating the second backing index. + putTSDBIndexTemplate(dataStreamName, "2022-01-08T23:40:53.384Z", "2023-01-08T23:40:53.384Z", lifecycle); + RolloverResponse rolloverResponse = safeGet(client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null))); + assertTrue(rolloverResponse.isRolledOver()); + String firstBackingIndex = rolloverResponse.getOldIndex(); + String secondBackingIndex = rolloverResponse.getNewIndex(); + indexDocuments(dataStreamName, randomIntBetween(1, 1000), "2022-01-08T23:50:00"); + + // Ensure that the first backing index has been downsampled + awaitClusterState(clusterState -> { + final var dataStream = clusterState.metadata().getProject().dataStreams().get(dataStreamName); + if (dataStream == null) { + return false; + } + return dataStream.getIndices().size() > 1 && dataStream.getIndices().getFirst().getName().startsWith("downsample-"); + }); + assertDownsamplingMethod(initialSamplingMethod, "downsample-5m-" + firstBackingIndex); + // We change the sampling method, but also we double the downsampling interval. We expect the data stream lifecycle to get the + // previous `5m` interval downsampled index, downsample it to `10m` and replace it in the data stream with the `5m` one. + DataStreamLifecycle updatedLifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingMethod(updatedSamplingMethod) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(5), new DateHistogramInterval("10m"))) + ) + .build(); + assertAcked( + client().execute( + PutDataStreamLifecycleAction.INSTANCE, + new PutDataStreamLifecycleAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + new String[] { dataStreamName }, + updatedLifecycle + ) + ) + ); + + // We roll over one more time, so the second backing index will be eligible for downsampling + putTSDBIndexTemplate(dataStreamName, null, null, lifecycle); + rolloverResponse = safeGet(client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null))); + assertTrue(rolloverResponse.isRolledOver()); + String downsampledPrefix = "downsample-10m-"; + final var waitForUpdatedDownsamplingRound = ClusterServiceUtils.addMasterTemporaryStateListener(clusterState -> { + ProjectMetadata projectMetadata = clusterState.metadata().getProject(); + final var dataStream = projectMetadata.dataStreams().get(dataStreamName); + if (dataStream == null) { + return false; + } + + return dataStream.getIndices().size() > 2 + && dataStream.getIndices().stream().filter(index -> index.getName().startsWith(downsampledPrefix)).count() == 2; + }); + safeAwait(waitForUpdatedDownsamplingRound); + assertDownsamplingMethod(initialSamplingMethod, downsampledPrefix + firstBackingIndex); + assertDownsamplingMethod(updatedSamplingMethod, downsampledPrefix + secondBackingIndex); + } + + private void assertDownsamplingMethod(DownsampleConfig.SamplingMethod downsamplingMethod, String... indexNames) { + String expected = DownsampleConfig.SamplingMethod.getOrDefault(downsamplingMethod).toString(); + GetSettingsResponse response = safeGet( + client().admin().indices().getSettings(new GetSettingsRequest(TimeValue.THIRTY_SECONDS).indices(indexNames)) + ); + for (String indexName : indexNames) { + assertThat(response.getSetting(indexName, IndexMetadata.INDEX_DOWNSAMPLE_METHOD_KEY), equalTo(expected)); + } } } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index 50589bd80946d..b8e03a295e75d 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -868,15 +868,16 @@ private static void validateDownsamplingConfiguration( Map meta = timestampFieldType.meta(); if (meta.isEmpty() == false) { - String interval = meta.get(config.getIntervalType()); - DownsampleConfig.SamplingMethod sourceSamplingMethod = DownsampleConfig.SamplingMethod.fromIndexMetadata(sourceIndexMetadata); - if (interval != null) { + String sourceInterval = meta.get(config.getIntervalType()); + if (sourceInterval != null) { try { - DownsampleConfig sourceConfig = new DownsampleConfig(new DateHistogramInterval(interval), sourceSamplingMethod); - DownsampleConfig.validateSourceAndTargetIntervals(sourceConfig, config); + DownsampleConfig.validateSourceAndTargetIntervals(new DateHistogramInterval(sourceInterval), config.getFixedInterval()); } catch (IllegalArgumentException exception) { e.addValidationError("Source index is a downsampled index. " + exception.getMessage()); } + DownsampleConfig.SamplingMethod sourceSamplingMethod = DownsampleConfig.SamplingMethod.fromIndexMetadata( + sourceIndexMetadata + ); if (Objects.equals(sourceSamplingMethod, config.getSamplingMethodOrDefault()) == false) { e.addValidationError( "Source index is a downsampled index. Downsampling method [" diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutorTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutorTests.java index 8f0254cdf586b..40bcfdac5b00a 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutorTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutorTests.java @@ -157,7 +157,7 @@ public void testGetStatelessAssignment() { .build(); var params = new DownsampleShardTaskParams( - new DownsampleConfig(new DateHistogramInterval("1h")), + new DownsampleConfig(new DateHistogramInterval("1h"), randomSamplingMethod()), shardId.getIndexName(), 1, 1, diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java index 1327db4bc0bea..80ae710a3c16c 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java @@ -37,6 +37,7 @@ import javax.lang.model.util.Elements; import static java.util.stream.Collectors.joining; +import static org.elasticsearch.compute.gen.Methods.getMethod; import static org.elasticsearch.compute.gen.Methods.optionalStaticMethod; import static org.elasticsearch.compute.gen.Methods.requireAnyArgs; import static org.elasticsearch.compute.gen.Methods.requireAnyType; @@ -53,7 +54,6 @@ import static org.elasticsearch.compute.gen.Types.BLOCK; import static org.elasticsearch.compute.gen.Types.BLOCK_ARRAY; import static org.elasticsearch.compute.gen.Types.BOOLEAN_VECTOR; -import static org.elasticsearch.compute.gen.Types.BYTES_REF; import static org.elasticsearch.compute.gen.Types.DRIVER_CONTEXT; import static org.elasticsearch.compute.gen.Types.ELEMENT_TYPE; import static org.elasticsearch.compute.gen.Types.INTERMEDIATE_STATE_DESC; @@ -62,6 +62,8 @@ import static org.elasticsearch.compute.gen.Types.PAGE; import static org.elasticsearch.compute.gen.Types.WARNINGS; import static org.elasticsearch.compute.gen.Types.blockType; +import static org.elasticsearch.compute.gen.Types.fromString; +import static org.elasticsearch.compute.gen.Types.scratchType; import static org.elasticsearch.compute.gen.Types.vectorType; /** @@ -85,7 +87,7 @@ public class AggregatorImplementer { private final AggregationState aggState; private final List aggParams; - private final boolean hasOnlyBlockArguments; + private final boolean tryToUseVectors; public AggregatorImplementer( Elements elements, @@ -119,7 +121,8 @@ public AggregatorImplementer( return a; }).filter(a -> a instanceof PositionArgument == false).toList(); - this.hasOnlyBlockArguments = this.aggParams.stream().allMatch(a -> a instanceof BlockArgument); + this.tryToUseVectors = aggParams.stream().anyMatch(a -> (a instanceof BlockArgument) == false) + && aggParams.stream().noneMatch(a -> a.supportsVectorReadAccess() == false); this.createParameters = init.getParameters() .stream() @@ -199,7 +202,7 @@ private TypeSpec type() { builder.addMethod(addRawInput()); builder.addMethod(addRawInputExploded(true)); builder.addMethod(addRawInputExploded(false)); - if (hasOnlyBlockArguments == false) { + if (tryToUseVectors) { builder.addMethod(addRawVector(false)); builder.addMethod(addRawVector(true)); } @@ -340,16 +343,18 @@ private MethodSpec addRawInputExploded(boolean hasMask) { builder.addStatement("$T $L = page.getBlock(channels.get($L))", a.dataType(true), a.blockName(), i); } - for (Argument a : aggParams) { - String rawBlock = "addRawBlock(" - + aggParams.stream().map(arg -> arg.blockName()).collect(joining(", ")) - + (hasMask ? ", mask" : "") - + ")"; + if (tryToUseVectors) { + for (Argument a : aggParams) { + String rawBlock = "addRawBlock(" + + aggParams.stream().map(arg -> arg.blockName()).collect(joining(", ")) + + (hasMask ? ", mask" : "") + + ")"; - a.resolveVectors(builder, rawBlock, "return"); + a.resolveVectors(builder, rawBlock, "return"); + } } - builder.addStatement(invokeAddRaw(hasOnlyBlockArguments, hasMask)); + builder.addStatement(invokeAddRaw(tryToUseVectors == false, hasMask)); return builder.build(); } @@ -499,9 +504,9 @@ private MethodSpec.Builder initAddRaw(boolean blockStyle, boolean masked) { builder.addParameter(BOOLEAN_VECTOR, "mask"); } for (Argument a : aggParams) { - if (a.isBytesRef()) { - // Add bytes_ref scratch var that will be used for bytes_ref blocks/vectors - builder.addStatement("$T $L = new $T()", BYTES_REF, a.scratchName(), BYTES_REF); + if (a.scratchType() != null) { + // Add scratch var that will be used for some blocks/vectors, e.g. for bytes_ref + builder.addStatement("$T $L = new $T()", a.scratchType(), a.scratchName(), a.scratchType()); } } return builder; @@ -610,8 +615,8 @@ private MethodSpec addIntermediateInput() { ).map(Methods::requireType).toArray(TypeMatcher[]::new) ) ); - if (intermediateState.stream().map(IntermediateStateDesc::elementType).anyMatch(n -> n.equals("BYTES_REF"))) { - builder.addStatement("$T scratch = new $T()", BYTES_REF, BYTES_REF); + for (IntermediateStateDesc interState : intermediateState) { + interState.addScratchDeclaration(builder); } builder.addStatement("$T.combineIntermediate(state, " + intermediateStateRowAccess() + ")", declarationType); } @@ -706,13 +711,25 @@ public String access(String position) { if (block) { return name(); } - String s = name() + "." + vectorAccessorName(elementType()) + "(" + position; - if (elementType().equals("BYTES_REF")) { - s += ", scratch"; + String s = name() + "."; + if (vectorType(elementType) != null) { + s += vectorAccessorName(elementType()) + "(" + position; + } else { + s += getMethod(fromString(elementType())) + "(" + name() + ".getFirstValueIndex(" + position + ")"; + } + if (scratchType(elementType()) != null) { + s += ", " + name() + "Scratch"; } return s + ")"; } + public void addScratchDeclaration(MethodSpec.Builder builder) { + ClassName scratchType = scratchType(elementType()); + if (scratchType != null) { + builder.addStatement("$T $L = new $T()", scratchType, name() + "Scratch", scratchType); + } + } + public void assignToVariable(MethodSpec.Builder builder, int offset) { builder.addStatement("Block $L = page.getBlock(channels.get($L))", name + "Uncast", offset); ClassName blockType = blockType(elementType()); @@ -721,7 +738,7 @@ public void assignToVariable(MethodSpec.Builder builder, int offset) { builder.addStatement("return"); builder.endControlFlow(); } - if (block) { + if (block || vectorType(elementType) == null) { builder.addStatement("$T $L = ($T) $L", blockType, name, blockType, name + "Uncast"); } else { builder.addStatement("$T $L = (($T) $L).asVector()", vectorType(elementType), name, blockType, name + "Uncast"); @@ -732,6 +749,7 @@ public TypeName combineArgType() { var type = Types.fromString(elementType); return block ? blockType(type) : type; } + } /** diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java index bba578f45f5b0..3b1b184cb02d0 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java @@ -51,6 +51,7 @@ public class EvaluatorImplementer { private final ProcessFunction processFunction; private final ClassName implementation; private final boolean processOutputsMultivalued; + private final boolean vectorsUnsupported; private final boolean allNullsIsNull; public EvaluatorImplementer( @@ -69,6 +70,8 @@ public EvaluatorImplementer( declarationType.getSimpleName() + extraName + "Evaluator" ); this.processOutputsMultivalued = this.processFunction.hasBlockType; + boolean anyParameterNotSupportingVectors = this.processFunction.args.stream().anyMatch(a -> a.supportsVectorReadAccess() == false); + vectorsUnsupported = processOutputsMultivalued || anyParameterNotSupportingVectors; this.allNullsIsNull = allNullsIsNull; } @@ -101,7 +104,7 @@ private TypeSpec type() { builder.addMethod(eval()); builder.addMethod(processFunction.baseRamBytesUsed()); - if (processOutputsMultivalued) { + if (vectorsUnsupported) { if (processFunction.args.stream().anyMatch(x -> x instanceof FixedArgument == false)) { builder.addMethod(realEval(true)); } @@ -145,7 +148,7 @@ private MethodSpec eval() { builder.addModifiers(Modifier.PUBLIC).returns(BLOCK).addParameter(PAGE, "page"); processFunction.args.forEach(a -> a.evalToBlock(builder)); String invokeBlockEval = invokeRealEval(true); - if (processOutputsMultivalued) { + if (vectorsUnsupported) { builder.addStatement(invokeBlockEval); } else { processFunction.args.forEach(a -> a.resolveVectors(builder, invokeBlockEval)); diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java index 399f45b5df667..76edfbb1f70e3 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java @@ -51,7 +51,6 @@ import static org.elasticsearch.compute.gen.Types.BIG_ARRAYS; import static org.elasticsearch.compute.gen.Types.BLOCK; import static org.elasticsearch.compute.gen.Types.BLOCK_ARRAY; -import static org.elasticsearch.compute.gen.Types.BYTES_REF; import static org.elasticsearch.compute.gen.Types.DRIVER_CONTEXT; import static org.elasticsearch.compute.gen.Types.ELEMENT_TYPE; import static org.elasticsearch.compute.gen.Types.GROUPING_AGGREGATOR_EVALUATOR_CONTEXT; @@ -93,6 +92,7 @@ public class GroupingAggregatorImplementer { private final AggregationState aggState; private final List aggParams; private final boolean hasOnlyBlockArguments; + private final boolean allArgumentsSupportVectors; public GroupingAggregatorImplementer( Elements elements, @@ -128,6 +128,7 @@ public GroupingAggregatorImplementer( }).filter(a -> a instanceof PositionArgument == false).toList(); this.hasOnlyBlockArguments = this.aggParams.stream().allMatch(a -> a instanceof BlockArgument); + this.allArgumentsSupportVectors = aggParams.stream().noneMatch(a -> a.supportsVectorReadAccess() == false); this.createParameters = init.getParameters() .stream() @@ -204,7 +205,7 @@ private TypeSpec type() { builder.addMethod(prepareProcessRawInputPage()); for (ClassName groupIdClass : GROUP_IDS_CLASSES) { builder.addMethod(addRawInputLoop(groupIdClass, false)); - if (hasOnlyBlockArguments == false) { + if (hasOnlyBlockArguments == false && allArgumentsSupportVectors) { builder.addMethod(addRawInputLoop(groupIdClass, true)); } builder.addMethod(addIntermediateInput(groupIdClass)); @@ -330,26 +331,31 @@ private MethodSpec prepareProcessRawInputPage() { builder.addStatement("$T $L = page.getBlock(channels.get($L))", a.dataType(true), a.blockName(), i); } - for (Argument a : aggParams) { - builder.addStatement( - "$T $L = $L.asVector()", - vectorType(a.elementType()), - (a instanceof BlockArgument) ? (a.name() + "Vector") : a.vectorName(), - a.blockName() - ); - builder.beginControlFlow("if ($L == null)", (a instanceof BlockArgument) ? (a.name() + "Vector") : a.vectorName()); - { + String groupIdTrackingStatement = "maybeEnableGroupIdTracking(seenGroupIds, " + + aggParams.stream().map(arg -> arg.blockName()).collect(joining(", ")) + + ")"; + + if (allArgumentsSupportVectors) { + + for (Argument a : aggParams) { builder.addStatement( - "maybeEnableGroupIdTracking(seenGroupIds, " - + aggParams.stream().map(arg -> arg.blockName()).collect(joining(", ")) - + ")" + "$T $L = $L.asVector()", + vectorType(a.elementType()), + (a instanceof BlockArgument) ? (a.name() + "Vector") : a.vectorName(), + a.blockName() ); - returnAddInput(builder, false); + builder.beginControlFlow("if ($L == null)", (a instanceof BlockArgument) ? (a.name() + "Vector") : a.vectorName()); + { + builder.addStatement(groupIdTrackingStatement); + returnAddInput(builder, false); + } + builder.endControlFlow(); } - builder.endControlFlow(); + returnAddInput(builder, true); + } else { + builder.addStatement(groupIdTrackingStatement); + returnAddInput(builder, false); } - - returnAddInput(builder, true); return builder.build(); } @@ -443,9 +449,9 @@ private MethodSpec addRawInputLoop(TypeName groupsType, boolean valuesAreVector) ); } for (Argument a : aggParams) { - if (a.isBytesRef()) { - // Add bytes_ref scratch var that will be used for bytes_ref blocks/vectors - builder.addStatement("$T $L = new $T()", BYTES_REF, a.scratchName(), BYTES_REF); + if (a.scratchType() != null) { + // Add scratch var that will be used for some blocks/vectors, e.g. for bytes_ref + builder.addStatement("$T $L = new $T()", a.scratchType(), a.scratchName(), a.scratchType()); } } @@ -645,11 +651,7 @@ private MethodSpec addIntermediateInput(TypeName groupsType) { .collect(Collectors.joining(", ")); builder.addStatement("$T.combineIntermediate(state, positionOffset, groups, " + states + ")", declarationType); } else { - if (intermediateState.stream() - .map(AggregatorImplementer.IntermediateStateDesc::elementType) - .anyMatch(n -> n.equals("BYTES_REF"))) { - builder.addStatement("$T scratch = new $T()", BYTES_REF, BYTES_REF); - } + intermediateState.forEach(state -> state.addScratchDeclaration(builder)); builder.beginControlFlow("for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++)"); { if (groupsIsBlock) { diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Methods.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Methods.java index 15520c6ed93ab..2a06777c62d6e 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Methods.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Methods.java @@ -314,6 +314,9 @@ public static String getMethod(TypeName elementType) { if (elementType.equals(TypeName.FLOAT)) { return "getFloat"; } + if (elementType.equals(Types.EXPONENTIAL_HISTOGRAM)) { + return "getExponentialHistogram"; + } throw new IllegalArgumentException("unknown get method for [" + elementType + "]"); } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java index 82b3450343ced..d31679665efb5 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java @@ -52,6 +52,8 @@ public class Types { public static final ClassName LONG_BLOCK = ClassName.get(DATA_PACKAGE, "LongBlock"); public static final ClassName DOUBLE_BLOCK = ClassName.get(DATA_PACKAGE, "DoubleBlock"); public static final ClassName FLOAT_BLOCK = ClassName.get(DATA_PACKAGE, "FloatBlock"); + public static final ClassName EXPONENTIAL_HISTOGRAM_BLOCK = ClassName.get(DATA_PACKAGE, "ExponentialHistogramBlock"); + public static final ClassName EXPONENTIAL_HISTOGRAM_SCRATCH = ClassName.get(DATA_PACKAGE, "ExponentialHistogramScratch"); static final ClassName BOOLEAN_BLOCK_BUILDER = BOOLEAN_BLOCK.nestedClass("Builder"); static final ClassName BYTES_REF_BLOCK_BUILDER = BYTES_REF_BLOCK.nestedClass("Builder"); @@ -59,6 +61,7 @@ public class Types { static final ClassName LONG_BLOCK_BUILDER = LONG_BLOCK.nestedClass("Builder"); static final ClassName DOUBLE_BLOCK_BUILDER = DOUBLE_BLOCK.nestedClass("Builder"); static final ClassName FLOAT_BLOCK_BUILDER = FLOAT_BLOCK.nestedClass("Builder"); + static final ClassName EXPONENTIAL_HISTOGRAM_BLOCK_BUILDER = ClassName.get(DATA_PACKAGE, "ExponentialHistogramBlockBuilder"); static final ClassName ELEMENT_TYPE = ClassName.get(DATA_PACKAGE, "ElementType"); @@ -133,24 +136,32 @@ public class Types { static final ClassName SOURCE = ClassName.get("org.elasticsearch.xpack.esql.core.tree", "Source"); public static final ClassName BYTES_REF = ClassName.get("org.apache.lucene.util", "BytesRef"); + public static final ClassName EXPONENTIAL_HISTOGRAM = ClassName.get("org.elasticsearch.exponentialhistogram", "ExponentialHistogram"); public static final ClassName RELEASABLE = ClassName.get("org.elasticsearch.core", "Releasable"); public static final ClassName RELEASABLES = ClassName.get("org.elasticsearch.core", "Releasables"); - private record TypeDef(TypeName type, String alias, ClassName block, ClassName vector) { + private record TypeDef(TypeName type, String alias, ClassName block, ClassName vector, ClassName scratch) { - public static TypeDef of(TypeName type, String alias, String block, String vector) { - return new TypeDef(type, alias, ClassName.get(DATA_PACKAGE, block), ClassName.get(DATA_PACKAGE, vector)); + public static TypeDef of(TypeName type, String alias, String block, String vector, ClassName scratch) { + return new TypeDef( + type, + alias, + ClassName.get(DATA_PACKAGE, block), + vector == null ? null : ClassName.get(DATA_PACKAGE, vector), + scratch + ); } } private static final Map TYPES = Stream.of( - TypeDef.of(TypeName.BOOLEAN, "BOOLEAN", "BooleanBlock", "BooleanVector"), - TypeDef.of(TypeName.INT, "INT", "IntBlock", "IntVector"), - TypeDef.of(TypeName.LONG, "LONG", "LongBlock", "LongVector"), - TypeDef.of(TypeName.FLOAT, "FLOAT", "FloatBlock", "FloatVector"), - TypeDef.of(TypeName.DOUBLE, "DOUBLE", "DoubleBlock", "DoubleVector"), - TypeDef.of(BYTES_REF, "BYTES_REF", "BytesRefBlock", "BytesRefVector") + TypeDef.of(TypeName.BOOLEAN, "BOOLEAN", "BooleanBlock", "BooleanVector", null), + TypeDef.of(TypeName.INT, "INT", "IntBlock", "IntVector", null), + TypeDef.of(TypeName.LONG, "LONG", "LongBlock", "LongVector", null), + TypeDef.of(TypeName.FLOAT, "FLOAT", "FloatBlock", "FloatVector", null), + TypeDef.of(TypeName.DOUBLE, "DOUBLE", "DoubleBlock", "DoubleVector", null), + TypeDef.of(BYTES_REF, "BYTES_REF", "BytesRefBlock", "BytesRefVector", BYTES_REF), + TypeDef.of(EXPONENTIAL_HISTOGRAM, "EXPONENTIAL_HISTOGRAM", "ExponentialHistogramBlock", null, EXPONENTIAL_HISTOGRAM_SCRATCH) ) .flatMap(def -> Stream.of(def.type.toString(), def.type + "[]", def.alias).map(alias -> Map.entry(alias, def))) .collect(toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -183,6 +194,14 @@ static ClassName vectorType(String elementType) { return findRequired(elementType, "vector").vector; } + public static ClassName scratchType(String elementType) { + TypeDef typeDef = TYPES.get(elementType); + if (typeDef != null) { + return typeDef.scratch; + } + return null; + } + static ClassName builderType(TypeName resultType) { if (resultType.equals(BOOLEAN_BLOCK)) { return BOOLEAN_BLOCK_BUILDER; @@ -220,6 +239,9 @@ static ClassName builderType(TypeName resultType) { if (resultType.equals(FLOAT_VECTOR)) { return FLOAT_VECTOR_BUILDER; } + if (resultType.equals(EXPONENTIAL_HISTOGRAM_BLOCK)) { + return EXPONENTIAL_HISTOGRAM_BLOCK_BUILDER; + } throw new IllegalArgumentException("unknown builder type for [" + resultType + "]"); } @@ -261,6 +283,9 @@ public static TypeName elementType(TypeName t) { if (t.equals(FLOAT_BLOCK) || t.equals(FLOAT_VECTOR) || t.equals(FLOAT_BLOCK_BUILDER)) { return TypeName.FLOAT; } + if (t.equals(EXPONENTIAL_HISTOGRAM_BLOCK) || t.equals(EXPONENTIAL_HISTOGRAM_BLOCK_BUILDER)) { + return EXPONENTIAL_HISTOGRAM; + } throw new IllegalArgumentException("unknown element type for [" + t + "]"); } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/Argument.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/Argument.java index 4a2100a177e92..420989cfd0ddf 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/Argument.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/Argument.java @@ -17,7 +17,6 @@ import org.elasticsearch.compute.gen.Types; import java.util.List; -import java.util.Objects; import javax.lang.model.element.VariableElement; import javax.lang.model.type.ArrayType; @@ -25,7 +24,6 @@ import javax.lang.model.type.TypeMirror; import static org.elasticsearch.compute.gen.Methods.getMethod; -import static org.elasticsearch.compute.gen.Types.BYTES_REF; import static org.elasticsearch.compute.gen.Types.blockType; import static org.elasticsearch.compute.gen.Types.vectorType; import static org.elasticsearch.compute.gen.argument.StandardArgument.isBlockType; @@ -93,18 +91,18 @@ default String offsetName() { return name() + "Offset"; } + default ClassName scratchType() { + return Types.scratchType(type().toString()); + } + default String scratchName() { - if (isBytesRef() == false) { - throw new IllegalStateException("can't build scratch for non-BytesRef"); + if (scratchType() == null) { + throw new IllegalStateException("can't build scratch for " + type()); } return name() + "Scratch"; } - default boolean isBytesRef() { - return Objects.equals(type(), BYTES_REF); - } - String name(); TypeName type(); @@ -119,6 +117,13 @@ default TypeName elementType() { */ TypeName dataType(boolean blockStyle); + /** + * False if and only if there is a block backing this parameter and that block does not support access as a vector. Otherwise true. + */ + default boolean supportsVectorReadAccess() { + return true; + } + /** * The parameter passed to the real evaluation function */ @@ -198,7 +203,7 @@ default TypeName elementType() { */ default void read(MethodSpec.Builder builder, String accessor, String firstParam) { String params = firstParam; - if (isBytesRef()) { + if (scratchType() != null) { params += ", " + scratchName(); } builder.addStatement("$T $L = $L.$L($L)", type(), valueName(), accessor, getMethod(type()), params); diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/ArrayArgument.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/ArrayArgument.java index 0ee2bf1998a74..08dd5fa478f2a 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/ArrayArgument.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/ArrayArgument.java @@ -8,6 +8,7 @@ package org.elasticsearch.compute.gen.argument; import com.squareup.javapoet.ArrayTypeName; +import com.squareup.javapoet.ClassName; import com.squareup.javapoet.MethodSpec; import com.squareup.javapoet.TypeName; import com.squareup.javapoet.TypeSpec; @@ -18,7 +19,6 @@ import javax.lang.model.element.Modifier; import static org.elasticsearch.compute.gen.Methods.getMethod; -import static org.elasticsearch.compute.gen.Types.BYTES_REF; import static org.elasticsearch.compute.gen.Types.EXPRESSION_EVALUATOR; import static org.elasticsearch.compute.gen.Types.EXPRESSION_EVALUATOR_FACTORY; import static org.elasticsearch.compute.gen.Types.RELEASABLE; @@ -35,6 +35,11 @@ public TypeName dataType(boolean blockStyle) { return ArrayTypeName.of(vectorType(type)); } + @Override + public boolean supportsVectorReadAccess() { + return vectorType(type) != null; + } + @Override public String paramName(boolean blockStyle) { return name + (blockStyle ? "Block" : "Vector") + "s"; @@ -105,10 +110,11 @@ public void resolveVectors(MethodSpec.Builder builder, String... invokeBlockEval @Override public void createScratch(MethodSpec.Builder builder) { builder.addStatement("$T[] $LValues = new $T[$L.length]", type, name, type, name); - if (isBytesRef()) { - builder.addStatement("$T[] $LScratch = new $T[$L.length]", type, name, type, name); + ClassName scratchType = scratchType(); + if (scratchType != null) { + builder.addStatement("$T[] $LScratch = new $T[$L.length]", scratchType, name, scratchType, name); builder.beginControlFlow("for (int i = 0; i < $L.length; i++)", name); - builder.addStatement("$LScratch[i] = new $T()", name, BYTES_REF); + builder.addStatement("$LScratch[i] = new $T()", name, scratchType); builder.endControlFlow(); } } @@ -136,8 +142,8 @@ public void read(MethodSpec.Builder builder, boolean blockStyle) { } else { lookupVar = "p"; } - if (isBytesRef()) { - builder.addStatement("$LValues[i] = $L[i].getBytesRef($L, $LScratch[i])", name, paramName(blockStyle), lookupVar, name); + if (scratchType() != null) { + builder.addStatement("$LValues[i] = $L[i].$L($L, $LScratch[i])", name, paramName(blockStyle), getMethod(type), lookupVar, name); } else { builder.addStatement("$LValues[i] = $L[i].$L($L)", name, paramName(blockStyle), getMethod(type), lookupVar); } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/StandardArgument.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/StandardArgument.java index 981e3de3f22f4..575c40a766ec1 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/StandardArgument.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/StandardArgument.java @@ -17,7 +17,6 @@ import static org.elasticsearch.compute.gen.Methods.getMethod; import static org.elasticsearch.compute.gen.Types.BOOLEAN_BLOCK; -import static org.elasticsearch.compute.gen.Types.BYTES_REF; import static org.elasticsearch.compute.gen.Types.BYTES_REF_BLOCK; import static org.elasticsearch.compute.gen.Types.DOUBLE_BLOCK; import static org.elasticsearch.compute.gen.Types.EXPRESSION_EVALUATOR; @@ -36,6 +35,11 @@ public TypeName dataType(boolean blockStyle) { return vectorType(type); } + @Override + public boolean supportsVectorReadAccess() { + return dataType(false) != null; + } + @Override public String paramName(boolean blockStyle) { return name + (blockStyle ? "Block" : "Vector"); @@ -93,8 +97,8 @@ public void resolveVectors(MethodSpec.Builder builder, String... invokeBlockEval @Override public void createScratch(MethodSpec.Builder builder) { - if (isBytesRef()) { - builder.addStatement("$T $LScratch = new $T()", BYTES_REF, name, BYTES_REF); + if (scratchType() != null) { + builder.addStatement("$T $LScratch = new $T()", scratchType(), name, scratchType()); } } @@ -123,7 +127,7 @@ static boolean isBlockType(TypeName type) { @Override public void read(MethodSpec.Builder builder, boolean blockStyle) { String params = blockStyle ? paramName(true) + ".getFirstValueIndex(p)" : "p"; - if (isBytesRef()) { + if (scratchType() != null) { params += ", " + scratchName(); } builder.addStatement("$T $L = $L.$L($L)", type, name, paramName(blockStyle), getMethod(type), params); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefAggregatorFunction.java index 71a3ec03278eb..b9737cc15976a 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefAggregatorFunction.java @@ -151,8 +151,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); assert hll.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - CountDistinctBytesRefAggregator.combineIntermediate(state, hll.getBytesRef(0, scratch)); + BytesRef hllScratch = new BytesRef(); + CountDistinctBytesRefAggregator.combineIntermediate(state, hll.getBytesRef(0, hllScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefGroupingAggregatorFunction.java index 1abea90e89ee3..8c11792fa4f4e 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefGroupingAggregatorFunction.java @@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -168,7 +168,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctBytesRefAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctBytesRefAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -223,7 +223,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -233,7 +233,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctBytesRefAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctBytesRefAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -274,11 +274,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - CountDistinctBytesRefAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctBytesRefAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleAggregatorFunction.java index 68d7270a49d0b..c791c3fd11b7a 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleAggregatorFunction.java @@ -149,8 +149,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); assert hll.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - CountDistinctDoubleAggregator.combineIntermediate(state, hll.getBytesRef(0, scratch)); + BytesRef hllScratch = new BytesRef(); + CountDistinctDoubleAggregator.combineIntermediate(state, hll.getBytesRef(0, hllScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleGroupingAggregatorFunction.java index c839f787c991f..3520b9f53f4c9 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleGroupingAggregatorFunction.java @@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -168,7 +168,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctDoubleAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctDoubleAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -221,7 +221,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -231,7 +231,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctDoubleAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctDoubleAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -270,11 +270,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - CountDistinctDoubleAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctDoubleAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java index c01c34887cf4e..06068bf343978 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java @@ -149,8 +149,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); assert hll.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - CountDistinctFloatAggregator.combineIntermediate(state, hll.getBytesRef(0, scratch)); + BytesRef hllScratch = new BytesRef(); + CountDistinctFloatAggregator.combineIntermediate(state, hll.getBytesRef(0, hllScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java index a8f284f7e8fdc..4a000e848c73b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java @@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -168,7 +168,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctFloatAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctFloatAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -221,7 +221,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -231,7 +231,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctFloatAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctFloatAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -270,11 +270,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - CountDistinctFloatAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctFloatAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntAggregatorFunction.java index 25a82171c54e4..1d378d7d8bb06 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntAggregatorFunction.java @@ -149,8 +149,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); assert hll.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - CountDistinctIntAggregator.combineIntermediate(state, hll.getBytesRef(0, scratch)); + BytesRef hllScratch = new BytesRef(); + CountDistinctIntAggregator.combineIntermediate(state, hll.getBytesRef(0, hllScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntGroupingAggregatorFunction.java index ad7e329d8ea7d..5320bff130fa5 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntGroupingAggregatorFunction.java @@ -157,7 +157,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -167,7 +167,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctIntAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctIntAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -220,7 +220,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -230,7 +230,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctIntAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctIntAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -269,11 +269,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - CountDistinctIntAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctIntAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongAggregatorFunction.java index 34c5b2ee2bda5..88ddb7261f378 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongAggregatorFunction.java @@ -149,8 +149,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); assert hll.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - CountDistinctLongAggregator.combineIntermediate(state, hll.getBytesRef(0, scratch)); + BytesRef hllScratch = new BytesRef(); + CountDistinctLongAggregator.combineIntermediate(state, hll.getBytesRef(0, hllScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongGroupingAggregatorFunction.java index 8bdff2e73ebb9..5a76c5440615c 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongGroupingAggregatorFunction.java @@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -168,7 +168,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctLongAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctLongAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -221,7 +221,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -231,7 +231,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctLongAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctLongAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -270,11 +270,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - CountDistinctLongAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctLongAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstBytesRefByTimestampAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstBytesRefByTimestampAggregatorFunction.java index eb88ed39323d2..91822de9973aa 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstBytesRefByTimestampAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstBytesRefByTimestampAggregatorFunction.java @@ -237,8 +237,8 @@ public void addIntermediateInput(Page page) { } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert seen.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - FirstBytesRefByTimestampAggregator.combineIntermediate(state, timestamps.getLong(0), values.getBytesRef(0, scratch), seen.getBoolean(0)); + BytesRef valuesScratch = new BytesRef(); + FirstBytesRefByTimestampAggregator.combineIntermediate(state, timestamps.getLong(0), values.getBytesRef(0, valuesScratch), seen.getBoolean(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstBytesRefByTimestampGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstBytesRefByTimestampGroupingAggregatorFunction.java index aee8d6296dc02..42b016f4580d8 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstBytesRefByTimestampGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstBytesRefByTimestampGroupingAggregatorFunction.java @@ -200,7 +200,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page } BytesRefBlock values = (BytesRefBlock) valuesUncast; assert timestamps.getPositionCount() == values.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef valuesScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -282,7 +282,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa } BytesRefBlock values = (BytesRefBlock) valuesUncast; assert timestamps.getPositionCount() == values.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef valuesScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -350,7 +350,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page } BytesRefBlock values = (BytesRefBlock) valuesUncast; assert timestamps.getPositionCount() == values.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef valuesScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/LastBytesRefByTimestampAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/LastBytesRefByTimestampAggregatorFunction.java index 0233a00ccd45f..70b05e5ae2910 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/LastBytesRefByTimestampAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/LastBytesRefByTimestampAggregatorFunction.java @@ -237,8 +237,8 @@ public void addIntermediateInput(Page page) { } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert seen.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - LastBytesRefByTimestampAggregator.combineIntermediate(state, timestamps.getLong(0), values.getBytesRef(0, scratch), seen.getBoolean(0)); + BytesRef valuesScratch = new BytesRef(); + LastBytesRefByTimestampAggregator.combineIntermediate(state, timestamps.getLong(0), values.getBytesRef(0, valuesScratch), seen.getBoolean(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/LastBytesRefByTimestampGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/LastBytesRefByTimestampGroupingAggregatorFunction.java index c5ee2afaf04c7..4ee67fdbdb818 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/LastBytesRefByTimestampGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/LastBytesRefByTimestampGroupingAggregatorFunction.java @@ -200,7 +200,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page } BytesRefBlock values = (BytesRefBlock) valuesUncast; assert timestamps.getPositionCount() == values.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef valuesScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -282,7 +282,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa } BytesRefBlock values = (BytesRefBlock) valuesUncast; assert timestamps.getPositionCount() == values.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef valuesScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -350,7 +350,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page } BytesRefBlock values = (BytesRefBlock) valuesUncast; assert timestamps.getPositionCount() == values.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef valuesScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java index 80810713866ce..e606864ba63f4 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java @@ -156,8 +156,8 @@ public void addIntermediateInput(Page page) { } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert seen.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - MaxBytesRefAggregator.combineIntermediate(state, max.getBytesRef(0, scratch), seen.getBoolean(0)); + BytesRef maxScratch = new BytesRef(); + MaxBytesRefAggregator.combineIntermediate(state, max.getBytesRef(0, maxScratch), seen.getBoolean(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunction.java index 793570db726a9..ab162190fa43d 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunction.java @@ -164,7 +164,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert max.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef maxScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -174,7 +174,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MaxBytesRefAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MaxBytesRefAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, maxScratch), seen.getBoolean(valuesPosition)); } } } @@ -236,7 +236,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert max.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef maxScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -246,7 +246,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MaxBytesRefAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MaxBytesRefAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, maxScratch), seen.getBoolean(valuesPosition)); } } } @@ -293,11 +293,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert max.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef maxScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - MaxBytesRefAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MaxBytesRefAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, maxScratch), seen.getBoolean(valuesPosition)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunction.java index d192718d14e10..119cdb5917c06 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunction.java @@ -156,8 +156,8 @@ public void addIntermediateInput(Page page) { } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert seen.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - MaxIpAggregator.combineIntermediate(state, max.getBytesRef(0, scratch), seen.getBoolean(0)); + BytesRef maxScratch = new BytesRef(); + MaxIpAggregator.combineIntermediate(state, max.getBytesRef(0, maxScratch), seen.getBoolean(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpGroupingAggregatorFunction.java index bfee9d26f3996..5de02faac858a 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpGroupingAggregatorFunction.java @@ -164,7 +164,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert max.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef maxScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -174,7 +174,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MaxIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MaxIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, maxScratch), seen.getBoolean(valuesPosition)); } } } @@ -236,7 +236,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert max.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef maxScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -246,7 +246,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MaxIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MaxIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, maxScratch), seen.getBoolean(valuesPosition)); } } } @@ -293,11 +293,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert max.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef maxScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - MaxIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MaxIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, maxScratch), seen.getBoolean(valuesPosition)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleAggregatorFunction.java index 6c50628613ae1..43b4170f68d53 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleAggregatorFunction.java @@ -146,8 +146,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); assert quart.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - MedianAbsoluteDeviationDoubleAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + BytesRef quartScratch = new BytesRef(); + MedianAbsoluteDeviationDoubleAggregator.combineIntermediate(state, quart.getBytesRef(0, quartScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunction.java index 3fd6a54970645..27531cb01ea14 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunction.java @@ -155,7 +155,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -165,7 +165,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -218,7 +218,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -228,7 +228,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -267,11 +267,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java index 7efedf93a8910..27a580e1da6f7 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java @@ -146,8 +146,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); assert quart.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + BytesRef quartScratch = new BytesRef(); + MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, quart.getBytesRef(0, quartScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java index c706d6a68b9ad..2f57442e14ea6 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java @@ -155,7 +155,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -165,7 +165,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -218,7 +218,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -228,7 +228,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -267,11 +267,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntAggregatorFunction.java index 73756681f5c18..68c9d8ce75174 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntAggregatorFunction.java @@ -146,8 +146,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); assert quart.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - MedianAbsoluteDeviationIntAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + BytesRef quartScratch = new BytesRef(); + MedianAbsoluteDeviationIntAggregator.combineIntermediate(state, quart.getBytesRef(0, quartScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntGroupingAggregatorFunction.java index 171e4238f68c7..6f6e711157a80 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntGroupingAggregatorFunction.java @@ -154,7 +154,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -164,7 +164,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -217,7 +217,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -227,7 +227,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -266,11 +266,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongAggregatorFunction.java index 4c6afaba77234..8f71e4cc249b9 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongAggregatorFunction.java @@ -146,8 +146,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); assert quart.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - MedianAbsoluteDeviationLongAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + BytesRef quartScratch = new BytesRef(); + MedianAbsoluteDeviationLongAggregator.combineIntermediate(state, quart.getBytesRef(0, quartScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongGroupingAggregatorFunction.java index 2136c8d0fd1fd..3a75aa3ba2e73 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongGroupingAggregatorFunction.java @@ -155,7 +155,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -165,7 +165,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -218,7 +218,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -228,7 +228,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -267,11 +267,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java index 92660be1185a0..a7ab1814d670d 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java @@ -156,8 +156,8 @@ public void addIntermediateInput(Page page) { } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert seen.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - MinBytesRefAggregator.combineIntermediate(state, min.getBytesRef(0, scratch), seen.getBoolean(0)); + BytesRef minScratch = new BytesRef(); + MinBytesRefAggregator.combineIntermediate(state, min.getBytesRef(0, minScratch), seen.getBoolean(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunction.java index e3acfe1ab1943..03e07ad54cb1f 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunction.java @@ -164,7 +164,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert min.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef minScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -174,7 +174,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MinBytesRefAggregator.combineIntermediate(state, groupId, min.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MinBytesRefAggregator.combineIntermediate(state, groupId, min.getBytesRef(valuesPosition, minScratch), seen.getBoolean(valuesPosition)); } } } @@ -236,7 +236,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert min.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef minScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -246,7 +246,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MinBytesRefAggregator.combineIntermediate(state, groupId, min.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MinBytesRefAggregator.combineIntermediate(state, groupId, min.getBytesRef(valuesPosition, minScratch), seen.getBoolean(valuesPosition)); } } } @@ -293,11 +293,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert min.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef minScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - MinBytesRefAggregator.combineIntermediate(state, groupId, min.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MinBytesRefAggregator.combineIntermediate(state, groupId, min.getBytesRef(valuesPosition, minScratch), seen.getBoolean(valuesPosition)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpAggregatorFunction.java index db61c405d440e..61898ca139071 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpAggregatorFunction.java @@ -156,8 +156,8 @@ public void addIntermediateInput(Page page) { } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert seen.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - MinIpAggregator.combineIntermediate(state, max.getBytesRef(0, scratch), seen.getBoolean(0)); + BytesRef maxScratch = new BytesRef(); + MinIpAggregator.combineIntermediate(state, max.getBytesRef(0, maxScratch), seen.getBoolean(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpGroupingAggregatorFunction.java index 69b3c227a2dab..ea678434b7dbb 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpGroupingAggregatorFunction.java @@ -164,7 +164,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert max.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef maxScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -174,7 +174,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MinIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MinIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, maxScratch), seen.getBoolean(valuesPosition)); } } } @@ -236,7 +236,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert max.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef maxScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -246,7 +246,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MinIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MinIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, maxScratch), seen.getBoolean(valuesPosition)); } } } @@ -293,11 +293,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert max.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef maxScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - MinIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MinIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, maxScratch), seen.getBoolean(valuesPosition)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleAggregatorFunction.java index 7ae73ed7e1eb9..48a54247588d3 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleAggregatorFunction.java @@ -149,8 +149,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); assert quart.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - PercentileDoubleAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + BytesRef quartScratch = new BytesRef(); + PercentileDoubleAggregator.combineIntermediate(state, quart.getBytesRef(0, quartScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleGroupingAggregatorFunction.java index f726423df4d4d..81e2d2189e133 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleGroupingAggregatorFunction.java @@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -168,7 +168,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - PercentileDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -221,7 +221,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -231,7 +231,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - PercentileDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -270,11 +270,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - PercentileDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java index 29acca005738d..5b172e43a7cfe 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java @@ -149,8 +149,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); assert quart.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - PercentileFloatAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + BytesRef quartScratch = new BytesRef(); + PercentileFloatAggregator.combineIntermediate(state, quart.getBytesRef(0, quartScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java index b5be9cc7655bd..c5cf407d86dc0 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java @@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -168,7 +168,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - PercentileFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -221,7 +221,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -231,7 +231,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - PercentileFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -270,11 +270,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - PercentileFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntAggregatorFunction.java index 21784c215f3f6..081120592aebd 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntAggregatorFunction.java @@ -149,8 +149,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); assert quart.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - PercentileIntAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + BytesRef quartScratch = new BytesRef(); + PercentileIntAggregator.combineIntermediate(state, quart.getBytesRef(0, quartScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntGroupingAggregatorFunction.java index c94cc37109e4e..a278b28d1d317 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntGroupingAggregatorFunction.java @@ -157,7 +157,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -167,7 +167,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - PercentileIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -220,7 +220,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -230,7 +230,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - PercentileIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -269,11 +269,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - PercentileIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongAggregatorFunction.java index df49223605ec2..6d81e228ca662 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongAggregatorFunction.java @@ -149,8 +149,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); assert quart.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - PercentileLongAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + BytesRef quartScratch = new BytesRef(); + PercentileLongAggregator.combineIntermediate(state, quart.getBytesRef(0, quartScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongGroupingAggregatorFunction.java index d48770a77abf2..2f22604be77c4 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongGroupingAggregatorFunction.java @@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -168,7 +168,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - PercentileLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -221,7 +221,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -231,7 +231,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - PercentileLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -270,11 +270,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - PercentileLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBooleanAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBooleanAggregatorFunction.java index a5cfcbb1e03e2..9ce0bffb4e642 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBooleanAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBooleanAggregatorFunction.java @@ -147,7 +147,7 @@ public void addIntermediateInput(Page page) { } BytesRefBlock sample = (BytesRefBlock) sampleUncast; assert sample.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); SampleBooleanAggregator.combineIntermediate(state, sample); } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBooleanGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBooleanGroupingAggregatorFunction.java index c675030a16bf2..c2d86ef1a9e54 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBooleanGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBooleanGroupingAggregatorFunction.java @@ -157,7 +157,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -220,7 +220,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -269,7 +269,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBytesRefAggregatorFunction.java index 45e55a2f9a5a3..dc3af45e9bec2 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBytesRefAggregatorFunction.java @@ -151,7 +151,7 @@ public void addIntermediateInput(Page page) { } BytesRefBlock sample = (BytesRefBlock) sampleUncast; assert sample.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); SampleBytesRefAggregator.combineIntermediate(state, sample); } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBytesRefGroupingAggregatorFunction.java index db0d675499c44..ae911a74c4ca4 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBytesRefGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBytesRefGroupingAggregatorFunction.java @@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -224,7 +224,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -275,7 +275,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleDoubleAggregatorFunction.java index fc5681e427572..e63c9fe414ce7 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleDoubleAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleDoubleAggregatorFunction.java @@ -148,7 +148,7 @@ public void addIntermediateInput(Page page) { } BytesRefBlock sample = (BytesRefBlock) sampleUncast; assert sample.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); SampleDoubleAggregator.combineIntermediate(state, sample); } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleDoubleGroupingAggregatorFunction.java index b81c56db60297..f62100ee2e0ec 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleDoubleGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleDoubleGroupingAggregatorFunction.java @@ -157,7 +157,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -220,7 +220,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -269,7 +269,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleIntAggregatorFunction.java index 232a664854401..b4035708a567b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleIntAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleIntAggregatorFunction.java @@ -148,7 +148,7 @@ public void addIntermediateInput(Page page) { } BytesRefBlock sample = (BytesRefBlock) sampleUncast; assert sample.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); SampleIntAggregator.combineIntermediate(state, sample); } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleIntGroupingAggregatorFunction.java index 41f2f7a608e8e..199676fa2fd2d 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleIntGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleIntGroupingAggregatorFunction.java @@ -156,7 +156,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -219,7 +219,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -268,7 +268,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleLongAggregatorFunction.java index df885bd2292db..b78d48381ef8e 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleLongAggregatorFunction.java @@ -148,7 +148,7 @@ public void addIntermediateInput(Page page) { } BytesRefBlock sample = (BytesRefBlock) sampleUncast; assert sample.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); SampleLongAggregator.combineIntermediate(state, sample); } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleLongGroupingAggregatorFunction.java index 675827ad0caa3..4310849a9ebb7 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleLongGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleLongGroupingAggregatorFunction.java @@ -157,7 +157,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -220,7 +220,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -269,7 +269,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefAggregatorFunction.java index c9aad2f067c4c..5042ba27a83a6 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefAggregatorFunction.java @@ -154,7 +154,7 @@ public void addIntermediateInput(Page page) { } BytesRefBlock top = (BytesRefBlock) topUncast; assert top.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); + BytesRef topScratch = new BytesRef(); TopBytesRefAggregator.combineIntermediate(state, top); } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefGroupingAggregatorFunction.java index dc6db83fc0f0d..676c70b480abe 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefGroupingAggregatorFunction.java @@ -162,7 +162,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefBlock top = (BytesRefBlock) topUncast; - BytesRef scratch = new BytesRef(); + BytesRef topScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -227,7 +227,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefBlock top = (BytesRefBlock) topUncast; - BytesRef scratch = new BytesRef(); + BytesRef topScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -278,7 +278,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefBlock top = (BytesRefBlock) topUncast; - BytesRef scratch = new BytesRef(); + BytesRef topScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunction.java index f5781c44f5e65..38c6e2c88c9f5 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunction.java @@ -154,7 +154,7 @@ public void addIntermediateInput(Page page) { } BytesRefBlock top = (BytesRefBlock) topUncast; assert top.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); + BytesRef topScratch = new BytesRef(); TopIpAggregator.combineIntermediate(state, top); } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunction.java index 18a6925ebbb34..a813402811302 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunction.java @@ -162,7 +162,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefBlock top = (BytesRefBlock) topUncast; - BytesRef scratch = new BytesRef(); + BytesRef topScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -227,7 +227,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefBlock top = (BytesRefBlock) topUncast; - BytesRef scratch = new BytesRef(); + BytesRef topScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -278,7 +278,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefBlock top = (BytesRefBlock) topUncast; - BytesRef scratch = new BytesRef(); + BytesRef topScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregatorFunction.java index 048467dddad48..d5da83e215f0c 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregatorFunction.java @@ -148,7 +148,7 @@ public void addIntermediateInput(Page page) { } BytesRefBlock values = (BytesRefBlock) valuesUncast; assert values.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); + BytesRef valuesScratch = new BytesRef(); ValuesBytesRefAggregator.combineIntermediate(state, values); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 369a5ea61d9e0..d056050f8fe97 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -17,7 +17,6 @@ import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.logging.log4j.core.config.plugins.util.PluginManager; -import org.apache.lucene.util.IOConsumer; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; @@ -31,7 +30,6 @@ import org.elasticsearch.inference.TaskType; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; -import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentType; @@ -45,7 +43,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.Semaphore; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -58,7 +55,6 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.reader; public class CsvTestsDataLoader { - private static final int PARALLEL_THREADS = 10; private static final int BULK_DATA_SIZE = 100_000; private static final TestDataset EMPLOYEES = new TestDataset("employees", "mapping-default.json", "employees.csv").noSubfields(); private static final TestDataset EMPLOYEES_INCOMPATIBLE = new TestDataset( @@ -433,42 +429,18 @@ private static void loadDataSetIntoEs( IndexCreator indexCreator ) throws IOException { Logger logger = LogManager.getLogger(CsvTestsDataLoader.class); - List datasets = availableDatasetsForEs( - supportsIndexModeLookup, - supportsSourceFieldMapping, - inferenceEnabled, - timeSeriesOnly - ).stream().toList(); - - logger.info("Creating test indices"); - executeInParallel(datasets, dataset -> createIndex(client, dataset, indexCreator), "Failed to create indices in parallel"); + Set loadedDatasets = new HashSet<>(); logger.info("Loading test datasets"); - executeInParallel(datasets, dataset -> loadData(client, dataset, logger), "Failed to load data in parallel"); - - forceMerge(client, datasets.stream().map(d -> d.indexName).collect(Collectors.toSet()), logger); - + for (var dataset : availableDatasetsForEs(supportsIndexModeLookup, supportsSourceFieldMapping, inferenceEnabled, timeSeriesOnly)) { + load(client, dataset, logger, indexCreator); + loadedDatasets.add(dataset.indexName); + } + forceMerge(client, loadedDatasets, logger); logger.info("Loading enrich policies"); - executeInParallel( - ENRICH_POLICIES, - policy -> loadEnrichPolicy(client, policy.policyName, policy.policyFileName, logger), - "Failed to load enrich policies in parallel" - ); - - } - - private static void executeInParallel(List items, IOConsumer consumer, String errorMessage) { - Semaphore semaphore = new Semaphore(PARALLEL_THREADS); - ESTestCase.runInParallel(items.size(), i -> { - try { - semaphore.acquire(); - consumer.accept(items.get(i)); - } catch (IOException | InterruptedException e) { - throw new RuntimeException(errorMessage, e); - } finally { - semaphore.release(); - } - }); + for (var policy : ENRICH_POLICIES) { + loadEnrichPolicy(client, policy.policyName, policy.policyFileName, logger); + } } public static void createInferenceEndpoints(RestClient client) throws IOException { @@ -626,14 +598,12 @@ private static URL getResource(String name) { return result; } - private static void createIndex(RestClient client, TestDataset dataset, IndexCreator indexCreator) throws IOException { + private static void load(RestClient client, TestDataset dataset, Logger logger, IndexCreator indexCreator) throws IOException { + logger.info("Loading dataset [{}] into ES index [{}]", dataset.dataFileName, dataset.indexName); URL mapping = getResource("/" + dataset.mappingFileName); Settings indexSettings = dataset.readSettingsFile(); indexCreator.createIndex(client, dataset.indexName, readMappingFile(mapping, dataset.typeMapping), indexSettings); - } - private static void loadData(RestClient client, TestDataset dataset, Logger logger) throws IOException { - logger.info("Loading dataset [{}] into ES index [{}]", dataset.dataFileName, dataset.indexName); // Some examples only test that the query and mappings are valid, and don't need example data. Use .noData() for those if (dataset.dataFileName != null) { URL data = getResource("/data/" + dataset.dataFileName); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/test/java/org/elasticsearch/xpack/esql/CsvTestsDataLoaderTests.java b/x-pack/plugin/esql/qa/testFixtures/src/test/java/org/elasticsearch/xpack/esql/CsvTestsDataLoaderTests.java index 612e007c4b13f..5b40e1d03e92f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/test/java/org/elasticsearch/xpack/esql/CsvTestsDataLoaderTests.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/test/java/org/elasticsearch/xpack/esql/CsvTestsDataLoaderTests.java @@ -11,18 +11,12 @@ import java.net.ConnectException; -import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.startsWith; public class CsvTestsDataLoaderTests extends ESTestCase { public void testCsvTestsDataLoaderExecution() { - Throwable cause = expectThrows(AssertionError.class, () -> CsvTestsDataLoader.main(new String[] {})); - // find the root cause - while (cause.getCause() != null) { - cause = cause.getCause(); - } - assertThat(cause, instanceOf(ConnectException.class)); - assertThat(cause.getMessage(), startsWith("Connection refused")); + ConnectException ce = expectThrows(ConnectException.class, () -> CsvTestsDataLoader.main(new String[] {})); + assertThat(ce.getMessage(), startsWith("Connection refused")); } } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/RemoteIndexResolutionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/RemoteIndexResolutionIT.java index d37f68494ebba..085f58d6c09b2 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/RemoteIndexResolutionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/RemoteIndexResolutionIT.java @@ -7,15 +7,18 @@ package org.elasticsearch.xpack.esql.plugin; -import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.MatchNoneQueryBuilder; import org.elasticsearch.transport.NoSuchRemoteClusterException; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.AbstractCrossClusterTestCase; +import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo.Cluster.Status; import org.elasticsearch.xpack.esql.action.EsqlQueryAction; import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; import org.elasticsearch.xpack.esql.action.EsqlQueryResponse; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; +import java.util.List; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -29,9 +32,14 @@ public class RemoteIndexResolutionIT extends AbstractCrossClusterTestCase { public void testResolvesRemoteIndex() { indexRandom(REMOTE_CLUSTER_1, true, "index-1", 1); - try (var response = run(syncEsqlQueryRequest().query("FROM " + REMOTE_CLUSTER_1 + ":index-1 METADATA _index"))) { + try ( + var response = run( + syncEsqlQueryRequest().query("FROM " + REMOTE_CLUSTER_1 + ":index-1 METADATA _index").includeCCSMetadata(true) + ) + ) { assertOk(response); assertResultConcreteIndices(response, REMOTE_CLUSTER_1 + ":index-1"); + assertExecutionInfo(response, new EsqlResponseExecutionInfo(REMOTE_CLUSTER_1, "index-1", Status.SUCCESSFUL)); } } @@ -40,6 +48,17 @@ public void testResolveRemoteUnknownIndex() { // This index is mixed into the resultset to test error handling of the missing concrete remote, not the empty result. indexRandom(LOCAL_CLUSTER, true, "data", 1); + expectThrows( + VerificationException.class, + containsString("Unknown index [" + REMOTE_CLUSTER_1 + ":fake]"), + () -> run(syncEsqlQueryRequest().query("FROM data," + REMOTE_CLUSTER_1 + ":fake")) + ); + expectThrows( + VerificationException.class, + containsString("Unknown index [" + REMOTE_CLUSTER_1 + ":fake]"), + () -> run(syncEsqlQueryRequest().query("FROM data," + REMOTE_CLUSTER_1 + ":fake").allowPartialResults(true)) + ); + setSkipUnavailable(REMOTE_CLUSTER_1, false); expectThrows( VerificationException.class, @@ -48,15 +67,33 @@ public void testResolveRemoteUnknownIndex() { ); setSkipUnavailable(REMOTE_CLUSTER_1, true); - try (var response = run(syncEsqlQueryRequest().query("FROM data," + REMOTE_CLUSTER_1 + ":fake METADATA _index"))) { + try ( + var response = run( + syncEsqlQueryRequest().query("FROM data," + REMOTE_CLUSTER_1 + ":fake METADATA _index").includeCCSMetadata(true) + ) + ) { assertPartial(response); assertResultConcreteIndices(response, "data"); + assertExecutionInfo( + response, + new EsqlResponseExecutionInfo(LOCAL_CLUSTER, "data", Status.SUCCESSFUL), + new EsqlResponseExecutionInfo(REMOTE_CLUSTER_1, "fake", Status.SKIPPED) + ); } setSkipUnavailable(REMOTE_CLUSTER_1, null); - try (var response = run(syncEsqlQueryRequest().query("FROM data," + REMOTE_CLUSTER_1 + ":fake METADATA _index"))) { + try ( + var response = run( + syncEsqlQueryRequest().query("FROM data," + REMOTE_CLUSTER_1 + ":fake METADATA _index").includeCCSMetadata(true) + ) + ) { assertPartial(response); assertResultConcreteIndices(response, "data"); + assertExecutionInfo( + response, + new EsqlResponseExecutionInfo(LOCAL_CLUSTER, "data", Status.SUCCESSFUL), + new EsqlResponseExecutionInfo(REMOTE_CLUSTER_1, "fake", Status.SKIPPED) + ); } } @@ -64,9 +101,18 @@ public void testResolvesLocalAndRemoteIndex() { indexRandom(LOCAL_CLUSTER, true, "index-1", 1); indexRandom(REMOTE_CLUSTER_1, true, "index-1", 1); - try (var response = run(syncEsqlQueryRequest().query("FROM index-1," + REMOTE_CLUSTER_1 + ":index-1 METADATA _index"))) { + try ( + var response = run( + syncEsqlQueryRequest().query("FROM index-1," + REMOTE_CLUSTER_1 + ":index-1 METADATA _index").includeCCSMetadata(true) + ) + ) { assertOk(response); assertResultConcreteIndices(response, "index-1", REMOTE_CLUSTER_1 + ":index-1"); + assertExecutionInfo( + response, + new EsqlResponseExecutionInfo(LOCAL_CLUSTER, "index-1", Status.SUCCESSFUL), + new EsqlResponseExecutionInfo(REMOTE_CLUSTER_1, "index-1", Status.SUCCESSFUL) + ); } } @@ -75,13 +121,19 @@ public void testResolvesRemotesWithPattern() { indexRandom(REMOTE_CLUSTER_1, true, "index-1", 1); indexRandom(REMOTE_CLUSTER_2, true, "index-1", 1); - try (var response = run(syncEsqlQueryRequest().query("FROM *:index-1 METADATA _index"))) { + try (var response = run(syncEsqlQueryRequest().query("FROM *:index-1 METADATA _index").includeCCSMetadata(true))) { assertOk(response); assertResultConcreteIndices(response, REMOTE_CLUSTER_1 + ":index-1", REMOTE_CLUSTER_2 + ":index-1"); // local is not included + assertExecutionInfo( + response, + new EsqlResponseExecutionInfo(REMOTE_CLUSTER_1, "index-1", Status.SUCCESSFUL), + new EsqlResponseExecutionInfo(REMOTE_CLUSTER_2, "index-1", Status.SUCCESSFUL) + ); } - try (var response = run(syncEsqlQueryRequest().query("FROM fake*:index-1 METADATA _index"))) { + try (var response = run(syncEsqlQueryRequest().query("FROM fake*:index-1 METADATA _index").includeCCSMetadata(true))) { assertOk(response); assertResultConcreteIndices(response); // empty + assertExecutionInfo(response); // empty } } @@ -93,14 +145,33 @@ public void testDoesNotResolvesUnknownRemote() { ); } - private EsqlQueryResponse run(EsqlQueryRequest request) { - try { - return client(LOCAL_CLUSTER).execute(EsqlQueryAction.INSTANCE, request).actionGet(30, TimeUnit.SECONDS); - } catch (ElasticsearchTimeoutException e) { - throw new AssertionError("timeout", e); + public void testResolutionWithFilter() { + indexRandom(REMOTE_CLUSTER_1, true, "index-1", 1); + + try ( + var response = run( + syncEsqlQueryRequest().query("FROM " + REMOTE_CLUSTER_1 + ":index-1 METADATA _index").filter(new MatchAllQueryBuilder()) + ) + ) { + assertOk(response); + assertResultConcreteIndices(response, REMOTE_CLUSTER_1 + ":index-1"); + assertExecutionInfo(response, new EsqlResponseExecutionInfo(REMOTE_CLUSTER_1, "index-1", Status.SUCCESSFUL)); + } + try ( + var response = run( + syncEsqlQueryRequest().query("FROM " + REMOTE_CLUSTER_1 + ":index-1 METADATA _index").filter(new MatchNoneQueryBuilder()) + ) + ) { + assertOk(response); + assertResultConcreteIndices(response); + assertExecutionInfo(response, new EsqlResponseExecutionInfo(REMOTE_CLUSTER_1, "index-1", Status.SUCCESSFUL)); } } + private EsqlQueryResponse run(EsqlQueryRequest request) { + return client(LOCAL_CLUSTER).execute(EsqlQueryAction.INSTANCE, request).actionGet(30, TimeUnit.SECONDS); + } + private void indexRandom(String clusterAlias, boolean forceRefresh, String index, int numDocs) { assert numDocs == 1; var client = client(clusterAlias); @@ -123,6 +194,10 @@ private static void assertResultConcreteIndices(EsqlQueryResponse response, Obje assertThat(() -> response.column(indexColumn), containsInAnyOrder(indices)); } + private static void assertExecutionInfo(EsqlQueryResponse response, EsqlResponseExecutionInfo... infos) { + assertThat(executionInfo(response), containsInAnyOrder(infos)); + } + private static int findIndexColumn(EsqlQueryResponse response) { for (int c = 0; c < response.columns().size(); c++) { if (Objects.equals(response.columns().get(c).name(), MetadataAttribute.INDEX)) { @@ -131,4 +206,15 @@ private static int findIndexColumn(EsqlQueryResponse response) { } throw new AssertionError("no _index column found"); } + + private static List executionInfo(EsqlQueryResponse response) { + return response.getExecutionInfo() + .getClusters() + .values() + .stream() + .map(cluster -> new EsqlResponseExecutionInfo(cluster.getClusterAlias(), cluster.getIndexExpression(), cluster.getStatus())) + .toList(); + } + + private record EsqlResponseExecutionInfo(String alias, String index, Status status) {} } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java index b106a9a2d34b7..8b71536bcf7f9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java @@ -143,16 +143,18 @@ public void profile(boolean profile) { this.profile = profile; } - public void includeCCSMetadata(Boolean include) { + public EsqlQueryRequest includeCCSMetadata(Boolean include) { this.includeCCSMetadata = include; + return this; } public Boolean includeCCSMetadata() { return includeCCSMetadata; } - public void includeExecutionMetadata(Boolean include) { + public EsqlQueryRequest includeExecutionMetadata(Boolean include) { this.includeExecutionMetadata = include; + return this; } public Boolean includeExecutionMetadata() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index e31bfdd5364b2..75fc686cf5d0d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -3317,10 +3317,11 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithCa } public void testResolveDenseVector() { - FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( - List.of(fieldCapabilitiesIndexResponse("foo", Map.of("v", new IndexFieldCapabilitiesBuilder("v", "dense_vector").build()))), - List.of() - ); + FieldCapabilitiesResponse caps = FieldCapabilitiesResponse.builder() + .withIndexResponses( + List.of(fieldCapabilitiesIndexResponse("foo", Map.of("v", new IndexFieldCapabilitiesBuilder("v", "dense_vector").build()))) + ) + .build(); { IndexResolution resolution = IndexResolver.mergedMappings( "foo", @@ -3342,15 +3343,16 @@ public void testResolveDenseVector() { } public void testResolveAggregateMetricDouble() { - FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse( - "foo", - Map.of("v", new IndexFieldCapabilitiesBuilder("v", "aggregate_metric_double").build()) + FieldCapabilitiesResponse caps = FieldCapabilitiesResponse.builder() + .withIndexResponses( + List.of( + fieldCapabilitiesIndexResponse( + "foo", + Map.of("v", new IndexFieldCapabilitiesBuilder("v", "aggregate_metric_double").build()) + ) ) - ), - List.of() - ); + ) + .build(); { IndexResolution resolution = IndexResolver.mergedMappings( "foo", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java index 8ea5ecf231407..54e100e962c4b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java @@ -506,9 +506,9 @@ protected void fieldCaps.put(e.getKey(), f); } var indexResponse = new FieldCapabilitiesIndexResponse(alias, null, fieldCaps, true, IndexMode.STANDARD); - response = new FieldCapabilitiesResponse(List.of(indexResponse), List.of()); + response = FieldCapabilitiesResponse.builder().withIndexResponses(List.of(indexResponse)).build(); } else { - response = new FieldCapabilitiesResponse(List.of(), List.of()); + response = FieldCapabilitiesResponse.empty(); } threadPool().executor(ThreadPool.Names.SEARCH_COORDINATION) .execute(ActionRunnable.supply(listener, () -> (Response) new EsqlResolveFieldsResponse(response))); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java index c201f544372db..eda583457243c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java @@ -52,7 +52,7 @@ private void resolve(String esTypeName, TimeSeriesParams.MetricType metricType, ) ); - FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse(idxResponses, List.of()); + FieldCapabilitiesResponse caps = FieldCapabilitiesResponse.builder().withIndexResponses(idxResponses).build(); // IndexResolver uses EsqlDataTypeRegistry directly IndexResolution resolution = IndexResolver.mergedMappings( "idx-*", diff --git a/x-pack/plugin/gpu/src/main/java/org/elasticsearch/xpack/gpu/GPUPlugin.java b/x-pack/plugin/gpu/src/main/java/org/elasticsearch/xpack/gpu/GPUPlugin.java index bcd5e34ed8b6e..f052b48a037d9 100644 --- a/x-pack/plugin/gpu/src/main/java/org/elasticsearch/xpack/gpu/GPUPlugin.java +++ b/x-pack/plugin/gpu/src/main/java/org/elasticsearch/xpack/gpu/GPUPlugin.java @@ -118,7 +118,7 @@ private static KnnVectorsFormat getVectorsFormat( int m = int8HnswIndexOptions.m(); int gpuM = 2 + m * 2 / 3; int gpuEfConstruction = m + m * efConstruction / 256; - return new ES92GpuHnswSQVectorsFormat(gpuM, gpuEfConstruction, int8HnswIndexOptions.confidenceInterval(), 7, false); + return new ES92GpuHnswSQVectorsFormat(gpuM, gpuEfConstruction, null, 7, false); } else { throw new IllegalArgumentException( "GPU vector indexing is not supported on this vector type: [" + indexOptions.getType() + "]" diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java index 366fcc2f821df..ffd2b2b681824 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java @@ -184,8 +184,8 @@ public void testSearchableSnapshotAction() throws Exception { // Manual rollover the original index such that it's not the write index in the data stream anymore rolloverMaxOneDocCondition(client(), dataStream); - awaitIndexExists(restoredIndexName); - awaitIndexDoesNotExist(backingIndexName, TimeValue.timeValueSeconds(60)); + awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20)); + awaitIndexDoesNotExist(backingIndexName); assertBusy( () -> assertThat(explainIndex(client(), restoredIndexName).get("step"), is(PhaseCompleteStep.NAME)), 30, diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java index 735f64821045d..ce70f73d7c28f 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java @@ -111,7 +111,7 @@ public void testSearchableSnapshotAction() throws Exception { assertThat(backingIndices.size(), equalTo(2)); String backingIndexName = backingIndices.getFirst(); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; - awaitIndexExists(restoredIndexName); + awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), restoredIndexName, null, null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished @@ -233,7 +233,7 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception { Map coldActions = Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo)); Map phases = new HashMap<>(); phases.put("cold", new Phase("cold", TimeValue.ZERO, coldActions)); - phases.put("delete", new Phase("delete", TimeValue.timeValueMillis(10000), Map.of(DeleteAction.NAME, WITH_SNAPSHOT_DELETE))); + phases.put("delete", new Phase("delete", TimeValue.ZERO, Map.of(DeleteAction.NAME, WITH_SNAPSHOT_DELETE))); LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, phases); // PUT policy XContentBuilder builder = jsonBuilder(); @@ -261,7 +261,7 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception { String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; // let's wait for ILM to finish - awaitIndexDoesNotExist(backingIndexName); + awaitIndexDoesNotExist(backingIndexName, TimeValue.timeValueSeconds(20)); awaitIndexDoesNotExist(restoredIndexName); List> snapshots = getSnapshots(); @@ -347,7 +347,7 @@ public void testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped() throws String backingIndexName = backingIndices.getFirst(); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; - awaitIndexExists(restoredIndexName); + awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), restoredIndexName, "hot", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(backingIndexName); @@ -417,7 +417,7 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex String backingIndexName = backingIndices.getFirst(); String searchableSnapMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; - awaitIndexExists(searchableSnapMountedIndexName); + awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "hot", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(backingIndexName); @@ -459,7 +459,7 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex restoreSnapshot.setJsonEntity("{\"indices\": \"" + dataStream + "\", \"include_global_state\": false}"); assertOK(client().performRequest(restoreSnapshot)); - assertThat(indexExists(searchableSnapMountedIndexName), is(true)); + awaitIndexExists(searchableSnapMountedIndexName); ensureGreen(searchableSnapMountedIndexName); // the restored index is now managed by the now updated ILM policy and needs to go through the warm and cold phase @@ -523,7 +523,7 @@ public void testIdenticalSearchableSnapshotActionIsNoop() throws Exception { final String searchableSnapMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index; logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName); - awaitIndexExists(searchableSnapMountedIndexName); + awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "cold", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(index); @@ -569,7 +569,7 @@ public void testConvertingSearchableSnapshotFromFullToPartial() throws Exception + SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index; logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName); - awaitIndexExists(searchableSnapMountedIndexName); + awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "frozen", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(index); @@ -635,7 +635,7 @@ public void testResumingSearchableSnapshotFromFullToPartial() throws Exception { final String fullMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index; logger.info("--> waiting for [{}] to exist...", fullMountedIndexName); - awaitIndexExists(fullMountedIndexName); + awaitIndexExists(fullMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), fullMountedIndexName, "cold", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(index); @@ -651,7 +651,7 @@ public void testResumingSearchableSnapshotFromFullToPartial() throws Exception { String partiallyMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX + fullMountedIndexName; logger.info("--> waiting for [{}] to exist...", partiallyMountedIndexName); - awaitIndexExists(partiallyMountedIndexName); + awaitIndexExists(partiallyMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), partiallyMountedIndexName, "frozen", null, PhaseCompleteStep.NAME); // Ensure the searchable snapshot is not deleted when the index was deleted because it was not created by this @@ -727,7 +727,7 @@ public void testResumingSearchableSnapshotFromPartialToFull() throws Exception { final String fullMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index; final String partialMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX + fullMountedIndexName; logger.info("--> waiting for [{}] to exist...", partialMountedIndexName); - awaitIndexExists(partialMountedIndexName); + awaitIndexExists(partialMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), partialMountedIndexName, "frozen", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(index); @@ -743,7 +743,7 @@ public void testResumingSearchableSnapshotFromPartialToFull() throws Exception { String restoredPartiallyMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + partialMountedIndexName; logger.info("--> waiting for [{}] to exist...", restoredPartiallyMountedIndexName); - awaitIndexExists(restoredPartiallyMountedIndexName); + awaitIndexExists(restoredPartiallyMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), restoredPartiallyMountedIndexName, "cold", null, PhaseCompleteStep.NAME); // Ensure the searchable snapshot is not deleted when the index was deleted because it was not created by this @@ -848,7 +848,7 @@ public void testSearchableSnapshotsInHotPhasePinnedToHotNodes() throws Exception final String restoredIndex = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + firstGenIndex; logger.info("--> waiting for [{}] to exist...", restoredIndex); - awaitIndexExists(restoredIndex); + awaitIndexExists(restoredIndex, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), restoredIndex, "hot", PhaseCompleteStep.NAME, PhaseCompleteStep.NAME); Map hotIndexSettings = getIndexSettingsAsMap(restoredIndex); @@ -887,7 +887,7 @@ public void testSearchableSnapshotInvokesAsyncActionOnNewIndex() throws Exceptio assertThat(backingIndices.size(), equalTo(2)); String backingIndexName = backingIndices.getFirst(); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; - awaitIndexExists(restoredIndexName); + awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), restoredIndexName, null, null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished @@ -931,7 +931,7 @@ public void testSearchableSnapshotTotalShardsPerNode() throws Exception { final String searchableSnapMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX + SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index; logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName); - awaitIndexExists(searchableSnapMountedIndexName); + awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "frozen", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(index); @@ -984,7 +984,7 @@ public void testSearchableSnapshotReplicateFor() throws Exception { assertThat(backingIndices.size(), equalTo(2)); String backingIndexName = backingIndices.getFirst(); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; - awaitIndexExists(restoredIndexName); + awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20)); // check that the index is in the expected step and has the expected step_info.message assertBusy(() -> { @@ -1133,7 +1133,7 @@ private void assertForceMergedSnapshotDone(String phase, String backingIndexName ? SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX : SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX; final String restoredIndexName = prefix + backingIndexName; - awaitIndexExists(restoredIndexName); + awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20)); assertBusy(() -> assertThat(explainIndex(client(), restoredIndexName).get("step"), is(PhaseCompleteStep.NAME))); // Wait for the original index to be deleted, to ensure ILM has finished diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java index 45ecb3dedf3f1..ebd881a9ed141 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestDenseInferenceServiceExtension.java @@ -255,7 +255,7 @@ private static List generateEmbedding(String input, int dimensions, Dense // Copied from DenseVectorFieldMapperTestUtils due to dependency restrictions private static int getEmbeddingLength(DenseVectorFieldMapper.ElementType elementType, int dimensions) { return switch (elementType) { - case FLOAT, BYTE -> dimensions; + case FLOAT, BFLOAT16, BYTE -> dimensions; case BIT -> { assert dimensions % Byte.SIZE == 0; yield dimensions / Byte.SIZE; diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticTextIndexOptionsIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticTextIndexOptionsIT.java index 1bd79aab95a4f..cb7115b858701 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticTextIndexOptionsIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticTextIndexOptionsIT.java @@ -120,6 +120,7 @@ public void testValidateIndexOptionsWithBasicLicense() throws Exception { randomIntBetween(1, 100), randomIntBetween(1, 10_000), null, + false, null ); assertAcked( diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index 109646cf0e827..c894e76c86476 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -1455,7 +1455,7 @@ public static DenseVectorFieldMapper.DenseVectorIndexOptions defaultBbqHnswDense int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; DenseVectorFieldMapper.RescoreVector rescoreVector = new DenseVectorFieldMapper.RescoreVector(DEFAULT_RESCORE_OVERSAMPLE); - return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, rescoreVector); + return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, false, rescoreVector); } static SemanticTextIndexOptions defaultIndexOptions(IndexVersion indexVersionCreated, MinimalServiceSettings modelSettings) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java index 5fcb23e1fa23b..a0cb0ea1016be 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java @@ -333,7 +333,7 @@ public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) { /** * Returns the default similarity measure for the embedding type. * Cohere embeddings are expected to be normalized to unit vectors, but due to floating point precision issues, - * our check ({@link DenseVectorFieldMapper#isNotUnitVector(float)}) often fails. + * our check ({@link DenseVectorFieldMapper.Element#isUnitVector(float)}) often fails. * Therefore, we use cosine similarity to ensure compatibility. * * @return The default similarity measure. diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/sagemaker/schema/elastic/ElasticTextEmbeddingPayload.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/sagemaker/schema/elastic/ElasticTextEmbeddingPayload.java index dbbf82f5703b9..78647304dcfa8 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/sagemaker/schema/elastic/ElasticTextEmbeddingPayload.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/sagemaker/schema/elastic/ElasticTextEmbeddingPayload.java @@ -98,6 +98,7 @@ public DenseEmbeddingResults responseBody(SageMakerModel model, InvokeEndpoin case BIT -> TextEmbeddingBinary.PARSER.apply(p, null); case BYTE -> TextEmbeddingBytes.PARSER.apply(p, null); case FLOAT -> TextEmbeddingFloat.PARSER.apply(p, null); + case BFLOAT16 -> throw new UnsupportedOperationException("Bfloat16 not supported"); }; } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java index b3ee52c8dede5..384a321d524be 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java @@ -214,4 +214,9 @@ public void testInvalidOffsets() throws IOException { }))); assertThat(exc.getCause().getCause().getCause().getMessage(), containsString("Illegal offsets")); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsRecoveryTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsRecoveryTests.java index 175c3e90f798d..9bc1736a85c7b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsRecoveryTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsRecoveryTests.java @@ -269,7 +269,7 @@ private static SemanticTextField randomSemanticText( ) throws IOException { ChunkedInference results = switch (model.getTaskType()) { case TEXT_EMBEDDING -> switch (model.getServiceSettings().elementType()) { - case FLOAT -> randomChunkedInferenceEmbeddingFloat(model, inputs); + case FLOAT, BFLOAT16 -> randomChunkedInferenceEmbeddingFloat(model, inputs); case BYTE, BIT -> randomChunkedInferenceEmbeddingByte(model, inputs); }; case SPARSE_EMBEDDING -> randomChunkedInferenceEmbeddingSparse(inputs, false); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 273a202baa7d6..1b3c37269a9cb 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -1850,7 +1850,7 @@ private static DenseVectorFieldMapper.DenseVectorIndexOptions defaultDenseVector // These are the default index options for dense_vector fields, and used for semantic_text fields incompatible with BBQ. int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; - return new DenseVectorFieldMapper.Int8HnswIndexOptions(m, efConstruction, null, null); + return new DenseVectorFieldMapper.Int8HnswIndexOptions(m, efConstruction, null, false, null); } private static SemanticTextIndexOptions defaultDenseVectorSemanticIndexOptions() { @@ -1861,7 +1861,7 @@ private static DenseVectorFieldMapper.DenseVectorIndexOptions defaultBbqHnswDens int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; DenseVectorFieldMapper.RescoreVector rescoreVector = new DenseVectorFieldMapper.RescoreVector(DEFAULT_RESCORE_OVERSAMPLE); - return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, rescoreVector); + return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, false, rescoreVector); } private static SemanticTextIndexOptions defaultBbqHnswSemanticTextIndexOptions() { @@ -1950,7 +1950,7 @@ public void testDefaultIndexOptions() throws IOException { null, new SemanticTextIndexOptions( SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR, - new DenseVectorFieldMapper.Int4HnswIndexOptions(25, 100, null, null) + new DenseVectorFieldMapper.Int4HnswIndexOptions(25, 100, null, false, null) ) ); @@ -2059,7 +2059,7 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { b.field("type", "int4_hnsw"); b.field("m", 20); b.field("ef_construction", 90); - b.field("confidence_interval", 0.4); + b.field("on_disk_rescore", true); b.endObject(); b.endObject(); }), useLegacyFormat, IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT); @@ -2070,7 +2070,7 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { null, new SemanticTextIndexOptions( SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR, - new DenseVectorFieldMapper.Int4HnswIndexOptions(20, 90, 0.4f, null) + new DenseVectorFieldMapper.Int4HnswIndexOptions(20, 90, null, true, null) ) ); @@ -2097,7 +2097,7 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { null, new SemanticTextIndexOptions( SemanticTextIndexOptions.SupportedIndexOptions.DENSE_VECTOR, - new DenseVectorFieldMapper.Int4HnswIndexOptions(16, 100, 0f, null) + new DenseVectorFieldMapper.Int4HnswIndexOptions(16, 100, null, false, null) ) ); @@ -2369,4 +2369,9 @@ private static void assertSparseFeatures(LuceneDocument doc, String fieldName, i private void givenModelSettings(String inferenceId, MinimalServiceSettings modelSettings) { when(globalModelRegistry.getMinimalServiceSettings(inferenceId)).thenReturn(modelSettings); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java index 04f9dfbc4bebf..5eb64696b5917 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java @@ -192,6 +192,7 @@ public static ChunkedInferenceEmbedding randomChunkedInferenceEmbedding(Model mo case TEXT_EMBEDDING -> switch (model.getServiceSettings().elementType()) { case FLOAT -> randomChunkedInferenceEmbeddingFloat(model, inputs); case BIT, BYTE -> randomChunkedInferenceEmbeddingByte(model, inputs); + case BFLOAT16 -> throw new AssertionError(); }; default -> throw new AssertionError("invalid task type: " + model.getTaskType().name()); }; @@ -222,7 +223,7 @@ public static ChunkedInferenceEmbedding randomChunkedInferenceEmbeddingByte(Mode public static ChunkedInferenceEmbedding randomChunkedInferenceEmbeddingFloat(Model model, List inputs) { DenseVectorFieldMapper.ElementType elementType = model.getServiceSettings().elementType(); int embeddingLength = DenseVectorFieldMapperTestUtils.getEmbeddingLength(elementType, model.getServiceSettings().dimensions()); - assert elementType == DenseVectorFieldMapper.ElementType.FLOAT; + assert elementType == DenseVectorFieldMapper.ElementType.FLOAT || elementType == DenseVectorFieldMapper.ElementType.BFLOAT16; List chunks = new ArrayList<>(); for (String input : inputs) { @@ -272,7 +273,7 @@ public static SemanticTextField randomSemanticText( ) throws IOException { ChunkedInference results = switch (model.getTaskType()) { case TEXT_EMBEDDING -> switch (model.getServiceSettings().elementType()) { - case FLOAT -> randomChunkedInferenceEmbeddingFloat(model, inputs); + case FLOAT, BFLOAT16 -> randomChunkedInferenceEmbeddingFloat(model, inputs); case BIT, BYTE -> randomChunkedInferenceEmbeddingByte(model, inputs); }; case SPARSE_EMBEDDING -> randomChunkedInferenceEmbeddingSparse(inputs); @@ -417,6 +418,7 @@ public static ChunkedInference toChunkedResult( EmbeddingResults.Embedding embedding = switch (elementType) { case FLOAT -> new DenseEmbeddingFloatResults.Embedding(FloatConversionUtils.floatArrayOf(values)); case BYTE, BIT -> new DenseEmbeddingByteResults.Embedding(byteArrayOf(values)); + case BFLOAT16 -> throw new AssertionError(); }; chunks.add(new EmbeddingResults.Chunk(embedding, offset)); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/model/TestModel.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/model/TestModel.java index f99e8ce562b42..5eb6d41be1bd0 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/model/TestModel.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/model/TestModel.java @@ -54,7 +54,12 @@ public static TestModel createRandomInstance(TaskType taskType, List excludedSimilarities, int maxDimensions) { if (taskType == TaskType.TEXT_EMBEDDING) { - var elementType = randomFrom(DenseVectorFieldMapper.ElementType.values()); + // TODO: bfloat16 + var elementType = randomFrom( + DenseVectorFieldMapper.ElementType.FLOAT, + DenseVectorFieldMapper.ElementType.BYTE, + DenseVectorFieldMapper.ElementType.BIT + ); var dimensions = DenseVectorFieldMapperTestUtils.randomCompatibleDimensions(elementType, maxDimensions); List supportedSimilarities = new ArrayList<>( diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java index cb5d1d40e2c2a..a475b8a1f4342 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/queries/SemanticQueryBuilderTests.java @@ -277,7 +277,7 @@ private void assertTextEmbeddingLuceneQuery(Query query) { Query innerQuery = assertOuterBooleanQuery(query); Class expectedKnnQueryClass = switch (denseVectorElementType) { - case FLOAT -> KnnFloatVectorQuery.class; + case FLOAT, BFLOAT16 -> KnnFloatVectorQuery.class; case BYTE, BIT -> KnnByteVectorQuery.class; }; assertThat(innerQuery, instanceOf(expectedKnnQueryClass)); diff --git a/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsIndexingIT.java b/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsIndexingIT.java index bea3b7343de3b..d52b364d56888 100644 --- a/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsIndexingIT.java +++ b/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsIndexingIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.get.GetRequest; @@ -37,6 +38,7 @@ import java.util.List; import java.util.UUID; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.resizeRequest; import static org.elasticsearch.index.mapper.DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; @@ -233,12 +235,9 @@ public void testShrink() throws Exception { client().bulk(bulkRequest).actionGet(); client().admin().indices().prepareFlush("my-logs").get(); client().admin().indices().prepareUpdateSettings("my-logs").setSettings(Settings.builder().put("index.blocks.write", true)).get(); - client().admin() - .indices() - .prepareResizeIndex("my-logs", "shrink-my-logs") - .setResizeType(ResizeType.SHRINK) - .setSettings(indexSettings(1, 0).build()) - .get(); + + client().execute(TransportResizeAction.TYPE, resizeRequest(ResizeType.SHRINK, "my-logs", "shrink-my-logs", indexSettings(1, 0))) + .actionGet(); assertNoFailures(client().admin().indices().prepareForceMerge("shrink-my-logs").setMaxNumSegments(1).setFlush(true).get()); } diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java index 1b3ccb376febf..8a684122cb658 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java @@ -470,4 +470,9 @@ public void testSyntheticSourceKeepArrays() { protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java index 874ced5436f69..b1e31882fbf1a 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java @@ -618,4 +618,9 @@ public void testSyntheticSourceKeepArrays() { protected boolean supportsCopyTo() { return false; } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index d5405018dd64c..cdbfc6c3b9582 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -346,4 +346,9 @@ protected boolean supportsEmptyInputArray() { protected boolean addsValueWhenNotSupplied() { return true; } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapperTests.java b/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapperTests.java index d6b762ab4b79e..5c26222ff1618 100644 --- a/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapperTests.java @@ -219,4 +219,9 @@ public void testDisableIndex() throws IOException { assertEquals(IndexOptions.NONE, fields.get(0).fieldType().indexOptions()); assertEquals(DocValuesType.SORTED_SET, fields.get(0).fieldType().docValuesType()); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java b/x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java index 730d0b591ec7a..f782e7f30b7e3 100644 --- a/x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java +++ b/x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java @@ -605,4 +605,9 @@ public void testSyntheticSourceKeepArrays() { protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not yet implemented"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index cb28a6168c561..ea3e4df43d615 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -520,4 +520,9 @@ protected Object[] getThreeEncodedSampleValues() { protected boolean supportsBulkLongBlockReading() { return true; } + + @Override + protected List getSortShortcutSupport() { + return List.of(new SortShortcutSupport(this::minimalMapping, this::writeField, true)); + } } diff --git a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index 710b6652ed05d..61d6021005f56 100644 --- a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -198,4 +198,9 @@ public List invalidExample() throws IOException { return List.of(); } } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 532f9cf5c71d8..3c6e6d685f377 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -1527,7 +1527,7 @@ public List getRestHandlers( restHandlers.add(new RestDeleteTrainedModelAliasAction()); restHandlers.add(new RestPutTrainedModelDefinitionPartAction()); restHandlers.add(new RestInferTrainedModelAction()); - restHandlers.add(new RestCatTrainedModelsAction()); + restHandlers.add(new RestCatTrainedModelsAction(machineLearningExtension.get().isDataFrameAnalyticsEnabled())); if (machineLearningExtension.get().isDataFrameAnalyticsEnabled()) { restHandlers.add(new RestGetDataFrameAnalyticsAction()); restHandlers.add(new RestGetDataFrameAnalyticsStatsAction()); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java index d4a2b602759ad..6299aa4bca6af 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java @@ -49,6 +49,12 @@ @ServerlessScope(Scope.PUBLIC) public class RestCatTrainedModelsAction extends AbstractCatAction { + private final boolean areDataFrameAnalyticsEnabled; + + public RestCatTrainedModelsAction(boolean areDataFrameAnalyticsEnabled) { + this.areDataFrameAnalyticsEnabled = areDataFrameAnalyticsEnabled; + } + @Override public List routes() { return List.of( @@ -122,14 +128,18 @@ private void getDerivedData( listeners.acquire(response -> trainedModelsStats = response.getResources().results()) ); - final var dataFrameAnalyticsRequest = new GetDataFrameAnalyticsAction.Request(requestIdPattern); - dataFrameAnalyticsRequest.setAllowNoResources(true); - dataFrameAnalyticsRequest.setPageParams(new PageParams(0, potentialAnalyticsIds.size())); - client.execute( - GetDataFrameAnalyticsAction.INSTANCE, - dataFrameAnalyticsRequest, - listeners.acquire(response -> dataFrameAnalytics = response.getResources().results()) - ); + if (areDataFrameAnalyticsEnabled) { + final var dataFrameAnalyticsRequest = new GetDataFrameAnalyticsAction.Request(requestIdPattern); + dataFrameAnalyticsRequest.setAllowNoResources(true); + dataFrameAnalyticsRequest.setPageParams(new PageParams(0, potentialAnalyticsIds.size())); + client.execute( + GetDataFrameAnalyticsAction.INSTANCE, + dataFrameAnalyticsRequest, + listeners.acquire(response -> dataFrameAnalytics = response.getResources().results()) + ); + } else { + dataFrameAnalytics = List.of(); + } } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java index b7646f430726a..a3ebdadb20db8 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java @@ -285,14 +285,20 @@ private Map testCreateDestinationIndex(DataFrameAnalysis analysi doAnswer(callListenerOnResponse(getMappingsResponse)).when(client) .execute(eq(GetMappingsAction.INSTANCE), getMappingsRequestCaptor.capture(), any()); - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse(new String[0], new HashMap<>() { - { - put(NUMERICAL_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(OUTER_FIELD + "." + INNER_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(ALIAS_TO_NUMERICAL_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(ALIAS_TO_NESTED_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - } - }); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.builder() + .withFields( + Map.of( + NUMERICAL_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + OUTER_FIELD + "." + INNER_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + ALIAS_TO_NUMERICAL_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + ALIAS_TO_NESTED_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")) + ) + ) + .build(); doAnswer(callListenerOnResponse(fieldCapabilitiesResponse)).when(client) .execute(eq(TransportFieldCapabilitiesAction.TYPE), fieldCapabilitiesRequestCaptor.capture(), any()); @@ -615,14 +621,20 @@ private Map testUpdateMappingsToDestIndex(DataFrameAnalysis anal doAnswer(callListenerOnResponse(AcknowledgedResponse.TRUE)).when(client) .execute(eq(TransportPutMappingAction.TYPE), putMappingRequestCaptor.capture(), any()); - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse(new String[0], new HashMap<>() { - { - put(NUMERICAL_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(OUTER_FIELD + "." + INNER_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(ALIAS_TO_NUMERICAL_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(ALIAS_TO_NESTED_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - } - }); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.builder() + .withFields( + Map.of( + NUMERICAL_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + OUTER_FIELD + "." + INNER_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + ALIAS_TO_NUMERICAL_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + ALIAS_TO_NESTED_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")) + ) + ) + .build(); doAnswer(callListenerOnResponse(fieldCapabilitiesResponse)).when(client) .execute(eq(TransportFieldCapabilitiesAction.TYPE), fieldCapabilitiesRequestCaptor.capture(), any()); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java index f028f39c6069f..4b9c25d71f452 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java @@ -1724,7 +1724,7 @@ private MockFieldCapsResponseBuilder addField(String field, boolean isMetadataFi } private FieldCapabilitiesResponse build() { - return new FieldCapabilitiesResponse(new String[] { "test" }, fieldCaps); + return FieldCapabilitiesResponse.builder().withIndices(new String[] { "test" }).withFields(fieldCaps).build(); } } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsActionTests.java index f7cfba048e722..079cd5d9c4d0a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsActionTests.java @@ -7,28 +7,25 @@ package org.elasticsearch.xpack.ml.rest.cat; +import org.elasticsearch.common.Strings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsStatsAction; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsStatsActionResponseTests; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfigTests; import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfigTests; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TrainedModelSizeStats; -import org.junit.Before; import java.util.List; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; public class RestCatTrainedModelsActionTests extends ESTestCase { - private RestCatTrainedModelsAction action; - - @Before - public void setUpAction() { - action = new RestCatTrainedModelsAction(); - } - public void testBuildTableAccumulatedStats() { + var action = new RestCatTrainedModelsAction(true); + // GetTrainedModelsStatsActionResponseTests var deployment1 = new GetTrainedModelsStatsAction.Response.TrainedModelStats( "id1", @@ -48,10 +45,13 @@ public void testBuildTableAccumulatedStats() { null ); - var configs = List.of(TrainedModelConfigTests.createTestInstance("id1").build()); + var dataframeConfig = DataFrameAnalyticsConfigTests.createRandom("dataframe1"); + var configs = List.of( + TrainedModelConfigTests.createTestInstance(deployment1.getModelId()).setTags(List.of(dataframeConfig.getId())).build() + ); - var table = action.buildTable(new FakeRestRequest(), List.of(deployment1, deployment2), configs, List.of()); - assertThat(table.getRows().get(0).get(0).value, is("id1")); + var table = action.buildTable(new FakeRestRequest(), List.of(deployment1, deployment2), configs, List.of(dataframeConfig)); + assertThat(table.getRows().get(0).get(0).value, is(deployment1.getModelId())); // pipeline count assertThat(table.getRows().get(0).get(9).value, is(4)); // ingest count @@ -82,5 +82,12 @@ public void testBuildTableAccumulatedStats() { .ingestFailedCount() ) ); + assertThat(table.getRows().get(0).get(14).value, is(dataframeConfig.getId())); + assertThat(table.getRows().get(0).get(15).value, is(dataframeConfig.getCreateTime())); + assertThat(table.getRows().get(0).get(16).value, is(Strings.arrayToCommaDelimitedString(dataframeConfig.getSource().getIndex()))); + assertThat( + table.getRows().get(0).get(17).value, + dataframeConfig.getAnalysis() == null ? nullValue() : is(dataframeConfig.getAnalysis().getWriteableName()) + ); } } diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java index b858b935c1483..f56b974e0a95a 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsDVLeafFieldData.java @@ -128,6 +128,7 @@ public Object nextValue() { return vectors; } }; + case BFLOAT16 -> throw new IllegalArgumentException("Unsupported element type: bfloat16"); }; } @@ -140,6 +141,7 @@ public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { case BYTE -> new ByteRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims); case FLOAT -> new FloatRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims); case BIT -> new BitRankVectorsDocValuesField(values, magnitudeValues, name, elementType, dims); + case BFLOAT16 -> throw new IllegalArgumentException("Unsupported element type: bfloat16"); }; } catch (IOException e) { throw new IllegalStateException("Cannot load doc values for multi-vector field!", e); diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java index adb925757b6ca..7f2bd456db4da 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapper.java @@ -77,6 +77,9 @@ public static class Builder extends FieldMapper.Builder { "invalid element_type [" + o + "]; available types are " + namesToElementType.keySet() ); } + if (elementType == ElementType.BFLOAT16) { + throw new MapperParsingException("Rank vectors does not support bfloat16"); + } return elementType; }, m -> toType(m).fieldType().element.elementType(), @@ -342,7 +345,7 @@ public void parse(DocumentParserContext context) throws IOException { ByteBuffer buffer = ByteBuffer.allocate(bufferSize).order(ByteOrder.LITTLE_ENDIAN); ByteBuffer magnitudeBuffer = ByteBuffer.allocate(vectors.size() * Float.BYTES).order(ByteOrder.LITTLE_ENDIAN); for (VectorData vector : vectors) { - vector.addToBuffer(buffer); + vector.addToBuffer(element, buffer); magnitudeBuffer.putFloat((float) Math.sqrt(element.computeSquaredMagnitude(vector))); } String vectorFieldName = fieldType().name(); diff --git a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java index 1c533e9ec88cd..bd1c06f7c1dd1 100644 --- a/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java +++ b/x-pack/plugin/rank-vectors/src/main/java/org/elasticsearch/xpack/rank/vectors/script/RankVectorsScoreScriptUtils.java @@ -357,6 +357,7 @@ public MaxSimDotProduct(ScoreScript scoreScript, Object queryVector, String fiel } throw new IllegalArgumentException("Unsupported input object for float vectors: " + queryVector.getClass().getName()); } + case BFLOAT16 -> throw new IllegalArgumentException("Unsupported element type: bfloat16"); }; } diff --git a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java index 5834aca5fa0a5..33b2f084249b2 100644 --- a/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java +++ b/x-pack/plugin/rank-vectors/src/test/java/org/elasticsearch/xpack/rank/vectors/mapper/RankVectorsFieldMapperTests.java @@ -434,6 +434,7 @@ protected Object generateRandomInputValue(MappedFieldType ft) { } yield vectors; } + case BFLOAT16 -> throw new AssertionError(); }; } @@ -472,10 +473,9 @@ private static class DenseVectorSyntheticSourceSupport implements SyntheticSourc @Override public SyntheticSourceExample example(int maxValues) { Object value = switch (elementType) { - case BYTE, BIT: - yield randomList(numVecs, numVecs, () -> randomList(dims, dims, ESTestCase::randomByte)); - case FLOAT: - yield randomList(numVecs, numVecs, () -> randomList(dims, dims, ESTestCase::randomFloat)); + case BYTE, BIT -> randomList(numVecs, numVecs, () -> randomList(dims, dims, ESTestCase::randomByte)); + case FLOAT -> randomList(numVecs, numVecs, () -> randomList(dims, dims, ESTestCase::randomFloat)); + case BFLOAT16 -> throw new AssertionError(); }; return new SyntheticSourceExample(value, value, this::mapping); } diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java index 5c86ca4649194..83bb975131805 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java @@ -62,6 +62,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.executeResize; import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram; @@ -384,15 +385,15 @@ public void testCreateAndRestorePartialSearchableSnapshot() throws Exception { final String clonedIndexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); assertAcked( - indicesAdmin().prepareResizeIndex(restoredIndexName, clonedIndexName) - .setResizeType(ResizeType.CLONE) - .setSettings( - Settings.builder() - .putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()) - .putNull(IndexModule.INDEX_RECOVERY_TYPE_SETTING.getKey()) - .put(DataTier.TIER_PREFERENCE, DataTier.DATA_HOT) - .build() - ) + executeResize( + ResizeType.CLONE, + restoredIndexName, + clonedIndexName, + Settings.builder() + .putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()) + .putNull(IndexModule.INDEX_RECOVERY_TYPE_SETTING.getKey()) + .put(DataTier.TIER_PREFERENCE, DataTier.DATA_HOT) + ) ); ensureGreen(clonedIndexName); assertTotalHits(clonedIndexName, originalAllHits, originalBarHits); diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java index 0fe63df106862..69c3792a0f665 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java @@ -81,6 +81,7 @@ import java.util.stream.IntStream; import static org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanationUtils.getClusterAllocationExplanation; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.executeResize; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING; import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; @@ -330,14 +331,14 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception { final String clonedIndexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); assertAcked( - indicesAdmin().prepareResizeIndex(restoredIndexName, clonedIndexName) - .setResizeType(ResizeType.CLONE) - .setSettings( - Settings.builder() - .putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()) - .putNull(IndexModule.INDEX_RECOVERY_TYPE_SETTING.getKey()) - .build() - ) + executeResize( + ResizeType.CLONE, + restoredIndexName, + clonedIndexName, + Settings.builder() + .putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()) + .putNull(IndexModule.INDEX_RECOVERY_TYPE_SETTING.getKey()) + ) ); ensureGreen(clonedIndexName); assertTotalHits(clonedIndexName, originalAllHits, originalBarHits); diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsResizeIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsResizeIntegTests.java index 0a0cce5dd87ed..d6db625fa02c8 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsResizeIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsResizeIntegTests.java @@ -18,6 +18,7 @@ import java.util.List; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.executeResize; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING; import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; @@ -55,13 +56,14 @@ public void tearDown() throws Exception { super.tearDown(); } + private static void runResizeAction(String targetIndexName, ResizeType resizeType, Settings.Builder settings) { + assertAcked(executeResize(resizeType, "mounted-index", targetIndexName, settings)); + } + public void testShrinkSearchableSnapshotIndex() { final IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> indicesAdmin().prepareResizeIndex("mounted-index", "shrunk-index") - .setResizeType(ResizeType.SHRINK) - .setSettings(indexSettingsNoReplicas(1).build()) - .get() + () -> runResizeAction("shrunk-index", ResizeType.SHRINK, indexSettingsNoReplicas(1)) ); assertThat(exception.getMessage(), equalTo("can't shrink searchable snapshot index [mounted-index]")); } @@ -69,10 +71,7 @@ public void testShrinkSearchableSnapshotIndex() { public void testSplitSearchableSnapshotIndex() { final IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> indicesAdmin().prepareResizeIndex("mounted-index", "split-index") - .setResizeType(ResizeType.SPLIT) - .setSettings(indexSettingsNoReplicas(4).build()) - .get() + () -> runResizeAction("split-index", ResizeType.SPLIT, indexSettingsNoReplicas(4)) ); assertThat(exception.getMessage(), equalTo("can't split searchable snapshot index [mounted-index]")); } @@ -80,7 +79,7 @@ public void testSplitSearchableSnapshotIndex() { public void testCloneSearchableSnapshotIndex() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> indicesAdmin().prepareResizeIndex("mounted-index", "cloned-index").setResizeType(ResizeType.CLONE).get() + () -> runResizeAction("cloned-index", ResizeType.CLONE, Settings.builder()) ); assertThat( exception.getMessage(), @@ -89,27 +88,25 @@ public void testCloneSearchableSnapshotIndex() { exception = expectThrows( IllegalArgumentException.class, - () -> indicesAdmin().prepareResizeIndex("mounted-index", "cloned-index") - .setResizeType(ResizeType.CLONE) - .setSettings(Settings.builder().putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()).build()) - .get() + () -> runResizeAction( + "cloned-index", + ResizeType.CLONE, + Settings.builder().putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()) + ) ); assertThat( exception.getMessage(), equalTo("can't clone searchable snapshot index [mounted-index]; setting [index.recovery.type] should be overridden") ); - assertAcked( - indicesAdmin().prepareResizeIndex("mounted-index", "cloned-index") - .setResizeType(ResizeType.CLONE) - .setSettings( - Settings.builder() - .putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()) - .putNull(IndexModule.INDEX_RECOVERY_TYPE_SETTING.getKey()) - .put(DataTier.TIER_PREFERENCE, DataTier.DATA_HOT) - .put(INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) - .build() - ) + runResizeAction( + "cloned-index", + ResizeType.CLONE, + Settings.builder() + .putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()) + .putNull(IndexModule.INDEX_RECOVERY_TYPE_SETTING.getKey()) + .put(DataTier.TIER_PREFERENCE, DataTier.DATA_HOT) + .put(INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) ); ensureGreen("cloned-index"); assertAcked(indicesAdmin().prepareDelete("cloned-index")); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java index 9def00c964a7d..c37cd36dacb64 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java @@ -121,16 +121,11 @@ public void testDownsamplingAuthorized() throws Exception { String dataStreamName = "metrics-foo"; DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingMethod(randomSamplingMethod()) + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueSeconds(10), - new DownsampleConfig(new DateHistogramInterval("10m")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueSeconds(10), new DateHistogramInterval("10m")) ) ) .buildTemplate(); @@ -408,19 +403,22 @@ private void bulkIndex(Client client, String dataStreamName, Supplier Indexed [{}] documents. Dropped [{}] duplicates.", docsIndexed, duplicates); } + private static DownsampleConfig.SamplingMethod randomSamplingMethod() { + if (between(0, DownsampleConfig.SamplingMethod.values().length) == 0) { + return null; + } else { + return randomFrom(DownsampleConfig.SamplingMethod.values()); + } + } + public static class SystemDataStreamWithDownsamplingConfigurationPlugin extends Plugin implements SystemIndexPlugin { public static final DataStreamLifecycle.Template LIFECYCLE = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingMethod(randomSamplingMethod()) + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueSeconds(10), - new DownsampleConfig(new DateHistogramInterval("10m")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueSeconds(10), new DateHistogramInterval("10m")) ) ) .buildTemplate(); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/ShrinkIndexWithSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/ShrinkIndexWithSecurityTests.java index 05dc4325a302a..3926a047c43aa 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/ShrinkIndexWithSecurityTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/ShrinkIndexWithSecurityTests.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.integration; +import org.elasticsearch.action.admin.indices.ResizeIndexTestUtils; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.TermsQueryBuilder; @@ -46,7 +48,8 @@ public void testShrinkIndex() throws Exception { // wait for green and then shrink ensureGreen(); - assertAcked(indicesAdmin().prepareResizeIndex("bigindex", "shrunk_bigindex").setSettings(indexSettings(1, 0).build())); + + assertAcked(ResizeIndexTestUtils.executeResize(ResizeType.SHRINK, "bigindex", "shrunk_bigindex", indexSettings(1, 0))); // verify all docs ensureGreen(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java index d29200fdccc41..d1f8e5096dd23 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java @@ -120,7 +120,8 @@ public void testSortedSetDVOrdinalsIndexFieldData_global() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( this.ir, new CharacterRunAutomaton(Automata.makeEmpty()), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); global = sortedSetOrdinalsIndexFieldData.loadGlobal(ir); atomic = global.load(ir.leaves().get(0)); @@ -137,7 +138,8 @@ public void testSortedSetDVOrdinalsIndexFieldData_segment() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( this.ir, new CharacterRunAutomaton(Automata.makeEmpty()), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); for (LeafReaderContext context : ir.leaves()) { LeafOrdinalsFieldData atomic = sortedSetOrdinalsIndexFieldData.load(context); @@ -157,7 +159,8 @@ public void testPagedBytesIndexFieldData_global() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( this.ir, new CharacterRunAutomaton(Automata.makeEmpty()), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); global = pagedBytesIndexFieldData.loadGlobal(ir); atomic = global.load(ir.leaves().get(0)); @@ -176,7 +179,8 @@ public void testPagedBytesIndexFieldData_segment() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( this.ir, new CharacterRunAutomaton(Automata.makeEmpty()), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); for (LeafReaderContext context : ir.leaves()) { LeafOrdinalsFieldData atomic = pagedBytesIndexFieldData.load(context); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptorTests.java index e3b402d96d416..618d374984e41 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptorTests.java @@ -8,8 +8,9 @@ import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.shrink.ResizeAction; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -111,7 +112,12 @@ public void checkResizeWithDlsFlsConfigured(boolean dlsFlsFeatureEnabled, String ); PlainActionFuture plainActionFuture = new PlainActionFuture<>(); - RequestInfo requestInfo = new RequestInfo(authentication, new ResizeRequest("bar", "foo"), ResizeAction.NAME, null); + RequestInfo requestInfo = new RequestInfo( + authentication, + new ResizeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, randomFrom(ResizeType.values()), "foo", "bar"), + TransportResizeAction.TYPE.name(), + null + ); AuthorizationEngine mockEngine = mock(AuthorizationEngine.class); doAnswer(invocationOnMock -> { ActionListener listener = (ActionListener) invocationOnMock.getArguments()[3]; @@ -147,7 +153,12 @@ public void testResizeRequestInterceptorThrowsWhenTargetHasGreaterPermissions() AuthorizationEngine mockEngine = mock(AuthorizationEngine.class); { PlainActionFuture plainActionFuture = new PlainActionFuture<>(); - RequestInfo requestInfo = new RequestInfo(authentication, new ResizeRequest("target", "source"), ResizeAction.NAME, null); + RequestInfo requestInfo = new RequestInfo( + authentication, + new ResizeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, randomFrom(ResizeType.values()), "source", "target"), + TransportResizeAction.TYPE.name(), + null + ); doAnswer(invocationOnMock -> { ActionListener listener = (ActionListener) invocationOnMock.getArguments()[3]; listener.onResponse(AuthorizationResult.deny()); @@ -172,7 +183,12 @@ public void testResizeRequestInterceptorThrowsWhenTargetHasGreaterPermissions() // swap target and source for success { PlainActionFuture plainActionFuture = new PlainActionFuture<>(); - RequestInfo requestInfo = new RequestInfo(authentication, new ResizeRequest("source", "target"), ResizeAction.NAME, null); + RequestInfo requestInfo = new RequestInfo( + authentication, + new ResizeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, randomFrom(ResizeType.values()), "target", "source"), + TransportResizeAction.TYPE.name(), + null + ); doAnswer(invocationOnMock -> { ActionListener listener = (ActionListener) invocationOnMock.getArguments()[3]; listener.onResponse(AuthorizationResult.granted()); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java index 9482f45063b64..d656f407b3243 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java @@ -553,4 +553,9 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java index 70df2763e30c6..3cb9f531b23dc 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java @@ -553,4 +553,9 @@ protected Object[] getThreeEncodedSampleValues() { protected boolean supportsBulkLongBlockReading() { return true; } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java index ee6f6bc8b70c0..9530f24a153a1 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java @@ -372,4 +372,9 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index 0610721c04537..27a4adb910bf0 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -600,7 +600,7 @@ private static IndexResolution mergedMappings( return IndexResolver.mergedMappings( SqlDataTypeRegistry.INSTANCE, indexPattern, - new FieldCapabilitiesResponse(indexNames, fieldCaps) + FieldCapabilitiesResponse.builder().withIndices(indexNames).withFields(fieldCaps).build() ); } @@ -612,7 +612,7 @@ private static List separateMappings( return IndexResolver.separateMappings( SqlDataTypeRegistry.INSTANCE, javaRegex, - new FieldCapabilitiesResponse(indexNames, fieldCaps), + FieldCapabilitiesResponse.builder().withIndices(indexNames).withFields(fieldCaps).build(), null ); } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml index 301cb01acd2d3..c038c33f68f5a 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml @@ -24,7 +24,134 @@ Filter single field: name: type: keyword secret: + type: match_only_text + + - do: + bulk: + index: index_fls + refresh: true + body: + - '{"create": { }}' + - '{"name": "A", "secret":"squirrel"}' + - match: { errors: false } + + - do: + security.create_api_key: + body: + name: "test-fls" + expiration: "1d" + role_descriptors: + index_access: + indices: + - names: [ "index_fls" ] + privileges: [ "read" ] + field_security: + grant: [ "name" ] + - match: { name: "test-fls" } + - is_true: id + - set: + id: api_key_id + encoded: credentials + + # With superuser... + - do: + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.secret: squirrel } + + # With FLS API Key + - do: + headers: + Authorization: "ApiKey ${credentials}" + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - is_false: "hits.hits.0._source.secret" + +--- +match_only_text field type grant all except secret field: + - do: + indices.create: + index: index_fls + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: type: keyword + secret: + type: match_only_text + + - do: + bulk: + index: index_fls + refresh: true + body: + - '{"create": { }}' + - '{"name": "A", "secret":"squirrel"}' + - match: { errors: false } + + - do: + security.create_api_key: + body: + name: "test-fls" + expiration: "1d" + role_descriptors: + index_access: + indices: + - names: [ "index_fls" ] + privileges: [ "read" ] + field_security: + grant: [ "*" ] + except: [ "secret" ] + - match: { name: "test-fls" } + - is_true: id + - set: + id: api_key_id + encoded: credentials + + # With superuser... + - do: + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.secret: squirrel } + + # With FLS API Key + - do: + headers: + Authorization: "ApiKey ${credentials}" + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - is_false: "hits.hits.0._source.secret" + +--- +match_only_text field type grant name field: + - do: + indices.create: + index: index_fls + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: + type: match_only_text + secret: + type: match_only_text - do: bulk: @@ -73,6 +200,136 @@ Filter single field: - match: { hits.hits.0._source.name: A } - is_false: "hits.hits.0._source.secret" +--- +keyword field type with ignore_above: + - do: + indices.create: + index: index_fls + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: + type: keyword + secret: + type: keyword + ignore_above: 3 + + - do: + bulk: + index: index_fls + refresh: true + body: + - '{"create": { }}' + - '{"name": "A", "secret":"squirrel"}' + - match: { errors: false } + + - do: + security.create_api_key: + body: + name: "test-fls" + expiration: "1d" + role_descriptors: + index_access: + indices: + - names: [ "index_fls" ] + privileges: [ "read" ] + field_security: + grant: [ "*" ] + except: [ "secret" ] + - match: { name: "test-fls" } + - is_true: id + - set: + id: api_key_id + encoded: credentials + + # With superuser... + - do: + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.secret: squirrel } + + # With FLS API Key + - do: + headers: + Authorization: "ApiKey ${credentials}" + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - is_false: "hits.hits.0._source.secret" + +--- +long field type with ignore_malformed: + - do: + indices.create: + index: index_fls + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: + type: keyword + secret: + type: long + ignore_malformed: true + + - do: + bulk: + index: index_fls + refresh: true + body: + - '{"create": { }}' + - '{"name": "A", "secret":"squirrel"}' + - match: { errors: false } + + - do: + security.create_api_key: + body: + name: "test-fls" + expiration: "1d" + role_descriptors: + index_access: + indices: + - names: [ "index_fls" ] + privileges: [ "read" ] + field_security: + grant: [ "*" ] + except: [ "secret" ] + - match: { name: "test-fls" } + - is_true: id + - set: + id: api_key_id + encoded: credentials + + # With superuser... + - do: + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.secret: squirrel } + + # With FLS API Key + - do: + headers: + Authorization: "ApiKey ${credentials}" + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - is_false: "hits.hits.0._source.secret" + --- Filter fields in object: - do: diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java index 18af78d704646..49b58f0dde2b4 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java @@ -87,16 +87,18 @@ public void testRemoveInternalFields() { } public void testExtractFieldMappings() { - FieldCapabilitiesResponse response = new FieldCapabilitiesResponse( - new String[] { "some-index" }, - Map.ofEntries( - entry("field-1", Map.of("keyword", createFieldCapabilities("field-1", "keyword"))), - entry( - "field-2", - Map.of("long", createFieldCapabilities("field-2", "long"), "keyword", createFieldCapabilities("field-2", "keyword")) + FieldCapabilitiesResponse response = FieldCapabilitiesResponse.builder() + .withIndices(new String[] { "some-index" }) + .withFields( + Map.ofEntries( + entry("field-1", Map.of("keyword", createFieldCapabilities("field-1", "keyword"))), + entry( + "field-2", + Map.of("long", createFieldCapabilities("field-2", "long"), "keyword", createFieldCapabilities("field-2", "keyword")) + ) ) ) - ); + .build(); assertThat( DocumentConversionUtils.extractFieldMappings(response), diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java index d65428a3912de..580e3fb2bdbfa 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java @@ -287,7 +287,10 @@ protected void responseMap.put(field, singletonMap(field, createFieldCapabilities(field, type))); } - final FieldCapabilitiesResponse response = new FieldCapabilitiesResponse(fieldCapsRequest.indices(), responseMap); + final FieldCapabilitiesResponse response = FieldCapabilitiesResponse.builder() + .withIndices(fieldCapsRequest.indices()) + .withFields(responseMap) + .build(); listener.onResponse((Response) response); return; } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index f23536acba87b..ad28b0336d855 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -70,6 +70,7 @@ import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MappingParserContext; import org.elasticsearch.index.mapper.SourceValueFetcher; +import org.elasticsearch.index.mapper.TextFamilyFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.mapper.blockloader.docvalues.BytesRefsFromCustomBinaryBlockLoader; @@ -1023,7 +1024,7 @@ private WildcardFieldMapper( this.indexVersionCreated = builder.indexCreatedVersion; this.ignoreAboveDefault = builder.ignoreAboveDefault; this.ignoreAbove = new IgnoreAbove(builder.ignoreAbove.getValue(), builder.indexMode, builder.indexCreatedVersion); - this.originalName = storeIgnored ? fullPath() + "._original" : null; + this.originalName = storeIgnored ? fullPath() + TextFamilyFieldType.FALLBACK_FIELD_NAME_SUFFIX : null; } @Override diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index be216c55d49cf..0690d38fe79ce 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -1272,4 +1272,9 @@ public List invalidExample() throws IOException { return List.of(); } } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } }