From c10fd7605e4d7efa9671312d44401e93fcd0bf6b Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Wed, 15 Oct 2025 15:11:04 +0100 Subject: [PATCH 01/32] Add HNSW scalar quantized bfloat16 implementation --- server/src/main/java/module-info.java | 3 +- .../ES93HnswScalarQuantizedVectorsFormat.java | 64 +++++ .../ES93ScalarQuantizedVectorsFormat.java | 192 +++++++++++++++ .../org.apache.lucene.codecs.KnnVectorsFormat | 1 + ...arQuantizedBFloat16VectorsFormatTests.java | 103 ++++++++ ...HnswScalarQuantizedVectorsFormatTests.java | 220 ++++++++++++++++++ 6 files changed, 582 insertions(+), 1 deletion(-) create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java create mode 100644 server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedBFloat16VectorsFormatTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormatTests.java diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 2987b3849e663..234d43315a9d4 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -464,6 +464,7 @@ org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsFormat, org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat, org.elasticsearch.index.codec.vectors.diskbbq.next.ESNextDiskBBQVectorsFormat, + org.elasticsearch.index.codec.vectors.es93.ES93HnswScalarQuantizedVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93HnswBinaryQuantizedVectorsFormat; @@ -494,6 +495,6 @@ exports org.elasticsearch.inference.telemetry; exports org.elasticsearch.index.codec.vectors.diskbbq to org.elasticsearch.test.knn; exports org.elasticsearch.index.codec.vectors.cluster to org.elasticsearch.test.knn; - exports org.elasticsearch.index.codec.vectors.es93 to org.elasticsearch.test.knn; exports org.elasticsearch.search.crossproject; + exports org.elasticsearch.index.codec.vectors.es93 to org.elasticsearch.gpu, org.elasticsearch.test.knn; } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java new file mode 100644 index 0000000000000..fe5e2c5696371 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.elasticsearch.index.codec.vectors.AbstractHnswVectorsFormat; + +import java.io.IOException; + +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; + +public class ES93HnswScalarQuantizedVectorsFormat extends AbstractHnswVectorsFormat { + + static final String NAME = "ES93HnswScalarQuantizedVectorsFormat"; + + /** The format for storing, reading, merging vectors on disk */ + private final FlatVectorsFormat flatVectorsFormat; + + public ES93HnswScalarQuantizedVectorsFormat() { + this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, false, null, 7, false, false); + } + + public ES93HnswScalarQuantizedVectorsFormat( + int maxConn, + int beamWidth, + boolean useBFloat16, + Float confidenceInterval, + int bits, + boolean compress, + boolean useDirectIO + ) { + super(NAME, maxConn, beamWidth); + this.flatVectorsFormat = new ES93ScalarQuantizedVectorsFormat(useBFloat16, confidenceInterval, bits, compress, useDirectIO); + } + + @Override + protected FlatVectorsFormat flatVectorsFormat() { + return flatVectorsFormat; + } + + @Override + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return new Lucene99HnswVectorsWriter(state, maxConn, beamWidth, flatVectorsFormat.fieldsWriter(state), numMergeWorkers, mergeExec); + } + + @Override + public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { + return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state)); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java new file mode 100644 index 0000000000000..f1dd1a283f88f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java @@ -0,0 +1,192 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.codecs.hnsw.FlatVectorsScorer; +import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.codecs.hnsw.ScalarQuantizedVectorScorer; +import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsReader; +import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsWriter; +import org.apache.lucene.index.KnnVectorValues; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.util.hnsw.RandomVectorScorer; +import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier; +import org.apache.lucene.util.quantization.QuantizedByteVectorValues; +import org.elasticsearch.simdvec.VectorScorerFactory; +import org.elasticsearch.simdvec.VectorSimilarityType; + +import java.io.IOException; + +import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.DYNAMIC_CONFIDENCE_INTERVAL; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; + +public class ES93ScalarQuantizedVectorsFormat extends FlatVectorsFormat { + + static final String NAME = "ES93ScalarQuantizedVectorsFormat"; + private static final int ALLOWED_BITS = (1 << 8) | (1 << 7) | (1 << 4); + + static final FlatVectorsScorer flatVectorScorer = new ESFlatVectorsScorer( + new ScalarQuantizedVectorScorer(FlatVectorScorerUtil.getLucene99FlatVectorsScorer()) + ); + + /** The minimum confidence interval */ + private static final float MINIMUM_CONFIDENCE_INTERVAL = 0.9f; + + /** The maximum confidence interval */ + private static final float MAXIMUM_CONFIDENCE_INTERVAL = 1f; + + private final FlatVectorsFormat rawVectorFormat; + + /** + * Controls the confidence interval used to scalar quantize the vectors the default value is + * calculated as `1-1/(vector_dimensions + 1)` + */ + public final Float confidenceInterval; + + private final byte bits; + private final boolean compress; + + public ES93ScalarQuantizedVectorsFormat( + boolean useBFloat16, + Float confidenceInterval, + int bits, + boolean compress, + boolean useDirectIO + ) { + super(NAME); + if (confidenceInterval != null + && confidenceInterval != DYNAMIC_CONFIDENCE_INTERVAL + && (confidenceInterval < MINIMUM_CONFIDENCE_INTERVAL || confidenceInterval > MAXIMUM_CONFIDENCE_INTERVAL)) { + throw new IllegalArgumentException( + "confidenceInterval must be between " + + MINIMUM_CONFIDENCE_INTERVAL + + " and " + + MAXIMUM_CONFIDENCE_INTERVAL + + "; confidenceInterval=" + + confidenceInterval + ); + } + if (bits < 1 || bits > 8 || (ALLOWED_BITS & (1 << bits)) == 0) { + throw new IllegalArgumentException("bits must be one of: 4, 7, 8; bits=" + bits); + } + this.confidenceInterval = confidenceInterval; + this.bits = (byte) bits; + this.compress = compress; + this.rawVectorFormat = new ES93GenericFlatVectorsFormat(useBFloat16, useDirectIO); + } + + @Override + public int getMaxDimensions(String fieldName) { + return MAX_DIMS_COUNT; + } + + @Override + public String toString() { + return NAME + + "(name=" + + NAME + + ", confidenceInterval=" + + confidenceInterval + + ", bits=" + + bits + + ", compressed=" + + compress + + ", flatVectorScorer=" + + flatVectorScorer + + ", rawVectorFormat=" + + rawVectorFormat + + ")"; + } + + @Override + public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return new Lucene99ScalarQuantizedVectorsWriter( + state, + confidenceInterval, + bits, + compress, + rawVectorFormat.fieldsWriter(state), + flatVectorScorer + ); + } + + @Override + public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException { + return new Lucene99ScalarQuantizedVectorsReader(state, rawVectorFormat.fieldsReader(state), flatVectorScorer); + } + + static final class ESFlatVectorsScorer implements FlatVectorsScorer { + + final FlatVectorsScorer delegate; + final VectorScorerFactory factory; + + ESFlatVectorsScorer(FlatVectorsScorer delegate) { + this.delegate = delegate; + factory = VectorScorerFactory.instance().orElse(null); + } + + @Override + public String toString() { + return "ESFlatVectorsScorer(" + "delegate=" + delegate + ", factory=" + factory + ')'; + } + + @Override + public RandomVectorScorerSupplier getRandomVectorScorerSupplier(VectorSimilarityFunction sim, KnnVectorValues values) + throws IOException { + if (values instanceof QuantizedByteVectorValues qValues && qValues.getSlice() != null) { + // TODO: optimize int4 quantization + if (qValues.getScalarQuantizer().getBits() != 7) { + return delegate.getRandomVectorScorerSupplier(sim, values); + } + if (factory != null) { + var scorer = factory.getInt7SQVectorScorerSupplier( + VectorSimilarityType.of(sim), + qValues.getSlice(), + qValues, + qValues.getScalarQuantizer().getConstantMultiplier() + ); + if (scorer.isPresent()) { + return scorer.get(); + } + } + } + return delegate.getRandomVectorScorerSupplier(sim, values); + } + + @Override + public RandomVectorScorer getRandomVectorScorer(VectorSimilarityFunction sim, KnnVectorValues values, float[] query) + throws IOException { + if (values instanceof QuantizedByteVectorValues qValues && qValues.getSlice() != null) { + // TODO: optimize int4 quantization + if (qValues.getScalarQuantizer().getBits() != 7) { + return delegate.getRandomVectorScorer(sim, values, query); + } + if (factory != null) { + var scorer = factory.getInt7SQVectorScorer(sim, qValues, query); + if (scorer.isPresent()) { + return scorer.get(); + } + } + } + return delegate.getRandomVectorScorer(sim, values, query); + } + + @Override + public RandomVectorScorer getRandomVectorScorer(VectorSimilarityFunction sim, KnnVectorValues values, byte[] query) + throws IOException { + return delegate.getRandomVectorScorer(sim, values, query); + } + } +} diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat index 6c21437d71d28..8e8e17f4681ec 100644 --- a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat +++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat @@ -9,5 +9,6 @@ org.elasticsearch.index.codec.vectors.es818.ES818BinaryQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat org.elasticsearch.index.codec.vectors.diskbbq.next.ESNextDiskBBQVectorsFormat +org.elasticsearch.index.codec.vectors.es93.ES93HnswScalarQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.es93.ES93HnswBinaryQuantizedVectorsFormat diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedBFloat16VectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedBFloat16VectorsFormatTests.java new file mode 100644 index 0000000000000..5688ede94e1ec --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedBFloat16VectorsFormatTests.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.index.VectorEncoding; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.hamcrest.Matchers.closeTo; + +public class ES93HnswScalarQuantizedBFloat16VectorsFormatTests extends ES93HnswScalarQuantizedVectorsFormatTests { + @Override + boolean useBFloat16() { + return true; + } + + @Override + protected VectorEncoding randomVectorEncoding() { + return VectorEncoding.FLOAT32; + } + + @Override + public void testEmptyByteVectorData() throws Exception { + // no bytes + } + + @Override + public void testMergingWithDifferentByteKnnFields() throws Exception { + // no bytes + } + + @Override + public void testByteVectorScorerIteration() throws Exception { + // no bytes + } + + @Override + public void testSortedIndexBytes() throws Exception { + // no bytes + } + + @Override + public void testMismatchedFields() throws Exception { + // no bytes + } + + @Override + public void testRandomBytes() throws Exception { + // no bytes + } + + @Override + public void testWriterRamEstimate() throws Exception { + // estimate is different due to bfloat16 + } + + @Override + public void testRandom() throws Exception { + AssertionError err = expectThrows(AssertionError.class, super::testRandom); + assertFloatsWithinBounds(err); + } + + @Override + public void testRandomWithUpdatesAndGraph() throws Exception { + AssertionError err = expectThrows(AssertionError.class, super::testRandomWithUpdatesAndGraph); + assertFloatsWithinBounds(err); + } + + @Override + public void testSparseVectors() throws Exception { + AssertionError err = expectThrows(AssertionError.class, super::testSparseVectors); + assertFloatsWithinBounds(err); + } + + @Override + public void testVectorValuesReportCorrectDocs() throws Exception { + AssertionError err = expectThrows(AssertionError.class, super::testVectorValuesReportCorrectDocs); + assertFloatsWithinBounds(err); + } + + private static final Pattern FLOAT_ASSERTION_FAILURE = Pattern.compile(".*expected:<([0-9.-]+)> but was:<([0-9.-]+)>"); + + private static void assertFloatsWithinBounds(AssertionError error) { + Matcher m = FLOAT_ASSERTION_FAILURE.matcher(error.getMessage()); + if (m.matches() == false) { + throw error; // nothing to do with us, just rethrow + } + + // numbers just need to be in the same vicinity + double expected = Double.parseDouble(m.group(1)); + double actual = Double.parseDouble(m.group(2)); + double allowedError = expected * 0.01; // within 1% + assertThat(error.getMessage(), actual, closeTo(expected, allowedError)); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormatTests.java new file mode 100644 index 0000000000000..a6c12548821b9 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormatTests.java @@ -0,0 +1,220 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.CodecReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.KnnVectorValues; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.StoredFields; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.AcceptDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.MMapDirectory; +import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; +import org.apache.lucene.tests.util.TestUtil; +import org.elasticsearch.common.logging.LogConfigurator; +import org.elasticsearch.index.codec.vectors.BFloat16; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; +import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT; +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; + +public class ES93HnswScalarQuantizedVectorsFormatTests extends BaseKnnVectorsFormatTestCase { + + static { + LogConfigurator.loadLog4jPlugins(); + LogConfigurator.configureESLogging(); // native access requires logging to be initialized + } + + boolean useBFloat16() { + return false; + } + + @Override + protected Codec getCodec() { + return TestUtil.alwaysKnnVectorsFormat( + new ES93HnswScalarQuantizedVectorsFormat(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, useBFloat16(), null, 7, false, false) + ); + } + + // The following test scenarios are similar to their superclass namesakes, + // but here we ensure that the Directory implementation is a FSDirectory + // which helps test the native code vector distance implementation + + public void testAddIndexesDirectory0FS() throws Exception { + Path root = createTempDir(); + String fieldName = "field"; + Document doc = new Document(); + doc.add(new KnnFloatVectorField(fieldName, new float[4], VectorSimilarityFunction.DOT_PRODUCT)); + try (Directory dir = new MMapDirectory(root.resolve("dir1")); Directory dir2 = new MMapDirectory(root.resolve("dir2"))) { + try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + w.addDocument(doc); + } + try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) { + w2.addIndexes(dir); + w2.forceMerge(1); + try (IndexReader reader = DirectoryReader.open(w2)) { + LeafReader r = getOnlyLeafReader(reader); + FloatVectorValues vectorValues = r.getFloatVectorValues(fieldName); + KnnVectorValues.DocIndexIterator iterator = vectorValues.iterator(); + assertEquals(0, iterator.nextDoc()); + assertEquals(0, vectorValues.vectorValue(iterator.index())[0], 0); + assertEquals(NO_MORE_DOCS, iterator.nextDoc()); + } + } + } + } + + public void testAddIndexesDirectory01FSCosine() throws Exception { + testAddIndexesDirectory01FS(VectorSimilarityFunction.COSINE); + } + + public void testAddIndexesDirectory01FSDot() throws Exception { + testAddIndexesDirectory01FS(VectorSimilarityFunction.DOT_PRODUCT); + } + + public void testAddIndexesDirectory01FSEuclidean() throws Exception { + testAddIndexesDirectory01FS(VectorSimilarityFunction.EUCLIDEAN); + } + + public void testAddIndexesDirectory01FSMaxIP() throws Exception { + testAddIndexesDirectory01FS(VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT); + } + + private void testAddIndexesDirectory01FS(VectorSimilarityFunction similarityFunction) throws Exception { + Path root = createTempDir(); + String fieldName = "field"; + float[] vector = new float[] { 1f }; + Document doc = new Document(); + doc.add(new KnnFloatVectorField(fieldName, vector, similarityFunction)); + try (Directory dir = new MMapDirectory(root.resolve("dir1")); Directory dir2 = new MMapDirectory(root.resolve("dir2"))) { + try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + w.addDocument(doc); + } + try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) { + vector[0] = 2f; + w2.addDocument(doc); + w2.addIndexes(dir); + w2.forceMerge(1); + try (IndexReader reader = DirectoryReader.open(w2)) { + LeafReader r = getOnlyLeafReader(reader); + FloatVectorValues vectorValues = r.getFloatVectorValues(fieldName); + KnnVectorValues.DocIndexIterator iterator = vectorValues.iterator(); + assertEquals(0, iterator.nextDoc()); + // The merge order is randomized, we might get 1 first, or 2 + float value = vectorValues.vectorValue(iterator.index())[0]; + assertTrue(value == 1 || value == 2); + assertEquals(1, iterator.nextDoc()); + value += vectorValues.vectorValue(iterator.index())[0]; + assertEquals(3f, value, 0); + } + } + } + } + + public void testSingleVectorPerSegmentCosine() throws Exception { + testSingleVectorPerSegment(VectorSimilarityFunction.COSINE); + } + + public void testSingleVectorPerSegmentDot() throws Exception { + testSingleVectorPerSegment(VectorSimilarityFunction.DOT_PRODUCT); + } + + public void testSingleVectorPerSegmentEuclidean() throws Exception { + testSingleVectorPerSegment(VectorSimilarityFunction.EUCLIDEAN); + } + + public void testSingleVectorPerSegmentMIP() throws Exception { + testSingleVectorPerSegment(VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT); + } + + private void testSingleVectorPerSegment(VectorSimilarityFunction sim) throws Exception { + var codec = getCodec(); + try (Directory dir = new MMapDirectory(createTempDir().resolve("dir1"))) { + try (IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig().setCodec(codec))) { + Document doc2 = new Document(); + doc2.add(new KnnFloatVectorField("field", new float[] { 0.8f, 0.6f }, sim)); + doc2.add(newTextField("id", "A", Field.Store.YES)); + writer.addDocument(doc2); + writer.commit(); + + Document doc1 = new Document(); + doc1.add(new KnnFloatVectorField("field", new float[] { 0.6f, 0.8f }, sim)); + doc1.add(newTextField("id", "B", Field.Store.YES)); + writer.addDocument(doc1); + writer.commit(); + + Document doc3 = new Document(); + doc3.add(new KnnFloatVectorField("field", new float[] { -0.6f, -0.8f }, sim)); + doc3.add(newTextField("id", "C", Field.Store.YES)); + writer.addDocument(doc3); + writer.commit(); + + writer.forceMerge(1); + } + try (DirectoryReader reader = DirectoryReader.open(dir)) { + LeafReader leafReader = getOnlyLeafReader(reader); + StoredFields storedFields = reader.storedFields(); + float[] queryVector = new float[] { 0.6f, 0.8f }; + var hits = leafReader.searchNearestVectors( + "field", + queryVector, + 3, + AcceptDocs.fromLiveDocs(leafReader.getLiveDocs(), leafReader.maxDoc()), + 100 + ); + assertEquals(hits.scoreDocs.length, 3); + assertEquals("B", storedFields.document(hits.scoreDocs[0].doc).get("id")); + assertEquals("A", storedFields.document(hits.scoreDocs[1].doc).get("id")); + assertEquals("C", storedFields.document(hits.scoreDocs[2].doc).get("id")); + } + } + } + + public void testSimpleOffHeapSize() throws IOException { + float[] vector = randomVector(random().nextInt(12, 500)); + try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + Document doc = new Document(); + doc.add(new KnnFloatVectorField("f", vector, DOT_PRODUCT)); + w.addDocument(doc); + w.commit(); + try (IndexReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + if (r instanceof CodecReader codecReader) { + KnnVectorsReader knnVectorsReader = codecReader.getVectorReader(); + if (knnVectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader fieldsReader) { + knnVectorsReader = fieldsReader.getFieldReader("f"); + } + var fieldInfo = r.getFieldInfos().fieldInfo("f"); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); + assertEquals(3, offHeap.size()); + int bytes = useBFloat16() ? BFloat16.BYTES : Float.BYTES; + assertEquals(vector.length * bytes, (long) offHeap.get("vec")); + assertEquals(1L, (long) offHeap.get("vex")); + assertTrue(offHeap.get("veq") > 0L); + } + } + } + } +} From 3a6f7fccdeeb90fc2b98646ca06992333f275013 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Wed, 15 Oct 2025 15:26:40 +0100 Subject: [PATCH 02/32] Add flat format --- server/src/main/java/module-info.java | 1 + .../vectors/es93/ES93FlatVectorFormat.java | 124 ++++++++++++++++++ .../org.apache.lucene.codecs.KnnVectorsFormat | 1 + .../ES93FlatBFloat16VectorFormatTests.java | 103 +++++++++++++++ .../es93/ES93FlatVectorFormatTests.java | 75 +++++++++++ 5 files changed, 304 insertions(+) create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormat.java create mode 100644 server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatBFloat16VectorFormatTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormatTests.java diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 234d43315a9d4..1d121c38d3c6c 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -464,6 +464,7 @@ org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsFormat, org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat, org.elasticsearch.index.codec.vectors.diskbbq.next.ESNextDiskBBQVectorsFormat, + org.elasticsearch.index.codec.vectors.es93.ES93FlatVectorFormat, org.elasticsearch.index.codec.vectors.es93.ES93HnswScalarQuantizedVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93HnswBinaryQuantizedVectorsFormat; diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormat.java new file mode 100644 index 0000000000000..bd421abf167f0 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormat.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.search.AcceptDocs; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector; +import org.apache.lucene.util.hnsw.RandomVectorScorer; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; + +public class ES93FlatVectorFormat extends KnnVectorsFormat { + + static final String NAME = "ES93FlatVectorFormat"; + + private final FlatVectorsFormat format; + + /** + * Sole constructor + */ + public ES93FlatVectorFormat() { + super(NAME); + format = new ES93GenericFlatVectorsFormat(); + } + + public ES93FlatVectorFormat(boolean useBFloat16) { + super(NAME); + format = new ES93GenericFlatVectorsFormat(useBFloat16, false); + } + + @Override + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return format.fieldsWriter(state); + } + + @Override + public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { + return new ES93FlatVectorReader(format.fieldsReader(state)); + } + + @Override + public int getMaxDimensions(String fieldName) { + return MAX_DIMS_COUNT; + } + + static class ES93FlatVectorReader extends KnnVectorsReader { + + private final FlatVectorsReader reader; + + ES93FlatVectorReader(FlatVectorsReader reader) { + super(); + this.reader = reader; + } + + @Override + public void checkIntegrity() throws IOException { + reader.checkIntegrity(); + } + + @Override + public FloatVectorValues getFloatVectorValues(String field) throws IOException { + return reader.getFloatVectorValues(field); + } + + @Override + public ByteVectorValues getByteVectorValues(String field) throws IOException { + return reader.getByteVectorValues(field); + } + + @Override + public void search(String field, float[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { + collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); + } + + private void collectAllMatchingDocs(KnnCollector knnCollector, AcceptDocs acceptDocs, RandomVectorScorer scorer) + throws IOException { + OrdinalTranslatedKnnCollector collector = new OrdinalTranslatedKnnCollector(knnCollector, scorer::ordToDoc); + Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs.bits()); + for (int i = 0; i < scorer.maxOrd(); i++) { + if (acceptedOrds == null || acceptedOrds.get(i)) { + collector.collect(i, scorer.score(i)); + collector.incVisitedCount(1); + } + } + assert collector.earlyTerminated() == false; + } + + @Override + public void search(String field, byte[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { + collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); + } + + @Override + public Map getOffHeapByteSize(FieldInfo fieldInfo) { + return reader.getOffHeapByteSize(fieldInfo); + } + + @Override + public void close() throws IOException { + reader.close(); + } + } +} diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat index 8e8e17f4681ec..8ca17b76098c4 100644 --- a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat +++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat @@ -9,6 +9,7 @@ org.elasticsearch.index.codec.vectors.es818.ES818BinaryQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat org.elasticsearch.index.codec.vectors.diskbbq.next.ESNextDiskBBQVectorsFormat +org.elasticsearch.index.codec.vectors.es93.ES93FlatVectorFormat org.elasticsearch.index.codec.vectors.es93.ES93HnswScalarQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.es93.ES93HnswBinaryQuantizedVectorsFormat diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatBFloat16VectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatBFloat16VectorFormatTests.java new file mode 100644 index 0000000000000..4d017daa2724c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatBFloat16VectorFormatTests.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.index.VectorEncoding; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.hamcrest.Matchers.closeTo; + +public class ES93FlatBFloat16VectorFormatTests extends ES93FlatVectorFormatTests { + @Override + boolean useBFloat16() { + return true; + } + + @Override + protected VectorEncoding randomVectorEncoding() { + return VectorEncoding.FLOAT32; + } + + @Override + public void testEmptyByteVectorData() throws Exception { + // no bytes + } + + @Override + public void testMergingWithDifferentByteKnnFields() throws Exception { + // no bytes + } + + @Override + public void testByteVectorScorerIteration() throws Exception { + // no bytes + } + + @Override + public void testSortedIndexBytes() throws Exception { + // no bytes + } + + @Override + public void testMismatchedFields() throws Exception { + // no bytes + } + + @Override + public void testRandomBytes() throws Exception { + // no bytes + } + + @Override + public void testWriterRamEstimate() throws Exception { + // estimate is different due to bfloat16 + } + + @Override + public void testRandom() throws Exception { + AssertionError err = expectThrows(AssertionError.class, super::testRandom); + assertFloatsWithinBounds(err); + } + + @Override + public void testRandomWithUpdatesAndGraph() throws Exception { + AssertionError err = expectThrows(AssertionError.class, super::testRandomWithUpdatesAndGraph); + assertFloatsWithinBounds(err); + } + + @Override + public void testSparseVectors() throws Exception { + AssertionError err = expectThrows(AssertionError.class, super::testSparseVectors); + assertFloatsWithinBounds(err); + } + + @Override + public void testVectorValuesReportCorrectDocs() throws Exception { + AssertionError err = expectThrows(AssertionError.class, super::testVectorValuesReportCorrectDocs); + assertFloatsWithinBounds(err); + } + + private static final Pattern FLOAT_ASSERTION_FAILURE = Pattern.compile(".*expected:<([0-9.-]+)> but was:<([0-9.-]+)>"); + + private static void assertFloatsWithinBounds(AssertionError error) { + Matcher m = FLOAT_ASSERTION_FAILURE.matcher(error.getMessage()); + if (m.matches() == false) { + throw error; // nothing to do with us, just rethrow + } + + // numbers just need to be in the same vicinity + double expected = Double.parseDouble(m.group(1)); + double actual = Double.parseDouble(m.group(2)); + double allowedError = expected * 0.01; // within 1% + assertThat(error.getMessage(), actual, closeTo(expected, allowedError)); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormatTests.java new file mode 100644 index 0000000000000..f3faa98124a66 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormatTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.CodecReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; +import org.apache.lucene.tests.util.TestUtil; +import org.elasticsearch.common.logging.LogConfigurator; +import org.elasticsearch.index.codec.vectors.BFloat16; + +import java.io.IOException; + +import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT; + +public class ES93FlatVectorFormatTests extends BaseKnnVectorsFormatTestCase { + + static { + LogConfigurator.loadLog4jPlugins(); + LogConfigurator.configureESLogging(); // native access requires logging to be initialized + } + + boolean useBFloat16() { + return false; + } + + @Override + protected Codec getCodec() { + return TestUtil.alwaysKnnVectorsFormat(new ES93FlatVectorFormat(useBFloat16())); + } + + public void testSearchWithVisitedLimit() { + // requires graph-based vector codec + } + + public void testSimpleOffHeapSize() throws IOException { + float[] vector = randomVector(random().nextInt(12, 500)); + try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + Document doc = new Document(); + doc.add(new KnnFloatVectorField("f", vector, DOT_PRODUCT)); + w.addDocument(doc); + w.commit(); + try (IndexReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + if (r instanceof CodecReader codecReader) { + KnnVectorsReader knnVectorsReader = codecReader.getVectorReader(); + if (knnVectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader fieldsReader) { + knnVectorsReader = fieldsReader.getFieldReader("f"); + } + var fieldInfo = r.getFieldInfos().fieldInfo("f"); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); + int bytes = useBFloat16() ? BFloat16.BYTES : Float.BYTES; + assertEquals(vector.length * bytes, (long) offHeap.get("vec")); + assertEquals(1, offHeap.size()); + } + } + } + } +} From a85b7ca8faa7ab91a95775ed45b6e959f2a12481 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Thu, 16 Oct 2025 09:21:23 +0100 Subject: [PATCH 03/32] Add int8 implementation --- server/src/main/java/module-info.java | 3 +- .../es93/ES93Int8FlatVectorFormat.java | 129 ++++++++++++++++++ .../org.apache.lucene.codecs.KnnVectorsFormat | 1 + ...ES93Int8FlatBFloat16VectorFormatTests.java | 103 ++++++++++++++ .../es93/ES93Int8FlatVectorFormatTests.java | 77 +++++++++++ 5 files changed, 312 insertions(+), 1 deletion(-) create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatVectorFormat.java create mode 100644 server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatBFloat16VectorFormatTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatVectorFormatTests.java diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 1d121c38d3c6c..a9dc2b8d645f6 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -465,6 +465,7 @@ org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat, org.elasticsearch.index.codec.vectors.diskbbq.next.ESNextDiskBBQVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93FlatVectorFormat, + org.elasticsearch.index.codec.vectors.es93.ES93Int8FlatVectorFormat, org.elasticsearch.index.codec.vectors.es93.ES93HnswScalarQuantizedVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93HnswBinaryQuantizedVectorsFormat; @@ -496,6 +497,6 @@ exports org.elasticsearch.inference.telemetry; exports org.elasticsearch.index.codec.vectors.diskbbq to org.elasticsearch.test.knn; exports org.elasticsearch.index.codec.vectors.cluster to org.elasticsearch.test.knn; + exports org.elasticsearch.index.codec.vectors.es93 to org.elasticsearch.test.knn; exports org.elasticsearch.search.crossproject; - exports org.elasticsearch.index.codec.vectors.es93 to org.elasticsearch.gpu, org.elasticsearch.test.knn; } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatVectorFormat.java new file mode 100644 index 0000000000000..43cd21fd04b87 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatVectorFormat.java @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.search.AcceptDocs; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector; +import org.apache.lucene.util.hnsw.RandomVectorScorer; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; + +public class ES93Int8FlatVectorFormat extends KnnVectorsFormat { + + static final String NAME = "ES93Int8FlatVectorFormat"; + + private final FlatVectorsFormat format; + + public ES93Int8FlatVectorFormat() { + this(false, null, 7, false); + } + + public ES93Int8FlatVectorFormat(boolean useBFloat16) { + this(useBFloat16, null, 7, false); + } + + public ES93Int8FlatVectorFormat(boolean useBFloat16, Float confidenceInterval, int bits, boolean compress) { + super(NAME); + this.format = new ES93ScalarQuantizedVectorsFormat(useBFloat16, confidenceInterval, bits, compress, false); + } + + @Override + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return format.fieldsWriter(state); + } + + @Override + public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { + return new ES813FlatVectorReader(format.fieldsReader(state)); + } + + @Override + public int getMaxDimensions(String fieldName) { + return MAX_DIMS_COUNT; + } + + @Override + public String toString() { + return NAME + "(name=" + NAME + ", innerFormat=" + format + ")"; + } + + public static class ES813FlatVectorReader extends KnnVectorsReader { + + private final FlatVectorsReader reader; + + public ES813FlatVectorReader(FlatVectorsReader reader) { + super(); + this.reader = reader; + } + + @Override + public void checkIntegrity() throws IOException { + reader.checkIntegrity(); + } + + @Override + public FloatVectorValues getFloatVectorValues(String field) throws IOException { + return reader.getFloatVectorValues(field); + } + + @Override + public ByteVectorValues getByteVectorValues(String field) throws IOException { + return reader.getByteVectorValues(field); + } + + @Override + public void search(String field, float[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { + collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); + } + + private void collectAllMatchingDocs(KnnCollector knnCollector, AcceptDocs acceptDocs, RandomVectorScorer scorer) + throws IOException { + OrdinalTranslatedKnnCollector collector = new OrdinalTranslatedKnnCollector(knnCollector, scorer::ordToDoc); + Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs.bits()); + for (int i = 0; i < scorer.maxOrd(); i++) { + if (acceptedOrds == null || acceptedOrds.get(i)) { + collector.collect(i, scorer.score(i)); + collector.incVisitedCount(1); + } + } + assert collector.earlyTerminated() == false; + } + + @Override + public void search(String field, byte[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { + collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); + } + + @Override + public Map getOffHeapByteSize(FieldInfo fieldInfo) { + return reader.getOffHeapByteSize(fieldInfo); + } + + @Override + public void close() throws IOException { + reader.close(); + } + } +} diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat index 8ca17b76098c4..4944b734ea018 100644 --- a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat +++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat @@ -10,6 +10,7 @@ org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsForma org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat org.elasticsearch.index.codec.vectors.diskbbq.next.ESNextDiskBBQVectorsFormat org.elasticsearch.index.codec.vectors.es93.ES93FlatVectorFormat +org.elasticsearch.index.codec.vectors.es93.ES93Int8FlatVectorFormat org.elasticsearch.index.codec.vectors.es93.ES93HnswScalarQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.es93.ES93HnswBinaryQuantizedVectorsFormat diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatBFloat16VectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatBFloat16VectorFormatTests.java new file mode 100644 index 0000000000000..98d60e4723e28 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatBFloat16VectorFormatTests.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.index.VectorEncoding; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.hamcrest.Matchers.closeTo; + +public class ES93Int8FlatBFloat16VectorFormatTests extends ES93Int8FlatVectorFormatTests { + @Override + boolean useBFloat16() { + return true; + } + + @Override + protected VectorEncoding randomVectorEncoding() { + return VectorEncoding.FLOAT32; + } + + @Override + public void testEmptyByteVectorData() throws Exception { + // no bytes + } + + @Override + public void testMergingWithDifferentByteKnnFields() throws Exception { + // no bytes + } + + @Override + public void testByteVectorScorerIteration() throws Exception { + // no bytes + } + + @Override + public void testSortedIndexBytes() throws Exception { + // no bytes + } + + @Override + public void testMismatchedFields() throws Exception { + // no bytes + } + + @Override + public void testRandomBytes() throws Exception { + // no bytes + } + + @Override + public void testWriterRamEstimate() throws Exception { + // estimate is different due to bfloat16 + } + + @Override + public void testRandom() throws Exception { + AssertionError err = expectThrows(AssertionError.class, super::testRandom); + assertFloatsWithinBounds(err); + } + + @Override + public void testRandomWithUpdatesAndGraph() throws Exception { + AssertionError err = expectThrows(AssertionError.class, super::testRandomWithUpdatesAndGraph); + assertFloatsWithinBounds(err); + } + + @Override + public void testSparseVectors() throws Exception { + AssertionError err = expectThrows(AssertionError.class, super::testSparseVectors); + assertFloatsWithinBounds(err); + } + + @Override + public void testVectorValuesReportCorrectDocs() throws Exception { + AssertionError err = expectThrows(AssertionError.class, super::testVectorValuesReportCorrectDocs); + assertFloatsWithinBounds(err); + } + + private static final Pattern FLOAT_ASSERTION_FAILURE = Pattern.compile(".*expected:<([0-9.-]+)> but was:<([0-9.-]+)>"); + + private static void assertFloatsWithinBounds(AssertionError error) { + Matcher m = FLOAT_ASSERTION_FAILURE.matcher(error.getMessage()); + if (m.matches() == false) { + throw error; // nothing to do with us, just rethrow + } + + // numbers just need to be in the same vicinity + double expected = Double.parseDouble(m.group(1)); + double actual = Double.parseDouble(m.group(2)); + double allowedError = expected * 0.01; // within 1% + assertThat(error.getMessage(), actual, closeTo(expected, allowedError)); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatVectorFormatTests.java new file mode 100644 index 0000000000000..a120b1be50556 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatVectorFormatTests.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.vectors.es93; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.CodecReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; +import org.apache.lucene.tests.util.TestUtil; +import org.elasticsearch.common.logging.LogConfigurator; +import org.elasticsearch.index.codec.vectors.BFloat16; + +import java.io.IOException; + +import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT; + +public class ES93Int8FlatVectorFormatTests extends BaseKnnVectorsFormatTestCase { + + static { + LogConfigurator.loadLog4jPlugins(); + LogConfigurator.configureESLogging(); // native access requires logging to be initialized + } + + boolean useBFloat16() { + return false; + } + + @Override + protected Codec getCodec() { + return TestUtil.alwaysKnnVectorsFormat(new ES93Int8FlatVectorFormat(useBFloat16())); + } + + public void testSearchWithVisitedLimit() { + // requires graph vector codec + } + + public void testSimpleOffHeapSize() throws IOException { + float[] vector = randomVector(random().nextInt(12, 500)); + try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + Document doc = new Document(); + doc.add(new KnnFloatVectorField("f", vector, DOT_PRODUCT)); + w.addDocument(doc); + w.commit(); + try (IndexReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + if (r instanceof CodecReader codecReader) { + KnnVectorsReader knnVectorsReader = codecReader.getVectorReader(); + if (knnVectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader fieldsReader) { + knnVectorsReader = fieldsReader.getFieldReader("f"); + } + var fieldInfo = r.getFieldInfos().fieldInfo("f"); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); + assertEquals(2, offHeap.size()); + int bytes = useBFloat16() ? BFloat16.BYTES : Float.BYTES; + assertEquals(vector.length * bytes, (long) offHeap.get("vec")); + assertTrue(offHeap.get("veq") > 0L); + } + } + } + } + +} From a638b5993d1e5056b5f02818b1a04fd25991d55b Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Mon, 20 Oct 2025 11:37:33 +0100 Subject: [PATCH 04/32] Rename class --- server/src/main/java/module-info.java | 3 ++- ...=> ES93ScalarQuantizedFlatVectorsFormat.java} | 16 ++++++++-------- .../org.apache.lucene.codecs.KnnVectorsFormat | 2 +- ...rQuantizedFlatBFloat16VectorFormatTests.java} | 2 +- ...93ScalarQuantizedFlatVectorsFormatTests.java} | 4 ++-- 5 files changed, 14 insertions(+), 13 deletions(-) rename server/src/main/java/org/elasticsearch/index/codec/vectors/es93/{ES93Int8FlatVectorFormat.java => ES93ScalarQuantizedFlatVectorsFormat.java} (87%) rename server/src/test/java/org/elasticsearch/index/codec/vectors/es93/{ES93Int8FlatBFloat16VectorFormatTests.java => ES93ScalarQuantizedFlatBFloat16VectorFormatTests.java} (96%) rename server/src/test/java/org/elasticsearch/index/codec/vectors/es93/{ES93Int8FlatVectorFormatTests.java => ES93ScalarQuantizedFlatVectorsFormatTests.java} (93%) diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index a9dc2b8d645f6..fb4ec153ba635 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -7,6 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ +import org.elasticsearch.index.codec.vectors.es93.ES93ScalarQuantizedFlatVectorsFormat; import org.elasticsearch.plugins.internal.RestExtension; import org.elasticsearch.reservedstate.ReservedStateHandlerProvider; @@ -465,7 +466,7 @@ org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat, org.elasticsearch.index.codec.vectors.diskbbq.next.ESNextDiskBBQVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93FlatVectorFormat, - org.elasticsearch.index.codec.vectors.es93.ES93Int8FlatVectorFormat, + ES93ScalarQuantizedFlatVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93HnswScalarQuantizedVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93HnswBinaryQuantizedVectorsFormat; diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java similarity index 87% rename from server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatVectorFormat.java rename to server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java index 43cd21fd04b87..4bd80005bfa12 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatVectorFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java @@ -30,21 +30,21 @@ import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; -public class ES93Int8FlatVectorFormat extends KnnVectorsFormat { +public class ES93ScalarQuantizedFlatVectorsFormat extends KnnVectorsFormat { - static final String NAME = "ES93Int8FlatVectorFormat"; + static final String NAME = "ES93ScalarQuantizedFlatVectorsFormat"; private final FlatVectorsFormat format; - public ES93Int8FlatVectorFormat() { + public ES93ScalarQuantizedFlatVectorsFormat() { this(false, null, 7, false); } - public ES93Int8FlatVectorFormat(boolean useBFloat16) { + public ES93ScalarQuantizedFlatVectorsFormat(boolean useBFloat16) { this(useBFloat16, null, 7, false); } - public ES93Int8FlatVectorFormat(boolean useBFloat16, Float confidenceInterval, int bits, boolean compress) { + public ES93ScalarQuantizedFlatVectorsFormat(boolean useBFloat16, Float confidenceInterval, int bits, boolean compress) { super(NAME); this.format = new ES93ScalarQuantizedVectorsFormat(useBFloat16, confidenceInterval, bits, compress, false); } @@ -56,7 +56,7 @@ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException @Override public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { - return new ES813FlatVectorReader(format.fieldsReader(state)); + return new ES93FlatVectorsReader(format.fieldsReader(state)); } @Override @@ -69,11 +69,11 @@ public String toString() { return NAME + "(name=" + NAME + ", innerFormat=" + format + ")"; } - public static class ES813FlatVectorReader extends KnnVectorsReader { + public static class ES93FlatVectorsReader extends KnnVectorsReader { private final FlatVectorsReader reader; - public ES813FlatVectorReader(FlatVectorsReader reader) { + public ES93FlatVectorsReader(FlatVectorsReader reader) { super(); this.reader = reader; } diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat index 4944b734ea018..2dcbd21c86436 100644 --- a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat +++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat @@ -10,7 +10,7 @@ org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsForma org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat org.elasticsearch.index.codec.vectors.diskbbq.next.ESNextDiskBBQVectorsFormat org.elasticsearch.index.codec.vectors.es93.ES93FlatVectorFormat -org.elasticsearch.index.codec.vectors.es93.ES93Int8FlatVectorFormat +org.elasticsearch.index.codec.vectors.es93.ES93ScalarQuantizedFlatVectorsFormat org.elasticsearch.index.codec.vectors.es93.ES93HnswScalarQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.es93.ES93HnswBinaryQuantizedVectorsFormat diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatBFloat16VectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatBFloat16VectorFormatTests.java similarity index 96% rename from server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatBFloat16VectorFormatTests.java rename to server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatBFloat16VectorFormatTests.java index 98d60e4723e28..3681d0a1299eb 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatBFloat16VectorFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatBFloat16VectorFormatTests.java @@ -16,7 +16,7 @@ import static org.hamcrest.Matchers.closeTo; -public class ES93Int8FlatBFloat16VectorFormatTests extends ES93Int8FlatVectorFormatTests { +public class ES93ScalarQuantizedFlatBFloat16VectorFormatTests extends ES93ScalarQuantizedFlatVectorsFormatTests { @Override boolean useBFloat16() { return true; diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormatTests.java similarity index 93% rename from server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatVectorFormatTests.java rename to server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormatTests.java index a120b1be50556..a55147f6e5d9f 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93Int8FlatVectorFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormatTests.java @@ -29,7 +29,7 @@ import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT; -public class ES93Int8FlatVectorFormatTests extends BaseKnnVectorsFormatTestCase { +public class ES93ScalarQuantizedFlatVectorsFormatTests extends BaseKnnVectorsFormatTestCase { static { LogConfigurator.loadLog4jPlugins(); @@ -42,7 +42,7 @@ boolean useBFloat16() { @Override protected Codec getCodec() { - return TestUtil.alwaysKnnVectorsFormat(new ES93Int8FlatVectorFormat(useBFloat16())); + return TestUtil.alwaysKnnVectorsFormat(new ES93ScalarQuantizedFlatVectorsFormat(useBFloat16())); } public void testSearchWithVisitedLimit() { From 419032d4360884ece4a157d80c8cbe99a441059a Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Mon, 20 Oct 2025 11:48:11 +0100 Subject: [PATCH 05/32] Improve tests --- .../ES93ScalarQuantizedFlatVectorsFormatTests.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormatTests.java index a55147f6e5d9f..d05f33941f7b3 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormatTests.java @@ -28,6 +28,10 @@ import java.io.IOException; import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasEntry; public class ES93ScalarQuantizedFlatVectorsFormatTests extends BaseKnnVectorsFormatTestCase { @@ -65,10 +69,9 @@ public void testSimpleOffHeapSize() throws IOException { } var fieldInfo = r.getFieldInfos().fieldInfo("f"); var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); - assertEquals(2, offHeap.size()); - int bytes = useBFloat16() ? BFloat16.BYTES : Float.BYTES; - assertEquals(vector.length * bytes, (long) offHeap.get("vec")); - assertTrue(offHeap.get("veq") > 0L); + assertThat(offHeap, aMapWithSize(2)); + assertThat(offHeap, hasEntry("vec", vector.length * (useBFloat16() ? BFloat16.BYTES : Float.BYTES))); + assertThat(offHeap, hasEntry(equalTo("veq"), greaterThan(0L))); } } } From 1287a0b1d09d94cd017fa82543d6bffd47de7739 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Mon, 20 Oct 2025 12:16:50 +0100 Subject: [PATCH 06/32] Fix module reference --- server/src/main/java/module-info.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 6cbf55407f174..7fec6ce36874d 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -7,7 +7,6 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import org.elasticsearch.index.codec.vectors.es93.ES93ScalarQuantizedFlatVectorsFormat; import org.elasticsearch.plugins.internal.RestExtension; import org.elasticsearch.reservedstate.ReservedStateHandlerProvider; @@ -467,7 +466,7 @@ org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat, org.elasticsearch.index.codec.vectors.diskbbq.next.ESNextDiskBBQVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93FlatVectorFormat, - ES93ScalarQuantizedFlatVectorsFormat, + org.elasticsearch.index.codec.vectors.es93.ES93ScalarQuantizedFlatVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93HnswScalarQuantizedVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93HnswBinaryQuantizedVectorsFormat; From 7c343c56c07271108e4ac405a46b637b5c730e3a Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Thu, 23 Oct 2025 09:52:32 +0100 Subject: [PATCH 07/32] Update to use new Lucene104 format --- .../ES814ScalarQuantizedVectorsFormat.java | 1 + .../Lucene99ScalarQuantizedVectorsReader.java | 455 ------------------ .../ES93HnswScalarQuantizedVectorsFormat.java | 31 +- .../ES93ScalarQuantizedFlatVectorsFormat.java | 9 +- .../ES93ScalarQuantizedVectorsFormat.java | 143 +----- ...arQuantizedBFloat16VectorsFormatTests.java | 131 +++-- ...HnswScalarQuantizedVectorsFormatTests.java | 237 ++------- 7 files changed, 147 insertions(+), 860 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/Lucene99ScalarQuantizedVectorsReader.java diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java index 692ccdfa9222c..dd125b68d292a 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.codec.vectors; +import org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsReader; import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/Lucene99ScalarQuantizedVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/Lucene99ScalarQuantizedVectorsReader.java deleted file mode 100644 index 9c0e855faf6ad..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/Lucene99ScalarQuantizedVectorsReader.java +++ /dev/null @@ -1,455 +0,0 @@ -/* - * @notice - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Modifications copyright (C) 2024 Elasticsearch B.V. - */ - -package org.elasticsearch.index.codec.vectors; - -import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.codecs.KnnVectorsReader; -import org.apache.lucene.codecs.hnsw.FlatVectorsReader; -import org.apache.lucene.codecs.hnsw.FlatVectorsScorer; -import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration; -import org.apache.lucene.index.ByteVectorValues; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.FloatVectorValues; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.VectorEncoding; -import org.apache.lucene.index.VectorSimilarityFunction; -import org.apache.lucene.internal.hppc.IntObjectHashMap; -import org.apache.lucene.search.VectorScorer; -import org.apache.lucene.store.ChecksumIndexInput; -import org.apache.lucene.store.DataAccessHint; -import org.apache.lucene.store.FileDataHint; -import org.apache.lucene.store.FileTypeHint; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.RamUsageEstimator; -import org.apache.lucene.util.hnsw.RandomVectorScorer; -import org.apache.lucene.util.quantization.QuantizedByteVectorValues; -import org.apache.lucene.util.quantization.QuantizedVectorsReader; -import org.apache.lucene.util.quantization.ScalarQuantizer; -import org.elasticsearch.core.SuppressForbidden; - -import java.io.IOException; -import java.util.Map; - -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding; - -/** - * Copied from Lucene 10.3. - */ -@SuppressForbidden(reason = "Lucene classes") -final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReader implements QuantizedVectorsReader { - - private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(Lucene99ScalarQuantizedVectorsReader.class); - - static final int VERSION_START = 0; - static final int VERSION_ADD_BITS = 1; - static final int VERSION_CURRENT = VERSION_ADD_BITS; - static final String META_CODEC_NAME = "Lucene99ScalarQuantizedVectorsFormatMeta"; - static final String VECTOR_DATA_CODEC_NAME = "Lucene99ScalarQuantizedVectorsFormatData"; - static final String META_EXTENSION = "vemq"; - static final String VECTOR_DATA_EXTENSION = "veq"; - - /** Dynamic confidence interval */ - public static final float DYNAMIC_CONFIDENCE_INTERVAL = 0f; - - private final IntObjectHashMap fields = new IntObjectHashMap<>(); - private final IndexInput quantizedVectorData; - private final FlatVectorsReader rawVectorsReader; - private final FieldInfos fieldInfos; - - Lucene99ScalarQuantizedVectorsReader(SegmentReadState state, FlatVectorsReader rawVectorsReader, FlatVectorsScorer scorer) - throws IOException { - super(scorer); - this.rawVectorsReader = rawVectorsReader; - this.fieldInfos = state.fieldInfos; - int versionMeta = -1; - String metaFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, META_EXTENSION); - boolean success = false; - try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName)) { - Throwable priorE = null; - try { - versionMeta = CodecUtil.checkIndexHeader( - meta, - META_CODEC_NAME, - VERSION_START, - VERSION_CURRENT, - state.segmentInfo.getId(), - state.segmentSuffix - ); - readFields(meta, versionMeta, state.fieldInfos); - } catch (Throwable exception) { - priorE = exception; - } finally { - CodecUtil.checkFooter(meta, priorE); - } - quantizedVectorData = openDataInput( - state, - versionMeta, - VECTOR_DATA_EXTENSION, - VECTOR_DATA_CODEC_NAME, - // Quantized vectors are accessed randomly from their node ID stored in the HNSW - // graph. - state.context.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM) - ); - success = true; - } finally { - if (success == false) { - IOUtils.closeWhileHandlingException(this); - } - } - } - - private void readFields(ChecksumIndexInput meta, int versionMeta, FieldInfos infos) throws IOException { - for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) { - FieldInfo info = infos.fieldInfo(fieldNumber); - if (info == null) { - throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta); - } - FieldEntry fieldEntry = readField(meta, versionMeta, info); - validateFieldEntry(info, fieldEntry); - fields.put(info.number, fieldEntry); - } - } - - static void validateFieldEntry(FieldInfo info, FieldEntry fieldEntry) { - int dimension = info.getVectorDimension(); - if (dimension != fieldEntry.dimension) { - throw new IllegalStateException( - "Inconsistent vector dimension for field=\"" + info.name + "\"; " + dimension + " != " + fieldEntry.dimension - ); - } - - final long quantizedVectorBytes; - if (fieldEntry.bits <= 4 && fieldEntry.compress) { - // two dimensions -> one byte - quantizedVectorBytes = ((dimension + 1) >> 1) + Float.BYTES; - } else { - // one dimension -> one byte - quantizedVectorBytes = dimension + Float.BYTES; - } - long numQuantizedVectorBytes = Math.multiplyExact(quantizedVectorBytes, fieldEntry.size); - if (numQuantizedVectorBytes != fieldEntry.vectorDataLength) { - throw new IllegalStateException( - "Quantized vector data length " - + fieldEntry.vectorDataLength - + " not matching size=" - + fieldEntry.size - + " * (dim=" - + dimension - + " + 4)" - + " = " - + numQuantizedVectorBytes - ); - } - } - - @Override - public void checkIntegrity() throws IOException { - rawVectorsReader.checkIntegrity(); - CodecUtil.checksumEntireFile(quantizedVectorData); - } - - private FieldEntry getFieldEntry(String field) { - final FieldInfo info = fieldInfos.fieldInfo(field); - final FieldEntry fieldEntry; - if (info == null || (fieldEntry = fields.get(info.number)) == null) { - throw new IllegalArgumentException("field=\"" + field + "\" not found"); - } - if (fieldEntry.vectorEncoding != VectorEncoding.FLOAT32) { - throw new IllegalArgumentException( - "field=\"" + field + "\" is encoded as: " + fieldEntry.vectorEncoding + " expected: " + VectorEncoding.FLOAT32 - ); - } - return fieldEntry; - } - - @Override - public FloatVectorValues getFloatVectorValues(String field) throws IOException { - final FieldEntry fieldEntry = getFieldEntry(field); - final FloatVectorValues rawVectorValues = rawVectorsReader.getFloatVectorValues(field); - OffHeapQuantizedByteVectorValues quantizedByteVectorValues = OffHeapQuantizedByteVectorValues.load( - fieldEntry.ordToDoc, - fieldEntry.dimension, - fieldEntry.size, - fieldEntry.scalarQuantizer, - fieldEntry.similarityFunction, - vectorScorer, - fieldEntry.compress, - fieldEntry.vectorDataOffset, - fieldEntry.vectorDataLength, - quantizedVectorData - ); - return new QuantizedVectorValues(rawVectorValues, quantizedByteVectorValues); - } - - @Override - public ByteVectorValues getByteVectorValues(String field) throws IOException { - return rawVectorsReader.getByteVectorValues(field); - } - - private static IndexInput openDataInput( - SegmentReadState state, - int versionMeta, - String fileExtension, - String codecName, - IOContext context - ) throws IOException { - String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, fileExtension); - IndexInput in = state.directory.openInput(fileName, context); - boolean success = false; - try { - int versionVectorData = CodecUtil.checkIndexHeader( - in, - codecName, - VERSION_START, - VERSION_CURRENT, - state.segmentInfo.getId(), - state.segmentSuffix - ); - if (versionMeta != versionVectorData) { - throw new CorruptIndexException( - "Format versions mismatch: meta=" + versionMeta + ", " + codecName + "=" + versionVectorData, - in - ); - } - CodecUtil.retrieveChecksum(in); - success = true; - return in; - } finally { - if (success == false) { - IOUtils.closeWhileHandlingException(in); - } - } - } - - @Override - public RandomVectorScorer getRandomVectorScorer(String field, float[] target) throws IOException { - final FieldEntry fieldEntry = getFieldEntry(field); - if (fieldEntry.scalarQuantizer == null) { - return rawVectorsReader.getRandomVectorScorer(field, target); - } - OffHeapQuantizedByteVectorValues vectorValues = OffHeapQuantizedByteVectorValues.load( - fieldEntry.ordToDoc, - fieldEntry.dimension, - fieldEntry.size, - fieldEntry.scalarQuantizer, - fieldEntry.similarityFunction, - vectorScorer, - fieldEntry.compress, - fieldEntry.vectorDataOffset, - fieldEntry.vectorDataLength, - quantizedVectorData - ); - return vectorScorer.getRandomVectorScorer(fieldEntry.similarityFunction, vectorValues, target); - } - - @Override - public RandomVectorScorer getRandomVectorScorer(String field, byte[] target) throws IOException { - return rawVectorsReader.getRandomVectorScorer(field, target); - } - - @Override - public void close() throws IOException { - IOUtils.close(quantizedVectorData, rawVectorsReader); - } - - @Override - public long ramBytesUsed() { - return SHALLOW_SIZE + fields.ramBytesUsed() + rawVectorsReader.ramBytesUsed(); - } - - @Override - public Map getOffHeapByteSize(FieldInfo fieldInfo) { - var raw = rawVectorsReader.getOffHeapByteSize(fieldInfo); - var fieldEntry = fields.get(fieldInfo.number); - if (fieldEntry == null) { - assert fieldInfo.getVectorEncoding() == VectorEncoding.BYTE; - return raw; - } - var quant = Map.of(VECTOR_DATA_EXTENSION, fieldEntry.vectorDataLength()); - return KnnVectorsReader.mergeOffHeapByteSizeMaps(raw, quant); - } - - private FieldEntry readField(IndexInput input, int versionMeta, FieldInfo info) throws IOException { - VectorEncoding vectorEncoding = readVectorEncoding(input); - VectorSimilarityFunction similarityFunction = readSimilarityFunction(input); - if (similarityFunction != info.getVectorSimilarityFunction()) { - throw new IllegalStateException( - "Inconsistent vector similarity function for field=\"" - + info.name - + "\"; " - + similarityFunction - + " != " - + info.getVectorSimilarityFunction() - ); - } - return FieldEntry.create(input, versionMeta, vectorEncoding, info.getVectorSimilarityFunction()); - } - - @Override - public QuantizedByteVectorValues getQuantizedVectorValues(String field) throws IOException { - final FieldEntry fieldEntry = getFieldEntry(field); - return OffHeapQuantizedByteVectorValues.load( - fieldEntry.ordToDoc, - fieldEntry.dimension, - fieldEntry.size, - fieldEntry.scalarQuantizer, - fieldEntry.similarityFunction, - vectorScorer, - fieldEntry.compress, - fieldEntry.vectorDataOffset, - fieldEntry.vectorDataLength, - quantizedVectorData - ); - } - - @Override - public ScalarQuantizer getQuantizationState(String field) { - final FieldEntry fieldEntry = getFieldEntry(field); - return fieldEntry.scalarQuantizer; - } - - private record FieldEntry( - VectorSimilarityFunction similarityFunction, - VectorEncoding vectorEncoding, - int dimension, - long vectorDataOffset, - long vectorDataLength, - ScalarQuantizer scalarQuantizer, - int size, - byte bits, - boolean compress, - OrdToDocDISIReaderConfiguration ordToDoc - ) { - - static FieldEntry create( - IndexInput input, - int versionMeta, - VectorEncoding vectorEncoding, - VectorSimilarityFunction similarityFunction - ) throws IOException { - final var vectorDataOffset = input.readVLong(); - final var vectorDataLength = input.readVLong(); - final var dimension = input.readVInt(); - final var size = input.readInt(); - final ScalarQuantizer scalarQuantizer; - final byte bits; - final boolean compress; - if (size > 0) { - if (versionMeta < VERSION_ADD_BITS) { - int floatBits = input.readInt(); // confidenceInterval, unused - if (floatBits == -1) { // indicates a null confidence interval - throw new CorruptIndexException("Missing confidence interval for scalar quantizer", input); - } - float confidenceInterval = Float.intBitsToFloat(floatBits); - // indicates a dynamic interval, which shouldn't be provided in this version - if (confidenceInterval == DYNAMIC_CONFIDENCE_INTERVAL) { - throw new CorruptIndexException("Invalid confidence interval for scalar quantizer: " + confidenceInterval, input); - } - bits = (byte) 7; - compress = false; - float minQuantile = Float.intBitsToFloat(input.readInt()); - float maxQuantile = Float.intBitsToFloat(input.readInt()); - scalarQuantizer = new ScalarQuantizer(minQuantile, maxQuantile, (byte) 7); - } else { - input.readInt(); // confidenceInterval, unused - bits = input.readByte(); - compress = input.readByte() == 1; - float minQuantile = Float.intBitsToFloat(input.readInt()); - float maxQuantile = Float.intBitsToFloat(input.readInt()); - scalarQuantizer = new ScalarQuantizer(minQuantile, maxQuantile, bits); - } - } else { - scalarQuantizer = null; - bits = (byte) 7; - compress = false; - } - final var ordToDoc = OrdToDocDISIReaderConfiguration.fromStoredMeta(input, size); - return new FieldEntry( - similarityFunction, - vectorEncoding, - dimension, - vectorDataOffset, - vectorDataLength, - scalarQuantizer, - size, - bits, - compress, - ordToDoc - ); - } - } - - private static final class QuantizedVectorValues extends FloatVectorValues { - private final FloatVectorValues rawVectorValues; - private final QuantizedByteVectorValues quantizedVectorValues; - - QuantizedVectorValues(FloatVectorValues rawVectorValues, QuantizedByteVectorValues quantizedVectorValues) { - this.rawVectorValues = rawVectorValues; - this.quantizedVectorValues = quantizedVectorValues; - } - - @Override - public int dimension() { - return rawVectorValues.dimension(); - } - - @Override - public int size() { - return rawVectorValues.size(); - } - - @Override - public float[] vectorValue(int ord) throws IOException { - return rawVectorValues.vectorValue(ord); - } - - @Override - public int ordToDoc(int ord) { - return rawVectorValues.ordToDoc(ord); - } - - @Override - public QuantizedVectorValues copy() throws IOException { - return new QuantizedVectorValues(rawVectorValues.copy(), quantizedVectorValues.copy()); - } - - @Override - public VectorScorer scorer(float[] query) throws IOException { - return quantizedVectorValues.scorer(query); - } - - @Override - public VectorScorer rescorer(float[] query) throws IOException { - return rawVectorValues.rescorer(query); - } - - @Override - public DocIndexIterator iterator() { - return rawVectorValues.iterator(); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java index fe5e2c5696371..a2e14df6099ec 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java @@ -12,6 +12,7 @@ import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.KnnVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat; import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter; import org.apache.lucene.index.SegmentReadState; @@ -19,9 +20,7 @@ import org.elasticsearch.index.codec.vectors.AbstractHnswVectorsFormat; import java.io.IOException; - -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; +import java.util.concurrent.ExecutorService; public class ES93HnswScalarQuantizedVectorsFormat extends AbstractHnswVectorsFormat { @@ -31,20 +30,36 @@ public class ES93HnswScalarQuantizedVectorsFormat extends AbstractHnswVectorsFor private final FlatVectorsFormat flatVectorsFormat; public ES93HnswScalarQuantizedVectorsFormat() { - this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, false, null, 7, false, false); + super(NAME); + this.flatVectorsFormat = new ES93ScalarQuantizedVectorsFormat( + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, + false, + false + ); } public ES93HnswScalarQuantizedVectorsFormat( int maxConn, int beamWidth, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding, boolean useBFloat16, - Float confidenceInterval, - int bits, - boolean compress, boolean useDirectIO ) { super(NAME, maxConn, beamWidth); - this.flatVectorsFormat = new ES93ScalarQuantizedVectorsFormat(useBFloat16, confidenceInterval, bits, compress, useDirectIO); + this.flatVectorsFormat = new ES93ScalarQuantizedVectorsFormat(encoding, useBFloat16, useDirectIO); + } + + public ES93HnswScalarQuantizedVectorsFormat( + int maxConn, + int beamWidth, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding, + boolean useBFloat16, + boolean useDirectIO, + int numMergeWorkers, + ExecutorService mergeExec + ) { + super(NAME, maxConn, beamWidth, numMergeWorkers, mergeExec); + this.flatVectorsFormat = new ES93ScalarQuantizedVectorsFormat(encoding, useBFloat16, useDirectIO); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java index 4bd80005bfa12..f1375cb5469e7 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java @@ -14,6 +14,7 @@ import org.apache.lucene.codecs.KnnVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat; import org.apache.lucene.index.ByteVectorValues; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FloatVectorValues; @@ -37,16 +38,16 @@ public class ES93ScalarQuantizedFlatVectorsFormat extends KnnVectorsFormat { private final FlatVectorsFormat format; public ES93ScalarQuantizedFlatVectorsFormat() { - this(false, null, 7, false); + this(false, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT); } public ES93ScalarQuantizedFlatVectorsFormat(boolean useBFloat16) { - this(useBFloat16, null, 7, false); + this(useBFloat16, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT); } - public ES93ScalarQuantizedFlatVectorsFormat(boolean useBFloat16, Float confidenceInterval, int bits, boolean compress) { + public ES93ScalarQuantizedFlatVectorsFormat(boolean useBFloat16, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding) { super(NAME); - this.format = new ES93ScalarQuantizedVectorsFormat(useBFloat16, confidenceInterval, bits, compress, false); + this.format = new ES93ScalarQuantizedVectorsFormat(encoding, useBFloat16, false); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java index f1dd1a283f88f..b393d91dd0c72 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java @@ -12,78 +12,36 @@ import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; import org.apache.lucene.codecs.hnsw.FlatVectorsReader; -import org.apache.lucene.codecs.hnsw.FlatVectorsScorer; import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; -import org.apache.lucene.codecs.hnsw.ScalarQuantizedVectorScorer; -import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsWriter; -import org.apache.lucene.index.KnnVectorValues; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorScorer; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsReader; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsWriter; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.VectorSimilarityFunction; -import org.apache.lucene.util.hnsw.RandomVectorScorer; -import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier; -import org.apache.lucene.util.quantization.QuantizedByteVectorValues; -import org.elasticsearch.simdvec.VectorScorerFactory; -import org.elasticsearch.simdvec.VectorSimilarityType; import java.io.IOException; -import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.DYNAMIC_CONFIDENCE_INTERVAL; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; public class ES93ScalarQuantizedVectorsFormat extends FlatVectorsFormat { static final String NAME = "ES93ScalarQuantizedVectorsFormat"; - private static final int ALLOWED_BITS = (1 << 8) | (1 << 7) | (1 << 4); - static final FlatVectorsScorer flatVectorScorer = new ESFlatVectorsScorer( - new ScalarQuantizedVectorScorer(FlatVectorScorerUtil.getLucene99FlatVectorsScorer()) + static final Lucene104ScalarQuantizedVectorScorer flatVectorScorer = new Lucene104ScalarQuantizedVectorScorer( + FlatVectorScorerUtil.getLucene99FlatVectorsScorer() ); - /** The minimum confidence interval */ - private static final float MINIMUM_CONFIDENCE_INTERVAL = 0.9f; - - /** The maximum confidence interval */ - private static final float MAXIMUM_CONFIDENCE_INTERVAL = 1f; - + private final Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding; private final FlatVectorsFormat rawVectorFormat; - /** - * Controls the confidence interval used to scalar quantize the vectors the default value is - * calculated as `1-1/(vector_dimensions + 1)` - */ - public final Float confidenceInterval; - - private final byte bits; - private final boolean compress; - public ES93ScalarQuantizedVectorsFormat( + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding, boolean useBFloat16, - Float confidenceInterval, - int bits, - boolean compress, boolean useDirectIO ) { super(NAME); - if (confidenceInterval != null - && confidenceInterval != DYNAMIC_CONFIDENCE_INTERVAL - && (confidenceInterval < MINIMUM_CONFIDENCE_INTERVAL || confidenceInterval > MAXIMUM_CONFIDENCE_INTERVAL)) { - throw new IllegalArgumentException( - "confidenceInterval must be between " - + MINIMUM_CONFIDENCE_INTERVAL - + " and " - + MAXIMUM_CONFIDENCE_INTERVAL - + "; confidenceInterval=" - + confidenceInterval - ); - } - if (bits < 1 || bits > 8 || (ALLOWED_BITS & (1 << bits)) == 0) { - throw new IllegalArgumentException("bits must be one of: 4, 7, 8; bits=" + bits); - } - this.confidenceInterval = confidenceInterval; - this.bits = (byte) bits; - this.compress = compress; + this.encoding = encoding; this.rawVectorFormat = new ES93GenericFlatVectorsFormat(useBFloat16, useDirectIO); } @@ -97,12 +55,8 @@ public String toString() { return NAME + "(name=" + NAME - + ", confidenceInterval=" - + confidenceInterval - + ", bits=" - + bits - + ", compressed=" - + compress + + ", encoding=" + + encoding + ", flatVectorScorer=" + flatVectorScorer + ", rawVectorFormat=" @@ -112,81 +66,12 @@ public String toString() { @Override public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { - return new Lucene99ScalarQuantizedVectorsWriter( - state, - confidenceInterval, - bits, - compress, - rawVectorFormat.fieldsWriter(state), - flatVectorScorer - ); + return new Lucene104ScalarQuantizedVectorsWriter(state, encoding, rawVectorFormat.fieldsWriter(state), flatVectorScorer) { + }; } @Override public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException { - return new Lucene99ScalarQuantizedVectorsReader(state, rawVectorFormat.fieldsReader(state), flatVectorScorer); - } - - static final class ESFlatVectorsScorer implements FlatVectorsScorer { - - final FlatVectorsScorer delegate; - final VectorScorerFactory factory; - - ESFlatVectorsScorer(FlatVectorsScorer delegate) { - this.delegate = delegate; - factory = VectorScorerFactory.instance().orElse(null); - } - - @Override - public String toString() { - return "ESFlatVectorsScorer(" + "delegate=" + delegate + ", factory=" + factory + ')'; - } - - @Override - public RandomVectorScorerSupplier getRandomVectorScorerSupplier(VectorSimilarityFunction sim, KnnVectorValues values) - throws IOException { - if (values instanceof QuantizedByteVectorValues qValues && qValues.getSlice() != null) { - // TODO: optimize int4 quantization - if (qValues.getScalarQuantizer().getBits() != 7) { - return delegate.getRandomVectorScorerSupplier(sim, values); - } - if (factory != null) { - var scorer = factory.getInt7SQVectorScorerSupplier( - VectorSimilarityType.of(sim), - qValues.getSlice(), - qValues, - qValues.getScalarQuantizer().getConstantMultiplier() - ); - if (scorer.isPresent()) { - return scorer.get(); - } - } - } - return delegate.getRandomVectorScorerSupplier(sim, values); - } - - @Override - public RandomVectorScorer getRandomVectorScorer(VectorSimilarityFunction sim, KnnVectorValues values, float[] query) - throws IOException { - if (values instanceof QuantizedByteVectorValues qValues && qValues.getSlice() != null) { - // TODO: optimize int4 quantization - if (qValues.getScalarQuantizer().getBits() != 7) { - return delegate.getRandomVectorScorer(sim, values, query); - } - if (factory != null) { - var scorer = factory.getInt7SQVectorScorer(sim, qValues, query); - if (scorer.isPresent()) { - return scorer.get(); - } - } - } - return delegate.getRandomVectorScorer(sim, values, query); - } - - @Override - public RandomVectorScorer getRandomVectorScorer(VectorSimilarityFunction sim, KnnVectorValues values, byte[] query) - throws IOException { - return delegate.getRandomVectorScorer(sim, values, query); - } + return new Lucene104ScalarQuantizedVectorsReader(state, rawVectorFormat.fieldsReader(state), flatVectorScorer); } } diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedBFloat16VectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedBFloat16VectorsFormatTests.java index 5688ede94e1ec..e2bc3970398c5 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedBFloat16VectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedBFloat16VectorsFormatTests.java @@ -9,95 +9,74 @@ package org.elasticsearch.index.codec.vectors.es93; -import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat; +import org.apache.lucene.store.Directory; +import org.elasticsearch.index.codec.vectors.BFloat16; +import org.elasticsearch.index.codec.vectors.BaseHnswBFloat16VectorsFormatTestCase; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import java.io.IOException; +import java.util.concurrent.ExecutorService; -import static org.hamcrest.Matchers.closeTo; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasEntry; -public class ES93HnswScalarQuantizedBFloat16VectorsFormatTests extends ES93HnswScalarQuantizedVectorsFormatTests { - @Override - boolean useBFloat16() { - return true; - } - - @Override - protected VectorEncoding randomVectorEncoding() { - return VectorEncoding.FLOAT32; - } - - @Override - public void testEmptyByteVectorData() throws Exception { - // no bytes - } +public class ES93HnswScalarQuantizedBFloat16VectorsFormatTests extends BaseHnswBFloat16VectorsFormatTestCase { @Override - public void testMergingWithDifferentByteKnnFields() throws Exception { - // no bytes + protected KnnVectorsFormat createFormat() { + return new ES93HnswScalarQuantizedVectorsFormat( + DEFAULT_MAX_CONN, + DEFAULT_BEAM_WIDTH, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, + true, + random().nextBoolean() + ); } @Override - public void testByteVectorScorerIteration() throws Exception { - // no bytes + protected KnnVectorsFormat createFormat(int maxConn, int beamWidth) { + return new ES93HnswScalarQuantizedVectorsFormat( + maxConn, + beamWidth, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, + true, + random().nextBoolean() + ); } @Override - public void testSortedIndexBytes() throws Exception { - // no bytes + protected KnnVectorsFormat createFormat(int maxConn, int beamWidth, int numMergeWorkers, ExecutorService service) { + return new ES93HnswScalarQuantizedVectorsFormat( + maxConn, + beamWidth, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, + true, + random().nextBoolean(), + numMergeWorkers, + service + ); } - @Override - public void testMismatchedFields() throws Exception { - // no bytes - } - - @Override - public void testRandomBytes() throws Exception { - // no bytes - } - - @Override - public void testWriterRamEstimate() throws Exception { - // estimate is different due to bfloat16 - } - - @Override - public void testRandom() throws Exception { - AssertionError err = expectThrows(AssertionError.class, super::testRandom); - assertFloatsWithinBounds(err); - } - - @Override - public void testRandomWithUpdatesAndGraph() throws Exception { - AssertionError err = expectThrows(AssertionError.class, super::testRandomWithUpdatesAndGraph); - assertFloatsWithinBounds(err); - } - - @Override - public void testSparseVectors() throws Exception { - AssertionError err = expectThrows(AssertionError.class, super::testSparseVectors); - assertFloatsWithinBounds(err); - } - - @Override - public void testVectorValuesReportCorrectDocs() throws Exception { - AssertionError err = expectThrows(AssertionError.class, super::testVectorValuesReportCorrectDocs); - assertFloatsWithinBounds(err); - } - - private static final Pattern FLOAT_ASSERTION_FAILURE = Pattern.compile(".*expected:<([0-9.-]+)> but was:<([0-9.-]+)>"); - - private static void assertFloatsWithinBounds(AssertionError error) { - Matcher m = FLOAT_ASSERTION_FAILURE.matcher(error.getMessage()); - if (m.matches() == false) { - throw error; // nothing to do with us, just rethrow + public void testSimpleOffHeapSize() throws IOException { + float[] vector = randomVector(random().nextInt(12, 500)); + try (Directory dir = newDirectory()) { + testSimpleOffHeapSize( + dir, + newIndexWriterConfig(), + vector, + allOf( + aMapWithSize(3), + hasEntry("vec", (long) vector.length * BFloat16.BYTES), + hasEntry("vex", 1L), + hasEntry(equalTo("veq"), greaterThan(0L)) + ) + ); } - - // numbers just need to be in the same vicinity - double expected = Double.parseDouble(m.group(1)); - double actual = Double.parseDouble(m.group(2)); - double allowedError = expected * 0.01; // within 1% - assertThat(error.getMessage(), actual, closeTo(expected, allowedError)); } } diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormatTests.java index a6c12548821b9..2d2564056afca 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormatTests.java @@ -9,212 +9,73 @@ package org.elasticsearch.index.codec.vectors.es93; -import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.KnnVectorsReader; -import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.KnnFloatVectorField; -import org.apache.lucene.index.CodecReader; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.FloatVectorValues; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.KnnVectorValues; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.StoredFields; -import org.apache.lucene.index.VectorSimilarityFunction; -import org.apache.lucene.search.AcceptDocs; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.MMapDirectory; -import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; -import org.apache.lucene.tests.util.TestUtil; -import org.elasticsearch.common.logging.LogConfigurator; -import org.elasticsearch.index.codec.vectors.BFloat16; +import org.elasticsearch.index.codec.vectors.BaseHnswVectorsFormatTestCase; import java.io.IOException; -import java.nio.file.Path; +import java.util.concurrent.ExecutorService; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; -import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT; -import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasEntry; -public class ES93HnswScalarQuantizedVectorsFormatTests extends BaseKnnVectorsFormatTestCase { - - static { - LogConfigurator.loadLog4jPlugins(); - LogConfigurator.configureESLogging(); // native access requires logging to be initialized - } - - boolean useBFloat16() { - return false; - } +public class ES93HnswScalarQuantizedVectorsFormatTests extends BaseHnswVectorsFormatTestCase { @Override - protected Codec getCodec() { - return TestUtil.alwaysKnnVectorsFormat( - new ES93HnswScalarQuantizedVectorsFormat(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, useBFloat16(), null, 7, false, false) + protected KnnVectorsFormat createFormat() { + return new ES93HnswScalarQuantizedVectorsFormat( + DEFAULT_MAX_CONN, + DEFAULT_BEAM_WIDTH, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, + false, + random().nextBoolean() ); } - // The following test scenarios are similar to their superclass namesakes, - // but here we ensure that the Directory implementation is a FSDirectory - // which helps test the native code vector distance implementation - - public void testAddIndexesDirectory0FS() throws Exception { - Path root = createTempDir(); - String fieldName = "field"; - Document doc = new Document(); - doc.add(new KnnFloatVectorField(fieldName, new float[4], VectorSimilarityFunction.DOT_PRODUCT)); - try (Directory dir = new MMapDirectory(root.resolve("dir1")); Directory dir2 = new MMapDirectory(root.resolve("dir2"))) { - try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { - w.addDocument(doc); - } - try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) { - w2.addIndexes(dir); - w2.forceMerge(1); - try (IndexReader reader = DirectoryReader.open(w2)) { - LeafReader r = getOnlyLeafReader(reader); - FloatVectorValues vectorValues = r.getFloatVectorValues(fieldName); - KnnVectorValues.DocIndexIterator iterator = vectorValues.iterator(); - assertEquals(0, iterator.nextDoc()); - assertEquals(0, vectorValues.vectorValue(iterator.index())[0], 0); - assertEquals(NO_MORE_DOCS, iterator.nextDoc()); - } - } - } - } - - public void testAddIndexesDirectory01FSCosine() throws Exception { - testAddIndexesDirectory01FS(VectorSimilarityFunction.COSINE); - } - - public void testAddIndexesDirectory01FSDot() throws Exception { - testAddIndexesDirectory01FS(VectorSimilarityFunction.DOT_PRODUCT); - } - - public void testAddIndexesDirectory01FSEuclidean() throws Exception { - testAddIndexesDirectory01FS(VectorSimilarityFunction.EUCLIDEAN); - } - - public void testAddIndexesDirectory01FSMaxIP() throws Exception { - testAddIndexesDirectory01FS(VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT); - } - - private void testAddIndexesDirectory01FS(VectorSimilarityFunction similarityFunction) throws Exception { - Path root = createTempDir(); - String fieldName = "field"; - float[] vector = new float[] { 1f }; - Document doc = new Document(); - doc.add(new KnnFloatVectorField(fieldName, vector, similarityFunction)); - try (Directory dir = new MMapDirectory(root.resolve("dir1")); Directory dir2 = new MMapDirectory(root.resolve("dir2"))) { - try (IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { - w.addDocument(doc); - } - try (IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig())) { - vector[0] = 2f; - w2.addDocument(doc); - w2.addIndexes(dir); - w2.forceMerge(1); - try (IndexReader reader = DirectoryReader.open(w2)) { - LeafReader r = getOnlyLeafReader(reader); - FloatVectorValues vectorValues = r.getFloatVectorValues(fieldName); - KnnVectorValues.DocIndexIterator iterator = vectorValues.iterator(); - assertEquals(0, iterator.nextDoc()); - // The merge order is randomized, we might get 1 first, or 2 - float value = vectorValues.vectorValue(iterator.index())[0]; - assertTrue(value == 1 || value == 2); - assertEquals(1, iterator.nextDoc()); - value += vectorValues.vectorValue(iterator.index())[0]; - assertEquals(3f, value, 0); - } - } - } - } - - public void testSingleVectorPerSegmentCosine() throws Exception { - testSingleVectorPerSegment(VectorSimilarityFunction.COSINE); - } - - public void testSingleVectorPerSegmentDot() throws Exception { - testSingleVectorPerSegment(VectorSimilarityFunction.DOT_PRODUCT); - } - - public void testSingleVectorPerSegmentEuclidean() throws Exception { - testSingleVectorPerSegment(VectorSimilarityFunction.EUCLIDEAN); - } - - public void testSingleVectorPerSegmentMIP() throws Exception { - testSingleVectorPerSegment(VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT); + @Override + protected KnnVectorsFormat createFormat(int maxConn, int beamWidth) { + return new ES93HnswScalarQuantizedVectorsFormat( + maxConn, + beamWidth, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, + false, + random().nextBoolean() + ); } - private void testSingleVectorPerSegment(VectorSimilarityFunction sim) throws Exception { - var codec = getCodec(); - try (Directory dir = new MMapDirectory(createTempDir().resolve("dir1"))) { - try (IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig().setCodec(codec))) { - Document doc2 = new Document(); - doc2.add(new KnnFloatVectorField("field", new float[] { 0.8f, 0.6f }, sim)); - doc2.add(newTextField("id", "A", Field.Store.YES)); - writer.addDocument(doc2); - writer.commit(); - - Document doc1 = new Document(); - doc1.add(new KnnFloatVectorField("field", new float[] { 0.6f, 0.8f }, sim)); - doc1.add(newTextField("id", "B", Field.Store.YES)); - writer.addDocument(doc1); - writer.commit(); - - Document doc3 = new Document(); - doc3.add(new KnnFloatVectorField("field", new float[] { -0.6f, -0.8f }, sim)); - doc3.add(newTextField("id", "C", Field.Store.YES)); - writer.addDocument(doc3); - writer.commit(); - - writer.forceMerge(1); - } - try (DirectoryReader reader = DirectoryReader.open(dir)) { - LeafReader leafReader = getOnlyLeafReader(reader); - StoredFields storedFields = reader.storedFields(); - float[] queryVector = new float[] { 0.6f, 0.8f }; - var hits = leafReader.searchNearestVectors( - "field", - queryVector, - 3, - AcceptDocs.fromLiveDocs(leafReader.getLiveDocs(), leafReader.maxDoc()), - 100 - ); - assertEquals(hits.scoreDocs.length, 3); - assertEquals("B", storedFields.document(hits.scoreDocs[0].doc).get("id")); - assertEquals("A", storedFields.document(hits.scoreDocs[1].doc).get("id")); - assertEquals("C", storedFields.document(hits.scoreDocs[2].doc).get("id")); - } - } + @Override + protected KnnVectorsFormat createFormat(int maxConn, int beamWidth, int numMergeWorkers, ExecutorService service) { + return new ES93HnswScalarQuantizedVectorsFormat( + maxConn, + beamWidth, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, + false, + random().nextBoolean(), + numMergeWorkers, + service + ); } public void testSimpleOffHeapSize() throws IOException { float[] vector = randomVector(random().nextInt(12, 500)); - try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { - Document doc = new Document(); - doc.add(new KnnFloatVectorField("f", vector, DOT_PRODUCT)); - w.addDocument(doc); - w.commit(); - try (IndexReader reader = DirectoryReader.open(w)) { - LeafReader r = getOnlyLeafReader(reader); - if (r instanceof CodecReader codecReader) { - KnnVectorsReader knnVectorsReader = codecReader.getVectorReader(); - if (knnVectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader fieldsReader) { - knnVectorsReader = fieldsReader.getFieldReader("f"); - } - var fieldInfo = r.getFieldInfos().fieldInfo("f"); - var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); - assertEquals(3, offHeap.size()); - int bytes = useBFloat16() ? BFloat16.BYTES : Float.BYTES; - assertEquals(vector.length * bytes, (long) offHeap.get("vec")); - assertEquals(1L, (long) offHeap.get("vex")); - assertTrue(offHeap.get("veq") > 0L); - } - } + try (Directory dir = newDirectory()) { + testSimpleOffHeapSize( + dir, + newIndexWriterConfig(), + vector, + allOf( + aMapWithSize(3), + hasEntry("vec", (long) vector.length * Float.BYTES), + hasEntry("vex", 1L), + hasEntry(equalTo("veq"), greaterThan(0L)) + ) + ); } } } From 4fa433808db5c25dadb085bae84b160c94f785e3 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Thu, 23 Oct 2025 10:20:02 +0100 Subject: [PATCH 08/32] Update more tests --- .../ES93FlatBFloat16VectorFormatTests.java | 134 +++++++---------- .../es93/ES93FlatVectorFormatTests.java | 17 +-- ...uantizedFlatBFloat16VectorFormatTests.java | 138 ++++++++---------- ...ScalarQuantizedFlatVectorsFormatTests.java | 13 +- 4 files changed, 123 insertions(+), 179 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatBFloat16VectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatBFloat16VectorFormatTests.java index 4d017daa2724c..41d777dde2cf9 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatBFloat16VectorFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatBFloat16VectorFormatTests.java @@ -9,95 +9,65 @@ package org.elasticsearch.index.codec.vectors.es93; -import org.apache.lucene.index.VectorEncoding; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static org.hamcrest.Matchers.closeTo; - -public class ES93FlatBFloat16VectorFormatTests extends ES93FlatVectorFormatTests { - @Override - boolean useBFloat16() { - return true; - } - - @Override - protected VectorEncoding randomVectorEncoding() { - return VectorEncoding.FLOAT32; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.CodecReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.util.TestUtil; +import org.elasticsearch.common.logging.LogConfigurator; +import org.elasticsearch.index.codec.vectors.BFloat16; +import org.elasticsearch.index.codec.vectors.BaseBFloat16KnnVectorsFormatTestCase; +import org.junit.AssumptionViolatedException; + +import java.io.IOException; + +import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.hasEntry; + +public class ES93FlatBFloat16VectorFormatTests extends BaseBFloat16KnnVectorsFormatTestCase { + + static { + LogConfigurator.loadLog4jPlugins(); + LogConfigurator.configureESLogging(); // native access requires logging to be initialized } @Override - public void testEmptyByteVectorData() throws Exception { - // no bytes + protected Codec getCodec() { + return TestUtil.alwaysKnnVectorsFormat(new ES93FlatVectorFormat(true)); } - @Override - public void testMergingWithDifferentByteKnnFields() throws Exception { - // no bytes + public void testSearchWithVisitedLimit() { + throw new AssumptionViolatedException("requires graph-based vector codec"); } - @Override - public void testByteVectorScorerIteration() throws Exception { - // no bytes - } - - @Override - public void testSortedIndexBytes() throws Exception { - // no bytes - } - - @Override - public void testMismatchedFields() throws Exception { - // no bytes - } - - @Override - public void testRandomBytes() throws Exception { - // no bytes - } - - @Override - public void testWriterRamEstimate() throws Exception { - // estimate is different due to bfloat16 - } - - @Override - public void testRandom() throws Exception { - AssertionError err = expectThrows(AssertionError.class, super::testRandom); - assertFloatsWithinBounds(err); - } - - @Override - public void testRandomWithUpdatesAndGraph() throws Exception { - AssertionError err = expectThrows(AssertionError.class, super::testRandomWithUpdatesAndGraph); - assertFloatsWithinBounds(err); - } - - @Override - public void testSparseVectors() throws Exception { - AssertionError err = expectThrows(AssertionError.class, super::testSparseVectors); - assertFloatsWithinBounds(err); - } - - @Override - public void testVectorValuesReportCorrectDocs() throws Exception { - AssertionError err = expectThrows(AssertionError.class, super::testVectorValuesReportCorrectDocs); - assertFloatsWithinBounds(err); - } - - private static final Pattern FLOAT_ASSERTION_FAILURE = Pattern.compile(".*expected:<([0-9.-]+)> but was:<([0-9.-]+)>"); - - private static void assertFloatsWithinBounds(AssertionError error) { - Matcher m = FLOAT_ASSERTION_FAILURE.matcher(error.getMessage()); - if (m.matches() == false) { - throw error; // nothing to do with us, just rethrow + public void testSimpleOffHeapSize() throws IOException { + float[] vector = randomVector(random().nextInt(12, 500)); + try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + Document doc = new Document(); + doc.add(new KnnFloatVectorField("f", vector, DOT_PRODUCT)); + w.addDocument(doc); + w.commit(); + try (IndexReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + if (r instanceof CodecReader codecReader) { + KnnVectorsReader knnVectorsReader = codecReader.getVectorReader(); + if (knnVectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader fieldsReader) { + knnVectorsReader = fieldsReader.getFieldReader("f"); + } + var fieldInfo = r.getFieldInfos().fieldInfo("f"); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); + assertThat(offHeap, aMapWithSize(1)); + assertThat(offHeap, hasEntry("vec", (long) vector.length * BFloat16.BYTES)); + } + } } - - // numbers just need to be in the same vicinity - double expected = Double.parseDouble(m.group(1)); - double actual = Double.parseDouble(m.group(2)); - double allowedError = expected * 0.01; // within 1% - assertThat(error.getMessage(), actual, closeTo(expected, allowedError)); } } diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormatTests.java index f3faa98124a66..18335758f82d8 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormatTests.java @@ -23,11 +23,13 @@ import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; import org.apache.lucene.tests.util.TestUtil; import org.elasticsearch.common.logging.LogConfigurator; -import org.elasticsearch.index.codec.vectors.BFloat16; +import org.junit.AssumptionViolatedException; import java.io.IOException; import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.hasEntry; public class ES93FlatVectorFormatTests extends BaseKnnVectorsFormatTestCase { @@ -36,17 +38,13 @@ public class ES93FlatVectorFormatTests extends BaseKnnVectorsFormatTestCase { LogConfigurator.configureESLogging(); // native access requires logging to be initialized } - boolean useBFloat16() { - return false; - } - @Override protected Codec getCodec() { - return TestUtil.alwaysKnnVectorsFormat(new ES93FlatVectorFormat(useBFloat16())); + return TestUtil.alwaysKnnVectorsFormat(new ES93FlatVectorFormat(false)); } public void testSearchWithVisitedLimit() { - // requires graph-based vector codec + throw new AssumptionViolatedException("requires graph-based vector codec"); } public void testSimpleOffHeapSize() throws IOException { @@ -65,9 +63,8 @@ public void testSimpleOffHeapSize() throws IOException { } var fieldInfo = r.getFieldInfos().fieldInfo("f"); var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); - int bytes = useBFloat16() ? BFloat16.BYTES : Float.BYTES; - assertEquals(vector.length * bytes, (long) offHeap.get("vec")); - assertEquals(1, offHeap.size()); + assertThat(offHeap, aMapWithSize(1)); + assertThat(offHeap, hasEntry("vec", (long) vector.length * Float.BYTES)); } } } diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatBFloat16VectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatBFloat16VectorFormatTests.java index 3681d0a1299eb..bd8376a96f102 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatBFloat16VectorFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatBFloat16VectorFormatTests.java @@ -9,95 +9,77 @@ package org.elasticsearch.index.codec.vectors.es93; -import org.apache.lucene.index.VectorEncoding; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static org.hamcrest.Matchers.closeTo; - -public class ES93ScalarQuantizedFlatBFloat16VectorFormatTests extends ES93ScalarQuantizedFlatVectorsFormatTests { - @Override - boolean useBFloat16() { - return true; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.CodecReader; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.util.TestUtil; +import org.elasticsearch.common.logging.LogConfigurator; +import org.elasticsearch.index.codec.vectors.BFloat16; +import org.elasticsearch.index.codec.vectors.BaseBFloat16KnnVectorsFormatTestCase; +import org.junit.AssumptionViolatedException; + +import java.io.IOException; + +import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasEntry; + +public class ES93ScalarQuantizedFlatBFloat16VectorFormatTests extends BaseBFloat16KnnVectorsFormatTestCase { + + static { + LogConfigurator.loadLog4jPlugins(); + LogConfigurator.configureESLogging(); // native access requires logging to be initialized } - @Override - protected VectorEncoding randomVectorEncoding() { - return VectorEncoding.FLOAT32; - } + private KnnVectorsFormat format; @Override - public void testEmptyByteVectorData() throws Exception { - // no bytes + public void setUp() throws Exception { + format = new ES93ScalarQuantizedFlatVectorsFormat(true); + super.setUp(); } @Override - public void testMergingWithDifferentByteKnnFields() throws Exception { - // no bytes + protected Codec getCodec() { + return TestUtil.alwaysKnnVectorsFormat(format); } - @Override - public void testByteVectorScorerIteration() throws Exception { - // no bytes + public void testSearchWithVisitedLimit() { + throw new AssumptionViolatedException("requires graph vector codec"); } - @Override - public void testSortedIndexBytes() throws Exception { - // no bytes - } - - @Override - public void testMismatchedFields() throws Exception { - // no bytes - } - - @Override - public void testRandomBytes() throws Exception { - // no bytes - } - - @Override - public void testWriterRamEstimate() throws Exception { - // estimate is different due to bfloat16 - } - - @Override - public void testRandom() throws Exception { - AssertionError err = expectThrows(AssertionError.class, super::testRandom); - assertFloatsWithinBounds(err); - } - - @Override - public void testRandomWithUpdatesAndGraph() throws Exception { - AssertionError err = expectThrows(AssertionError.class, super::testRandomWithUpdatesAndGraph); - assertFloatsWithinBounds(err); - } - - @Override - public void testSparseVectors() throws Exception { - AssertionError err = expectThrows(AssertionError.class, super::testSparseVectors); - assertFloatsWithinBounds(err); - } - - @Override - public void testVectorValuesReportCorrectDocs() throws Exception { - AssertionError err = expectThrows(AssertionError.class, super::testVectorValuesReportCorrectDocs); - assertFloatsWithinBounds(err); - } - - private static final Pattern FLOAT_ASSERTION_FAILURE = Pattern.compile(".*expected:<([0-9.-]+)> but was:<([0-9.-]+)>"); - - private static void assertFloatsWithinBounds(AssertionError error) { - Matcher m = FLOAT_ASSERTION_FAILURE.matcher(error.getMessage()); - if (m.matches() == false) { - throw error; // nothing to do with us, just rethrow + public void testSimpleOffHeapSize() throws IOException { + float[] vector = randomVector(random().nextInt(12, 500)); + try (Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + Document doc = new Document(); + doc.add(new KnnFloatVectorField("f", vector, DOT_PRODUCT)); + w.addDocument(doc); + w.commit(); + try (IndexReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + if (r instanceof CodecReader codecReader) { + KnnVectorsReader knnVectorsReader = codecReader.getVectorReader(); + if (knnVectorsReader instanceof PerFieldKnnVectorsFormat.FieldsReader fieldsReader) { + knnVectorsReader = fieldsReader.getFieldReader("f"); + } + var fieldInfo = r.getFieldInfos().fieldInfo("f"); + var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); + assertThat(offHeap, aMapWithSize(2)); + assertThat(offHeap, hasEntry("vec", (long) vector.length * BFloat16.BYTES)); + assertThat(offHeap, hasEntry(equalTo("veq"), greaterThan(0L))); + } + } } - - // numbers just need to be in the same vicinity - double expected = Double.parseDouble(m.group(1)); - double actual = Double.parseDouble(m.group(2)); - double allowedError = expected * 0.01; // within 1% - assertThat(error.getMessage(), actual, closeTo(expected, allowedError)); } } diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormatTests.java index d05f33941f7b3..bb56b42d14400 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormatTests.java @@ -23,7 +23,7 @@ import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; import org.apache.lucene.tests.util.TestUtil; import org.elasticsearch.common.logging.LogConfigurator; -import org.elasticsearch.index.codec.vectors.BFloat16; +import org.junit.AssumptionViolatedException; import java.io.IOException; @@ -40,17 +40,13 @@ public class ES93ScalarQuantizedFlatVectorsFormatTests extends BaseKnnVectorsFor LogConfigurator.configureESLogging(); // native access requires logging to be initialized } - boolean useBFloat16() { - return false; - } - @Override protected Codec getCodec() { - return TestUtil.alwaysKnnVectorsFormat(new ES93ScalarQuantizedFlatVectorsFormat(useBFloat16())); + return TestUtil.alwaysKnnVectorsFormat(new ES93ScalarQuantizedFlatVectorsFormat(false)); } public void testSearchWithVisitedLimit() { - // requires graph vector codec + throw new AssumptionViolatedException("requires graph vector codec"); } public void testSimpleOffHeapSize() throws IOException { @@ -70,11 +66,10 @@ public void testSimpleOffHeapSize() throws IOException { var fieldInfo = r.getFieldInfos().fieldInfo("f"); var offHeap = knnVectorsReader.getOffHeapByteSize(fieldInfo); assertThat(offHeap, aMapWithSize(2)); - assertThat(offHeap, hasEntry("vec", vector.length * (useBFloat16() ? BFloat16.BYTES : Float.BYTES))); + assertThat(offHeap, hasEntry("vec", (long) vector.length * Float.BYTES)); assertThat(offHeap, hasEntry(equalTo("veq"), greaterThan(0L))); } } } } - } From 4815bc2a373dadebc96f7a828bef6187155015fb Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Thu, 23 Oct 2025 10:53:16 +0100 Subject: [PATCH 09/32] Class renames --- server/src/main/java/module-info.java | 4 +- .../ES93HnswScalarQuantizedVectorsFormat.java | 8 +- .../ES93ScalarQuantizedFlatVectorsFormat.java | 125 +++++------------- .../ES93ScalarQuantizedVectorsFormat.java | 125 +++++++++++++----- .../org.apache.lucene.codecs.KnnVectorsFormat | 4 +- ...arQuantizedBFloat16VectorFormatTests.java} | 4 +- ...S93ScalarQuantizedVectorsFormatTests.java} | 4 +- 7 files changed, 137 insertions(+), 137 deletions(-) rename server/src/test/java/org/elasticsearch/index/codec/vectors/es93/{ES93ScalarQuantizedFlatBFloat16VectorFormatTests.java => ES93ScalarQuantizedBFloat16VectorFormatTests.java} (95%) rename server/src/test/java/org/elasticsearch/index/codec/vectors/es93/{ES93ScalarQuantizedFlatVectorsFormatTests.java => ES93ScalarQuantizedVectorsFormatTests.java} (96%) diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index c56db1f377315..9ec9b42d34297 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -466,10 +466,10 @@ org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat, org.elasticsearch.index.codec.vectors.diskbbq.next.ESNextDiskBBQVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93FlatVectorFormat, - org.elasticsearch.index.codec.vectors.es93.ES93ScalarQuantizedFlatVectorsFormat, + org.elasticsearch.index.codec.vectors.es93.ES93HnswVectorsFormat, + org.elasticsearch.index.codec.vectors.es93.ES93ScalarQuantizedVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93HnswScalarQuantizedVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat, - org.elasticsearch.index.codec.vectors.es93.ES93HnswVectorsFormat, org.elasticsearch.index.codec.vectors.es93.ES93HnswBinaryQuantizedVectorsFormat; provides org.apache.lucene.codecs.Codec diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java index a2e14df6099ec..aec2120b97ace 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java @@ -31,7 +31,7 @@ public class ES93HnswScalarQuantizedVectorsFormat extends AbstractHnswVectorsFor public ES93HnswScalarQuantizedVectorsFormat() { super(NAME); - this.flatVectorsFormat = new ES93ScalarQuantizedVectorsFormat( + this.flatVectorsFormat = new ES93ScalarQuantizedFlatVectorsFormat( Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, false, false @@ -46,7 +46,7 @@ public ES93HnswScalarQuantizedVectorsFormat( boolean useDirectIO ) { super(NAME, maxConn, beamWidth); - this.flatVectorsFormat = new ES93ScalarQuantizedVectorsFormat(encoding, useBFloat16, useDirectIO); + this.flatVectorsFormat = new ES93ScalarQuantizedFlatVectorsFormat(encoding, useBFloat16, useDirectIO); } public ES93HnswScalarQuantizedVectorsFormat( @@ -59,7 +59,7 @@ public ES93HnswScalarQuantizedVectorsFormat( ExecutorService mergeExec ) { super(NAME, maxConn, beamWidth, numMergeWorkers, mergeExec); - this.flatVectorsFormat = new ES93ScalarQuantizedVectorsFormat(encoding, useBFloat16, useDirectIO); + this.flatVectorsFormat = new ES93ScalarQuantizedFlatVectorsFormat(encoding, useBFloat16, useDirectIO); } @Override @@ -69,7 +69,7 @@ protected FlatVectorsFormat flatVectorsFormat() { @Override public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { - return new Lucene99HnswVectorsWriter(state, maxConn, beamWidth, flatVectorsFormat.fieldsWriter(state), numMergeWorkers, mergeExec); + return new Lucene99HnswVectorsWriter(state, maxConn, beamWidth, flatVectorsFormat.fieldsWriter(state), numMergeWorkers, mergeExec, 0); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java index f1375cb5469e7..c4b99f75ee910 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java @@ -9,55 +9,40 @@ package org.elasticsearch.index.codec.vectors.es93; -import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.KnnVectorsReader; -import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorScorer; import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat; -import org.apache.lucene.index.ByteVectorValues; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsReader; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsWriter; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.search.AcceptDocs; -import org.apache.lucene.search.KnnCollector; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector; -import org.apache.lucene.util.hnsw.RandomVectorScorer; import java.io.IOException; -import java.util.Map; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; -public class ES93ScalarQuantizedFlatVectorsFormat extends KnnVectorsFormat { +public class ES93ScalarQuantizedFlatVectorsFormat extends FlatVectorsFormat { static final String NAME = "ES93ScalarQuantizedFlatVectorsFormat"; - private final FlatVectorsFormat format; + static final Lucene104ScalarQuantizedVectorScorer flatVectorScorer = new Lucene104ScalarQuantizedVectorScorer( + FlatVectorScorerUtil.getLucene99FlatVectorsScorer() + ); - public ES93ScalarQuantizedFlatVectorsFormat() { - this(false, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT); - } - - public ES93ScalarQuantizedFlatVectorsFormat(boolean useBFloat16) { - this(useBFloat16, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT); - } + private final Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding; + private final FlatVectorsFormat rawVectorFormat; - public ES93ScalarQuantizedFlatVectorsFormat(boolean useBFloat16, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding) { + public ES93ScalarQuantizedFlatVectorsFormat( + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding, + boolean useBFloat16, + boolean useDirectIO + ) { super(NAME); - this.format = new ES93ScalarQuantizedVectorsFormat(encoding, useBFloat16, false); - } - - @Override - public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { - return format.fieldsWriter(state); - } - - @Override - public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { - return new ES93FlatVectorsReader(format.fieldsReader(state)); + this.encoding = encoding; + this.rawVectorFormat = new ES93GenericFlatVectorsFormat(useBFloat16, useDirectIO); } @Override @@ -67,64 +52,26 @@ public int getMaxDimensions(String fieldName) { @Override public String toString() { - return NAME + "(name=" + NAME + ", innerFormat=" + format + ")"; + return NAME + + "(name=" + + NAME + + ", encoding=" + + encoding + + ", flatVectorScorer=" + + flatVectorScorer + + ", rawVectorFormat=" + + rawVectorFormat + + ")"; } - public static class ES93FlatVectorsReader extends KnnVectorsReader { - - private final FlatVectorsReader reader; - - public ES93FlatVectorsReader(FlatVectorsReader reader) { - super(); - this.reader = reader; - } - - @Override - public void checkIntegrity() throws IOException { - reader.checkIntegrity(); - } - - @Override - public FloatVectorValues getFloatVectorValues(String field) throws IOException { - return reader.getFloatVectorValues(field); - } - - @Override - public ByteVectorValues getByteVectorValues(String field) throws IOException { - return reader.getByteVectorValues(field); - } - - @Override - public void search(String field, float[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { - collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); - } - - private void collectAllMatchingDocs(KnnCollector knnCollector, AcceptDocs acceptDocs, RandomVectorScorer scorer) - throws IOException { - OrdinalTranslatedKnnCollector collector = new OrdinalTranslatedKnnCollector(knnCollector, scorer::ordToDoc); - Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs.bits()); - for (int i = 0; i < scorer.maxOrd(); i++) { - if (acceptedOrds == null || acceptedOrds.get(i)) { - collector.collect(i, scorer.score(i)); - collector.incVisitedCount(1); - } - } - assert collector.earlyTerminated() == false; - } - - @Override - public void search(String field, byte[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { - collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); - } - - @Override - public Map getOffHeapByteSize(FieldInfo fieldInfo) { - return reader.getOffHeapByteSize(fieldInfo); - } + @Override + public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return new Lucene104ScalarQuantizedVectorsWriter(state, encoding, rawVectorFormat.fieldsWriter(state), flatVectorScorer) { + }; + } - @Override - public void close() throws IOException { - reader.close(); - } + @Override + public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException { + return new Lucene104ScalarQuantizedVectorsReader(state, rawVectorFormat.fieldsReader(state), flatVectorScorer); } } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java index b393d91dd0c72..175b4a79ecce5 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java @@ -9,69 +9,122 @@ package org.elasticsearch.index.codec.vectors.es93; -import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; import org.apache.lucene.codecs.hnsw.FlatVectorsReader; -import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; -import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorScorer; import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat; -import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsReader; -import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsWriter; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FloatVectorValues; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.search.AcceptDocs; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector; +import org.apache.lucene.util.hnsw.RandomVectorScorer; import java.io.IOException; +import java.util.Map; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; -public class ES93ScalarQuantizedVectorsFormat extends FlatVectorsFormat { +public class ES93ScalarQuantizedVectorsFormat extends KnnVectorsFormat { static final String NAME = "ES93ScalarQuantizedVectorsFormat"; - static final Lucene104ScalarQuantizedVectorScorer flatVectorScorer = new Lucene104ScalarQuantizedVectorScorer( - FlatVectorScorerUtil.getLucene99FlatVectorsScorer() - ); + private final FlatVectorsFormat format; - private final Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding; - private final FlatVectorsFormat rawVectorFormat; + public ES93ScalarQuantizedVectorsFormat() { + this(false, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT); + } + + public ES93ScalarQuantizedVectorsFormat(boolean useBFloat16) { + this(useBFloat16, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT); + } - public ES93ScalarQuantizedVectorsFormat( - Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding, - boolean useBFloat16, - boolean useDirectIO - ) { + public ES93ScalarQuantizedVectorsFormat(boolean useBFloat16, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding) { super(NAME); - this.encoding = encoding; - this.rawVectorFormat = new ES93GenericFlatVectorsFormat(useBFloat16, useDirectIO); + this.format = new ES93ScalarQuantizedFlatVectorsFormat(encoding, useBFloat16, false); } @Override - public int getMaxDimensions(String fieldName) { - return MAX_DIMS_COUNT; + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + return format.fieldsWriter(state); } @Override - public String toString() { - return NAME - + "(name=" - + NAME - + ", encoding=" - + encoding - + ", flatVectorScorer=" - + flatVectorScorer - + ", rawVectorFormat=" - + rawVectorFormat - + ")"; + public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { + return new ES93FlatVectorsReader(format.fieldsReader(state)); } @Override - public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { - return new Lucene104ScalarQuantizedVectorsWriter(state, encoding, rawVectorFormat.fieldsWriter(state), flatVectorScorer) { - }; + public int getMaxDimensions(String fieldName) { + return MAX_DIMS_COUNT; } @Override - public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException { - return new Lucene104ScalarQuantizedVectorsReader(state, rawVectorFormat.fieldsReader(state), flatVectorScorer); + public String toString() { + return NAME + "(name=" + NAME + ", innerFormat=" + format + ")"; + } + + public static class ES93FlatVectorsReader extends KnnVectorsReader { + + private final FlatVectorsReader reader; + + public ES93FlatVectorsReader(FlatVectorsReader reader) { + super(); + this.reader = reader; + } + + @Override + public void checkIntegrity() throws IOException { + reader.checkIntegrity(); + } + + @Override + public FloatVectorValues getFloatVectorValues(String field) throws IOException { + return reader.getFloatVectorValues(field); + } + + @Override + public ByteVectorValues getByteVectorValues(String field) throws IOException { + return reader.getByteVectorValues(field); + } + + @Override + public void search(String field, float[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { + collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); + } + + private void collectAllMatchingDocs(KnnCollector knnCollector, AcceptDocs acceptDocs, RandomVectorScorer scorer) + throws IOException { + OrdinalTranslatedKnnCollector collector = new OrdinalTranslatedKnnCollector(knnCollector, scorer::ordToDoc); + Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs.bits()); + for (int i = 0; i < scorer.maxOrd(); i++) { + if (acceptedOrds == null || acceptedOrds.get(i)) { + collector.collect(i, scorer.score(i)); + collector.incVisitedCount(1); + } + } + assert collector.earlyTerminated() == false; + } + + @Override + public void search(String field, byte[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { + collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); + } + + @Override + public Map getOffHeapByteSize(FieldInfo fieldInfo) { + return reader.getOffHeapByteSize(fieldInfo); + } + + @Override + public void close() throws IOException { + reader.close(); + } } } diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat index d6b114fb57555..0dc34ea2e808d 100644 --- a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat +++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat @@ -10,8 +10,8 @@ org.elasticsearch.index.codec.vectors.es818.ES818HnswBinaryQuantizedVectorsForma org.elasticsearch.index.codec.vectors.diskbbq.ES920DiskBBQVectorsFormat org.elasticsearch.index.codec.vectors.diskbbq.next.ESNextDiskBBQVectorsFormat org.elasticsearch.index.codec.vectors.es93.ES93FlatVectorFormat -org.elasticsearch.index.codec.vectors.es93.ES93ScalarQuantizedFlatVectorsFormat +org.elasticsearch.index.codec.vectors.es93.ES93HnswVectorsFormat +org.elasticsearch.index.codec.vectors.es93.ES93ScalarQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.es93.ES93HnswScalarQuantizedVectorsFormat org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat -org.elasticsearch.index.codec.vectors.es93.ES93HnswVectorsFormat org.elasticsearch.index.codec.vectors.es93.ES93HnswBinaryQuantizedVectorsFormat diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatBFloat16VectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedBFloat16VectorFormatTests.java similarity index 95% rename from server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatBFloat16VectorFormatTests.java rename to server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedBFloat16VectorFormatTests.java index bd8376a96f102..7e69492f6e848 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatBFloat16VectorFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedBFloat16VectorFormatTests.java @@ -35,7 +35,7 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasEntry; -public class ES93ScalarQuantizedFlatBFloat16VectorFormatTests extends BaseBFloat16KnnVectorsFormatTestCase { +public class ES93ScalarQuantizedBFloat16VectorFormatTests extends BaseBFloat16KnnVectorsFormatTestCase { static { LogConfigurator.loadLog4jPlugins(); @@ -46,7 +46,7 @@ public class ES93ScalarQuantizedFlatBFloat16VectorFormatTests extends BaseBFloat @Override public void setUp() throws Exception { - format = new ES93ScalarQuantizedFlatVectorsFormat(true); + format = new ES93ScalarQuantizedVectorsFormat(true); super.setUp(); } diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormatTests.java similarity index 96% rename from server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormatTests.java rename to server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormatTests.java index bb56b42d14400..7c8461a6d033d 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormatTests.java @@ -33,7 +33,7 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasEntry; -public class ES93ScalarQuantizedFlatVectorsFormatTests extends BaseKnnVectorsFormatTestCase { +public class ES93ScalarQuantizedVectorsFormatTests extends BaseKnnVectorsFormatTestCase { static { LogConfigurator.loadLog4jPlugins(); @@ -42,7 +42,7 @@ public class ES93ScalarQuantizedFlatVectorsFormatTests extends BaseKnnVectorsFor @Override protected Codec getCodec() { - return TestUtil.alwaysKnnVectorsFormat(new ES93ScalarQuantizedFlatVectorsFormat(false)); + return TestUtil.alwaysKnnVectorsFormat(new ES93ScalarQuantizedVectorsFormat(false)); } public void testSearchWithVisitedLimit() { From 94ef4f1768df6283f370d1457e78b1838e0f4286 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 23 Oct 2025 13:10:19 +0000 Subject: [PATCH 10/32] [CI] Auto commit changes from spotless --- .../es93/ES93HnswScalarQuantizedVectorsFormat.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java index aec2120b97ace..9bd1ff1fbc113 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java @@ -69,7 +69,15 @@ protected FlatVectorsFormat flatVectorsFormat() { @Override public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { - return new Lucene99HnswVectorsWriter(state, maxConn, beamWidth, flatVectorsFormat.fieldsWriter(state), numMergeWorkers, mergeExec, 0); + return new Lucene99HnswVectorsWriter( + state, + maxConn, + beamWidth, + flatVectorsFormat.fieldsWriter(state), + numMergeWorkers, + mergeExec, + 0 + ); } @Override From 67203847e04d69c436b4d16dd94a6f591177bc24 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Thu, 23 Oct 2025 16:08:20 +0100 Subject: [PATCH 11/32] Update for ElementType change --- .../codec/vectors/es93/ES93FlatVectorFormat.java | 5 +++-- .../es93/ES93HnswScalarQuantizedVectorsFormat.java | 10 +++++----- .../es93/ES93ScalarQuantizedFlatVectorsFormat.java | 5 +++-- .../es93/ES93ScalarQuantizedVectorsFormat.java | 13 ++++++++----- .../es93/ES93FlatBFloat16VectorFormatTests.java | 2 +- .../vectors/es93/ES93FlatVectorFormatTests.java | 2 +- ...swScalarQuantizedBFloat16VectorsFormatTests.java | 6 +++--- .../ES93HnswScalarQuantizedVectorsFormatTests.java | 6 +++--- ...S93ScalarQuantizedBFloat16VectorFormatTests.java | 2 +- .../es93/ES93ScalarQuantizedVectorsFormatTests.java | 2 +- 10 files changed, 29 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormat.java index bd421abf167f0..bdad21596d479 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormat.java @@ -44,9 +44,10 @@ public ES93FlatVectorFormat() { format = new ES93GenericFlatVectorsFormat(); } - public ES93FlatVectorFormat(boolean useBFloat16) { + public ES93FlatVectorFormat(ES93GenericFlatVectorsFormat.ElementType elementType) { super(NAME); - format = new ES93GenericFlatVectorsFormat(useBFloat16, false); + assert elementType != ES93GenericFlatVectorsFormat.ElementType.BIT : "ES815BitFlatVectorFormat should be used for bits"; + format = new ES93GenericFlatVectorsFormat(elementType, false); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java index 9bd1ff1fbc113..f22b5d5f14c64 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java @@ -33,7 +33,7 @@ public ES93HnswScalarQuantizedVectorsFormat() { super(NAME); this.flatVectorsFormat = new ES93ScalarQuantizedFlatVectorsFormat( Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, - false, + ES93GenericFlatVectorsFormat.ElementType.STANDARD, false ); } @@ -42,24 +42,24 @@ public ES93HnswScalarQuantizedVectorsFormat( int maxConn, int beamWidth, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding, - boolean useBFloat16, + ES93GenericFlatVectorsFormat.ElementType elementType, boolean useDirectIO ) { super(NAME, maxConn, beamWidth); - this.flatVectorsFormat = new ES93ScalarQuantizedFlatVectorsFormat(encoding, useBFloat16, useDirectIO); + this.flatVectorsFormat = new ES93ScalarQuantizedFlatVectorsFormat(encoding, elementType, useDirectIO); } public ES93HnswScalarQuantizedVectorsFormat( int maxConn, int beamWidth, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding, - boolean useBFloat16, + ES93GenericFlatVectorsFormat.ElementType elementType, boolean useDirectIO, int numMergeWorkers, ExecutorService mergeExec ) { super(NAME, maxConn, beamWidth, numMergeWorkers, mergeExec); - this.flatVectorsFormat = new ES93ScalarQuantizedFlatVectorsFormat(encoding, useBFloat16, useDirectIO); + this.flatVectorsFormat = new ES93ScalarQuantizedFlatVectorsFormat(encoding, elementType, useDirectIO); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java index c4b99f75ee910..18967a39fe8b3 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java @@ -37,12 +37,13 @@ public class ES93ScalarQuantizedFlatVectorsFormat extends FlatVectorsFormat { public ES93ScalarQuantizedFlatVectorsFormat( Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding, - boolean useBFloat16, + ES93GenericFlatVectorsFormat.ElementType elementType, boolean useDirectIO ) { super(NAME); + assert elementType != ES93GenericFlatVectorsFormat.ElementType.BIT : "BIT should not be used with scalar quantization"; this.encoding = encoding; - this.rawVectorFormat = new ES93GenericFlatVectorsFormat(useBFloat16, useDirectIO); + this.rawVectorFormat = new ES93GenericFlatVectorsFormat(elementType, useDirectIO); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java index 175b4a79ecce5..647144cfaa2de 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java @@ -38,16 +38,19 @@ public class ES93ScalarQuantizedVectorsFormat extends KnnVectorsFormat { private final FlatVectorsFormat format; public ES93ScalarQuantizedVectorsFormat() { - this(false, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT); + this(ES93GenericFlatVectorsFormat.ElementType.STANDARD, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT); } - public ES93ScalarQuantizedVectorsFormat(boolean useBFloat16) { - this(useBFloat16, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT); + public ES93ScalarQuantizedVectorsFormat(ES93GenericFlatVectorsFormat.ElementType elementType) { + this(elementType, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT); } - public ES93ScalarQuantizedVectorsFormat(boolean useBFloat16, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding) { + public ES93ScalarQuantizedVectorsFormat( + ES93GenericFlatVectorsFormat.ElementType elementType, + Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding + ) { super(NAME); - this.format = new ES93ScalarQuantizedFlatVectorsFormat(encoding, useBFloat16, false); + this.format = new ES93ScalarQuantizedFlatVectorsFormat(encoding, elementType, false); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatBFloat16VectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatBFloat16VectorFormatTests.java index 41d777dde2cf9..91d4054ae94ed 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatBFloat16VectorFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatBFloat16VectorFormatTests.java @@ -41,7 +41,7 @@ public class ES93FlatBFloat16VectorFormatTests extends BaseBFloat16KnnVectorsFor @Override protected Codec getCodec() { - return TestUtil.alwaysKnnVectorsFormat(new ES93FlatVectorFormat(true)); + return TestUtil.alwaysKnnVectorsFormat(new ES93FlatVectorFormat(ES93GenericFlatVectorsFormat.ElementType.BFLOAT16)); } public void testSearchWithVisitedLimit() { diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormatTests.java index 18335758f82d8..1ada03a70bed6 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93FlatVectorFormatTests.java @@ -40,7 +40,7 @@ public class ES93FlatVectorFormatTests extends BaseKnnVectorsFormatTestCase { @Override protected Codec getCodec() { - return TestUtil.alwaysKnnVectorsFormat(new ES93FlatVectorFormat(false)); + return TestUtil.alwaysKnnVectorsFormat(new ES93FlatVectorFormat(ES93GenericFlatVectorsFormat.ElementType.STANDARD)); } public void testSearchWithVisitedLimit() { diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedBFloat16VectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedBFloat16VectorsFormatTests.java index e2bc3970398c5..a1bda3e4b2342 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedBFloat16VectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedBFloat16VectorsFormatTests.java @@ -34,7 +34,7 @@ protected KnnVectorsFormat createFormat() { DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, - true, + ES93GenericFlatVectorsFormat.ElementType.BFLOAT16, random().nextBoolean() ); } @@ -45,7 +45,7 @@ protected KnnVectorsFormat createFormat(int maxConn, int beamWidth) { maxConn, beamWidth, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, - true, + ES93GenericFlatVectorsFormat.ElementType.BFLOAT16, random().nextBoolean() ); } @@ -56,7 +56,7 @@ protected KnnVectorsFormat createFormat(int maxConn, int beamWidth, int numMerge maxConn, beamWidth, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, - true, + ES93GenericFlatVectorsFormat.ElementType.BFLOAT16, random().nextBoolean(), numMergeWorkers, service diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormatTests.java index 2d2564056afca..c2bf9e6352f15 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormatTests.java @@ -33,7 +33,7 @@ protected KnnVectorsFormat createFormat() { DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, - false, + ES93GenericFlatVectorsFormat.ElementType.STANDARD, random().nextBoolean() ); } @@ -44,7 +44,7 @@ protected KnnVectorsFormat createFormat(int maxConn, int beamWidth) { maxConn, beamWidth, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, - false, + ES93GenericFlatVectorsFormat.ElementType.STANDARD, random().nextBoolean() ); } @@ -55,7 +55,7 @@ protected KnnVectorsFormat createFormat(int maxConn, int beamWidth, int numMerge maxConn, beamWidth, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, - false, + ES93GenericFlatVectorsFormat.ElementType.STANDARD, random().nextBoolean(), numMergeWorkers, service diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedBFloat16VectorFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedBFloat16VectorFormatTests.java index 7e69492f6e848..57578097f2db4 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedBFloat16VectorFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedBFloat16VectorFormatTests.java @@ -46,7 +46,7 @@ public class ES93ScalarQuantizedBFloat16VectorFormatTests extends BaseBFloat16Kn @Override public void setUp() throws Exception { - format = new ES93ScalarQuantizedVectorsFormat(true); + format = new ES93ScalarQuantizedVectorsFormat(ES93GenericFlatVectorsFormat.ElementType.BFLOAT16); super.setUp(); } diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormatTests.java index 7c8461a6d033d..a880852378d61 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormatTests.java @@ -42,7 +42,7 @@ public class ES93ScalarQuantizedVectorsFormatTests extends BaseKnnVectorsFormatT @Override protected Codec getCodec() { - return TestUtil.alwaysKnnVectorsFormat(new ES93ScalarQuantizedVectorsFormat(false)); + return TestUtil.alwaysKnnVectorsFormat(new ES93ScalarQuantizedVectorsFormat(ES93GenericFlatVectorsFormat.ElementType.STANDARD)); } public void testSearchWithVisitedLimit() { From 115507aaca47a9ee65c3c63640339ec658a35189 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Fri, 24 Oct 2025 09:23:19 +0100 Subject: [PATCH 12/32] Revert "Use the reader in Lucene BWC" This reverts commit 064392fbf29afd634f54616b5a8db5bc7a71e96f. --- .../ES814ScalarQuantizedVectorsFormat.java | 1 - .../Lucene99ScalarQuantizedVectorsReader.java | 455 ++++++++++++++++++ 2 files changed, 455 insertions(+), 1 deletion(-) create mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/Lucene99ScalarQuantizedVectorsReader.java diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java index dd125b68d292a..692ccdfa9222c 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.codec.vectors; -import org.apache.lucene.backward_codecs.lucene99.Lucene99ScalarQuantizedVectorsReader; import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/Lucene99ScalarQuantizedVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/Lucene99ScalarQuantizedVectorsReader.java new file mode 100644 index 0000000000000..9c0e855faf6ad --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/Lucene99ScalarQuantizedVectorsReader.java @@ -0,0 +1,455 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2024 Elasticsearch B.V. + */ + +package org.elasticsearch.index.codec.vectors; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.codecs.hnsw.FlatVectorsScorer; +import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.internal.hppc.IntObjectHashMap; +import org.apache.lucene.search.VectorScorer; +import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.DataAccessHint; +import org.apache.lucene.store.FileDataHint; +import org.apache.lucene.store.FileTypeHint; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.RamUsageEstimator; +import org.apache.lucene.util.hnsw.RandomVectorScorer; +import org.apache.lucene.util.quantization.QuantizedByteVectorValues; +import org.apache.lucene.util.quantization.QuantizedVectorsReader; +import org.apache.lucene.util.quantization.ScalarQuantizer; +import org.elasticsearch.core.SuppressForbidden; + +import java.io.IOException; +import java.util.Map; + +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSimilarityFunction; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding; + +/** + * Copied from Lucene 10.3. + */ +@SuppressForbidden(reason = "Lucene classes") +final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReader implements QuantizedVectorsReader { + + private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(Lucene99ScalarQuantizedVectorsReader.class); + + static final int VERSION_START = 0; + static final int VERSION_ADD_BITS = 1; + static final int VERSION_CURRENT = VERSION_ADD_BITS; + static final String META_CODEC_NAME = "Lucene99ScalarQuantizedVectorsFormatMeta"; + static final String VECTOR_DATA_CODEC_NAME = "Lucene99ScalarQuantizedVectorsFormatData"; + static final String META_EXTENSION = "vemq"; + static final String VECTOR_DATA_EXTENSION = "veq"; + + /** Dynamic confidence interval */ + public static final float DYNAMIC_CONFIDENCE_INTERVAL = 0f; + + private final IntObjectHashMap fields = new IntObjectHashMap<>(); + private final IndexInput quantizedVectorData; + private final FlatVectorsReader rawVectorsReader; + private final FieldInfos fieldInfos; + + Lucene99ScalarQuantizedVectorsReader(SegmentReadState state, FlatVectorsReader rawVectorsReader, FlatVectorsScorer scorer) + throws IOException { + super(scorer); + this.rawVectorsReader = rawVectorsReader; + this.fieldInfos = state.fieldInfos; + int versionMeta = -1; + String metaFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, META_EXTENSION); + boolean success = false; + try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName)) { + Throwable priorE = null; + try { + versionMeta = CodecUtil.checkIndexHeader( + meta, + META_CODEC_NAME, + VERSION_START, + VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix + ); + readFields(meta, versionMeta, state.fieldInfos); + } catch (Throwable exception) { + priorE = exception; + } finally { + CodecUtil.checkFooter(meta, priorE); + } + quantizedVectorData = openDataInput( + state, + versionMeta, + VECTOR_DATA_EXTENSION, + VECTOR_DATA_CODEC_NAME, + // Quantized vectors are accessed randomly from their node ID stored in the HNSW + // graph. + state.context.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM) + ); + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } + } + + private void readFields(ChecksumIndexInput meta, int versionMeta, FieldInfos infos) throws IOException { + for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) { + FieldInfo info = infos.fieldInfo(fieldNumber); + if (info == null) { + throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta); + } + FieldEntry fieldEntry = readField(meta, versionMeta, info); + validateFieldEntry(info, fieldEntry); + fields.put(info.number, fieldEntry); + } + } + + static void validateFieldEntry(FieldInfo info, FieldEntry fieldEntry) { + int dimension = info.getVectorDimension(); + if (dimension != fieldEntry.dimension) { + throw new IllegalStateException( + "Inconsistent vector dimension for field=\"" + info.name + "\"; " + dimension + " != " + fieldEntry.dimension + ); + } + + final long quantizedVectorBytes; + if (fieldEntry.bits <= 4 && fieldEntry.compress) { + // two dimensions -> one byte + quantizedVectorBytes = ((dimension + 1) >> 1) + Float.BYTES; + } else { + // one dimension -> one byte + quantizedVectorBytes = dimension + Float.BYTES; + } + long numQuantizedVectorBytes = Math.multiplyExact(quantizedVectorBytes, fieldEntry.size); + if (numQuantizedVectorBytes != fieldEntry.vectorDataLength) { + throw new IllegalStateException( + "Quantized vector data length " + + fieldEntry.vectorDataLength + + " not matching size=" + + fieldEntry.size + + " * (dim=" + + dimension + + " + 4)" + + " = " + + numQuantizedVectorBytes + ); + } + } + + @Override + public void checkIntegrity() throws IOException { + rawVectorsReader.checkIntegrity(); + CodecUtil.checksumEntireFile(quantizedVectorData); + } + + private FieldEntry getFieldEntry(String field) { + final FieldInfo info = fieldInfos.fieldInfo(field); + final FieldEntry fieldEntry; + if (info == null || (fieldEntry = fields.get(info.number)) == null) { + throw new IllegalArgumentException("field=\"" + field + "\" not found"); + } + if (fieldEntry.vectorEncoding != VectorEncoding.FLOAT32) { + throw new IllegalArgumentException( + "field=\"" + field + "\" is encoded as: " + fieldEntry.vectorEncoding + " expected: " + VectorEncoding.FLOAT32 + ); + } + return fieldEntry; + } + + @Override + public FloatVectorValues getFloatVectorValues(String field) throws IOException { + final FieldEntry fieldEntry = getFieldEntry(field); + final FloatVectorValues rawVectorValues = rawVectorsReader.getFloatVectorValues(field); + OffHeapQuantizedByteVectorValues quantizedByteVectorValues = OffHeapQuantizedByteVectorValues.load( + fieldEntry.ordToDoc, + fieldEntry.dimension, + fieldEntry.size, + fieldEntry.scalarQuantizer, + fieldEntry.similarityFunction, + vectorScorer, + fieldEntry.compress, + fieldEntry.vectorDataOffset, + fieldEntry.vectorDataLength, + quantizedVectorData + ); + return new QuantizedVectorValues(rawVectorValues, quantizedByteVectorValues); + } + + @Override + public ByteVectorValues getByteVectorValues(String field) throws IOException { + return rawVectorsReader.getByteVectorValues(field); + } + + private static IndexInput openDataInput( + SegmentReadState state, + int versionMeta, + String fileExtension, + String codecName, + IOContext context + ) throws IOException { + String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, fileExtension); + IndexInput in = state.directory.openInput(fileName, context); + boolean success = false; + try { + int versionVectorData = CodecUtil.checkIndexHeader( + in, + codecName, + VERSION_START, + VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix + ); + if (versionMeta != versionVectorData) { + throw new CorruptIndexException( + "Format versions mismatch: meta=" + versionMeta + ", " + codecName + "=" + versionVectorData, + in + ); + } + CodecUtil.retrieveChecksum(in); + success = true; + return in; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(in); + } + } + } + + @Override + public RandomVectorScorer getRandomVectorScorer(String field, float[] target) throws IOException { + final FieldEntry fieldEntry = getFieldEntry(field); + if (fieldEntry.scalarQuantizer == null) { + return rawVectorsReader.getRandomVectorScorer(field, target); + } + OffHeapQuantizedByteVectorValues vectorValues = OffHeapQuantizedByteVectorValues.load( + fieldEntry.ordToDoc, + fieldEntry.dimension, + fieldEntry.size, + fieldEntry.scalarQuantizer, + fieldEntry.similarityFunction, + vectorScorer, + fieldEntry.compress, + fieldEntry.vectorDataOffset, + fieldEntry.vectorDataLength, + quantizedVectorData + ); + return vectorScorer.getRandomVectorScorer(fieldEntry.similarityFunction, vectorValues, target); + } + + @Override + public RandomVectorScorer getRandomVectorScorer(String field, byte[] target) throws IOException { + return rawVectorsReader.getRandomVectorScorer(field, target); + } + + @Override + public void close() throws IOException { + IOUtils.close(quantizedVectorData, rawVectorsReader); + } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + fields.ramBytesUsed() + rawVectorsReader.ramBytesUsed(); + } + + @Override + public Map getOffHeapByteSize(FieldInfo fieldInfo) { + var raw = rawVectorsReader.getOffHeapByteSize(fieldInfo); + var fieldEntry = fields.get(fieldInfo.number); + if (fieldEntry == null) { + assert fieldInfo.getVectorEncoding() == VectorEncoding.BYTE; + return raw; + } + var quant = Map.of(VECTOR_DATA_EXTENSION, fieldEntry.vectorDataLength()); + return KnnVectorsReader.mergeOffHeapByteSizeMaps(raw, quant); + } + + private FieldEntry readField(IndexInput input, int versionMeta, FieldInfo info) throws IOException { + VectorEncoding vectorEncoding = readVectorEncoding(input); + VectorSimilarityFunction similarityFunction = readSimilarityFunction(input); + if (similarityFunction != info.getVectorSimilarityFunction()) { + throw new IllegalStateException( + "Inconsistent vector similarity function for field=\"" + + info.name + + "\"; " + + similarityFunction + + " != " + + info.getVectorSimilarityFunction() + ); + } + return FieldEntry.create(input, versionMeta, vectorEncoding, info.getVectorSimilarityFunction()); + } + + @Override + public QuantizedByteVectorValues getQuantizedVectorValues(String field) throws IOException { + final FieldEntry fieldEntry = getFieldEntry(field); + return OffHeapQuantizedByteVectorValues.load( + fieldEntry.ordToDoc, + fieldEntry.dimension, + fieldEntry.size, + fieldEntry.scalarQuantizer, + fieldEntry.similarityFunction, + vectorScorer, + fieldEntry.compress, + fieldEntry.vectorDataOffset, + fieldEntry.vectorDataLength, + quantizedVectorData + ); + } + + @Override + public ScalarQuantizer getQuantizationState(String field) { + final FieldEntry fieldEntry = getFieldEntry(field); + return fieldEntry.scalarQuantizer; + } + + private record FieldEntry( + VectorSimilarityFunction similarityFunction, + VectorEncoding vectorEncoding, + int dimension, + long vectorDataOffset, + long vectorDataLength, + ScalarQuantizer scalarQuantizer, + int size, + byte bits, + boolean compress, + OrdToDocDISIReaderConfiguration ordToDoc + ) { + + static FieldEntry create( + IndexInput input, + int versionMeta, + VectorEncoding vectorEncoding, + VectorSimilarityFunction similarityFunction + ) throws IOException { + final var vectorDataOffset = input.readVLong(); + final var vectorDataLength = input.readVLong(); + final var dimension = input.readVInt(); + final var size = input.readInt(); + final ScalarQuantizer scalarQuantizer; + final byte bits; + final boolean compress; + if (size > 0) { + if (versionMeta < VERSION_ADD_BITS) { + int floatBits = input.readInt(); // confidenceInterval, unused + if (floatBits == -1) { // indicates a null confidence interval + throw new CorruptIndexException("Missing confidence interval for scalar quantizer", input); + } + float confidenceInterval = Float.intBitsToFloat(floatBits); + // indicates a dynamic interval, which shouldn't be provided in this version + if (confidenceInterval == DYNAMIC_CONFIDENCE_INTERVAL) { + throw new CorruptIndexException("Invalid confidence interval for scalar quantizer: " + confidenceInterval, input); + } + bits = (byte) 7; + compress = false; + float minQuantile = Float.intBitsToFloat(input.readInt()); + float maxQuantile = Float.intBitsToFloat(input.readInt()); + scalarQuantizer = new ScalarQuantizer(minQuantile, maxQuantile, (byte) 7); + } else { + input.readInt(); // confidenceInterval, unused + bits = input.readByte(); + compress = input.readByte() == 1; + float minQuantile = Float.intBitsToFloat(input.readInt()); + float maxQuantile = Float.intBitsToFloat(input.readInt()); + scalarQuantizer = new ScalarQuantizer(minQuantile, maxQuantile, bits); + } + } else { + scalarQuantizer = null; + bits = (byte) 7; + compress = false; + } + final var ordToDoc = OrdToDocDISIReaderConfiguration.fromStoredMeta(input, size); + return new FieldEntry( + similarityFunction, + vectorEncoding, + dimension, + vectorDataOffset, + vectorDataLength, + scalarQuantizer, + size, + bits, + compress, + ordToDoc + ); + } + } + + private static final class QuantizedVectorValues extends FloatVectorValues { + private final FloatVectorValues rawVectorValues; + private final QuantizedByteVectorValues quantizedVectorValues; + + QuantizedVectorValues(FloatVectorValues rawVectorValues, QuantizedByteVectorValues quantizedVectorValues) { + this.rawVectorValues = rawVectorValues; + this.quantizedVectorValues = quantizedVectorValues; + } + + @Override + public int dimension() { + return rawVectorValues.dimension(); + } + + @Override + public int size() { + return rawVectorValues.size(); + } + + @Override + public float[] vectorValue(int ord) throws IOException { + return rawVectorValues.vectorValue(ord); + } + + @Override + public int ordToDoc(int ord) { + return rawVectorValues.ordToDoc(ord); + } + + @Override + public QuantizedVectorValues copy() throws IOException { + return new QuantizedVectorValues(rawVectorValues.copy(), quantizedVectorValues.copy()); + } + + @Override + public VectorScorer scorer(float[] query) throws IOException { + return quantizedVectorValues.scorer(query); + } + + @Override + public VectorScorer rescorer(float[] query) throws IOException { + return rawVectorValues.rescorer(query); + } + + @Override + public DocIndexIterator iterator() { + return rawVectorValues.iterator(); + } + } +} From 44ecd39edd0b8be4df65bfd8aaf34b56560db971 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Fri, 24 Oct 2025 10:01:34 +0100 Subject: [PATCH 13/32] Remove intermediate class --- .../ES93HnswScalarQuantizedVectorsFormat.java | 35 ++++++--- .../ES93ScalarQuantizedFlatVectorsFormat.java | 78 ------------------- .../ES93ScalarQuantizedVectorsFormat.java | 33 ++++++-- 3 files changed, 51 insertions(+), 95 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java index f22b5d5f14c64..6bd225123d425 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java @@ -11,8 +11,12 @@ import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorScorer; import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsReader; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsWriter; import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter; import org.apache.lucene.index.SegmentReadState; @@ -26,16 +30,17 @@ public class ES93HnswScalarQuantizedVectorsFormat extends AbstractHnswVectorsFor static final String NAME = "ES93HnswScalarQuantizedVectorsFormat"; - /** The format for storing, reading, merging vectors on disk */ - private final FlatVectorsFormat flatVectorsFormat; + static final Lucene104ScalarQuantizedVectorScorer flatVectorScorer = new Lucene104ScalarQuantizedVectorScorer( + FlatVectorScorerUtil.getLucene99FlatVectorsScorer() + ); + + private final Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding; + private final FlatVectorsFormat rawVectorFormat; public ES93HnswScalarQuantizedVectorsFormat() { super(NAME); - this.flatVectorsFormat = new ES93ScalarQuantizedFlatVectorsFormat( - Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT, - ES93GenericFlatVectorsFormat.ElementType.STANDARD, - false - ); + this.encoding = Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT; + this.rawVectorFormat = new ES93GenericFlatVectorsFormat(ES93GenericFlatVectorsFormat.ElementType.STANDARD, false); } public ES93HnswScalarQuantizedVectorsFormat( @@ -46,7 +51,8 @@ public ES93HnswScalarQuantizedVectorsFormat( boolean useDirectIO ) { super(NAME, maxConn, beamWidth); - this.flatVectorsFormat = new ES93ScalarQuantizedFlatVectorsFormat(encoding, elementType, useDirectIO); + this.encoding = encoding; + this.rawVectorFormat = new ES93GenericFlatVectorsFormat(elementType, useDirectIO); } public ES93HnswScalarQuantizedVectorsFormat( @@ -59,12 +65,13 @@ public ES93HnswScalarQuantizedVectorsFormat( ExecutorService mergeExec ) { super(NAME, maxConn, beamWidth, numMergeWorkers, mergeExec); - this.flatVectorsFormat = new ES93ScalarQuantizedFlatVectorsFormat(encoding, elementType, useDirectIO); + this.encoding = encoding; + this.rawVectorFormat = new ES93GenericFlatVectorsFormat(elementType, useDirectIO); } @Override protected FlatVectorsFormat flatVectorsFormat() { - return flatVectorsFormat; + return rawVectorFormat; } @Override @@ -73,7 +80,8 @@ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException state, maxConn, beamWidth, - flatVectorsFormat.fieldsWriter(state), + new Lucene104ScalarQuantizedVectorsWriter(state, encoding, rawVectorFormat.fieldsWriter(state), flatVectorScorer) { + }, numMergeWorkers, mergeExec, 0 @@ -82,6 +90,9 @@ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException @Override public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { - return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state)); + return new Lucene99HnswVectorsReader( + state, + new Lucene104ScalarQuantizedVectorsReader(state, rawVectorFormat.fieldsReader(state), flatVectorScorer) + ); } } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java deleted file mode 100644 index 18967a39fe8b3..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedFlatVectorsFormat.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.codec.vectors.es93; - -import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; -import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; -import org.apache.lucene.codecs.hnsw.FlatVectorsReader; -import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; -import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorScorer; -import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat; -import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsReader; -import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsWriter; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.SegmentWriteState; - -import java.io.IOException; - -import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; - -public class ES93ScalarQuantizedFlatVectorsFormat extends FlatVectorsFormat { - - static final String NAME = "ES93ScalarQuantizedFlatVectorsFormat"; - - static final Lucene104ScalarQuantizedVectorScorer flatVectorScorer = new Lucene104ScalarQuantizedVectorScorer( - FlatVectorScorerUtil.getLucene99FlatVectorsScorer() - ); - - private final Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding; - private final FlatVectorsFormat rawVectorFormat; - - public ES93ScalarQuantizedFlatVectorsFormat( - Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding, - ES93GenericFlatVectorsFormat.ElementType elementType, - boolean useDirectIO - ) { - super(NAME); - assert elementType != ES93GenericFlatVectorsFormat.ElementType.BIT : "BIT should not be used with scalar quantization"; - this.encoding = encoding; - this.rawVectorFormat = new ES93GenericFlatVectorsFormat(elementType, useDirectIO); - } - - @Override - public int getMaxDimensions(String fieldName) { - return MAX_DIMS_COUNT; - } - - @Override - public String toString() { - return NAME - + "(name=" - + NAME - + ", encoding=" - + encoding - + ", flatVectorScorer=" - + flatVectorScorer - + ", rawVectorFormat=" - + rawVectorFormat - + ")"; - } - - @Override - public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { - return new Lucene104ScalarQuantizedVectorsWriter(state, encoding, rawVectorFormat.fieldsWriter(state), flatVectorScorer) { - }; - } - - @Override - public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException { - return new Lucene104ScalarQuantizedVectorsReader(state, rawVectorFormat.fieldsReader(state), flatVectorScorer); - } -} diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java index 647144cfaa2de..0d08f738f65a7 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java @@ -12,9 +12,13 @@ import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorScorer; import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsReader; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsWriter; import org.apache.lucene.index.ByteVectorValues; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FloatVectorValues; @@ -35,7 +39,12 @@ public class ES93ScalarQuantizedVectorsFormat extends KnnVectorsFormat { static final String NAME = "ES93ScalarQuantizedVectorsFormat"; - private final FlatVectorsFormat format; + static final Lucene104ScalarQuantizedVectorScorer flatVectorScorer = new Lucene104ScalarQuantizedVectorScorer( + FlatVectorScorerUtil.getLucene99FlatVectorsScorer() + ); + + private final Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding; + private final FlatVectorsFormat rawVectorFormat; public ES93ScalarQuantizedVectorsFormat() { this(ES93GenericFlatVectorsFormat.ElementType.STANDARD, Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding.SEVEN_BIT); @@ -50,17 +59,22 @@ public ES93ScalarQuantizedVectorsFormat( Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding encoding ) { super(NAME); - this.format = new ES93ScalarQuantizedFlatVectorsFormat(encoding, elementType, false); + assert elementType != ES93GenericFlatVectorsFormat.ElementType.BIT : "BIT should not be used with scalar quantization"; + this.encoding = encoding; + this.rawVectorFormat = new ES93GenericFlatVectorsFormat(elementType, false); } @Override public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { - return format.fieldsWriter(state); + return new Lucene104ScalarQuantizedVectorsWriter(state, encoding, rawVectorFormat.fieldsWriter(state), flatVectorScorer) { + }; } @Override public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { - return new ES93FlatVectorsReader(format.fieldsReader(state)); + return new ES93FlatVectorsReader( + new Lucene104ScalarQuantizedVectorsReader(state, rawVectorFormat.fieldsReader(state), flatVectorScorer) + ); } @Override @@ -70,7 +84,16 @@ public int getMaxDimensions(String fieldName) { @Override public String toString() { - return NAME + "(name=" + NAME + ", innerFormat=" + format + ")"; + return NAME + + "(name=" + + NAME + + ", encoding=" + + encoding + + ", flatVectorScorer=" + + flatVectorScorer + + ", rawVectorFormat=" + + rawVectorFormat + + ")"; } public static class ES93FlatVectorsReader extends KnnVectorsReader { From f6ee76915a1b7f0c035774b455fe567f0acd1065 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Fri, 31 Oct 2025 12:38:07 +0000 Subject: [PATCH 14/32] Use public constructor --- .../vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java | 3 +-- .../codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java index 6bd225123d425..4f47b82c3b5a6 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93HnswScalarQuantizedVectorsFormat.java @@ -80,8 +80,7 @@ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException state, maxConn, beamWidth, - new Lucene104ScalarQuantizedVectorsWriter(state, encoding, rawVectorFormat.fieldsWriter(state), flatVectorScorer) { - }, + new Lucene104ScalarQuantizedVectorsWriter(state, encoding, rawVectorFormat.fieldsWriter(state), flatVectorScorer), numMergeWorkers, mergeExec, 0 diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java index 0d08f738f65a7..90cb3aec59ed8 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java @@ -66,8 +66,7 @@ public ES93ScalarQuantizedVectorsFormat( @Override public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { - return new Lucene104ScalarQuantizedVectorsWriter(state, encoding, rawVectorFormat.fieldsWriter(state), flatVectorScorer) { - }; + return new Lucene104ScalarQuantizedVectorsWriter(state, encoding, rawVectorFormat.fieldsWriter(state), flatVectorScorer); } @Override From 3017e334274a7292997b0fea77f90d2c73b58eba Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Mon, 3 Nov 2025 12:21:57 +0100 Subject: [PATCH 15/32] Mute org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTests testUpdates #137512 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 231138c6b0efd..4a97af263c9d5 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -504,6 +504,9 @@ tests: - class: org.elasticsearch.xpack.esql.qa.single_node.PushQueriesIT method: testEqualityAndOther {SEMANTIC_TEXT_WITH_KEYWORD} issue: https://github.com/elastic/elasticsearch/issues/137491 +- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTests + method: testUpdates + issue: https://github.com/elastic/elasticsearch/issues/137512 # Examples: # From 70507a6d8dfca5972297b0b9968586301bce8c07 Mon Sep 17 00:00:00 2001 From: Ievgen Degtiarenko Date: Mon, 3 Nov 2025 14:12:33 +0100 Subject: [PATCH 16/32] Verify execution info in remote index resolution tests (#137361) --- .../esql/plugin/RemoteIndexResolutionIT.java | 110 ++++++++++++++++-- .../xpack/esql/action/EsqlQueryRequest.java | 6 +- 2 files changed, 102 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/RemoteIndexResolutionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/RemoteIndexResolutionIT.java index d37f68494ebba..085f58d6c09b2 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/RemoteIndexResolutionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/RemoteIndexResolutionIT.java @@ -7,15 +7,18 @@ package org.elasticsearch.xpack.esql.plugin; -import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.MatchNoneQueryBuilder; import org.elasticsearch.transport.NoSuchRemoteClusterException; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.AbstractCrossClusterTestCase; +import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo.Cluster.Status; import org.elasticsearch.xpack.esql.action.EsqlQueryAction; import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; import org.elasticsearch.xpack.esql.action.EsqlQueryResponse; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; +import java.util.List; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -29,9 +32,14 @@ public class RemoteIndexResolutionIT extends AbstractCrossClusterTestCase { public void testResolvesRemoteIndex() { indexRandom(REMOTE_CLUSTER_1, true, "index-1", 1); - try (var response = run(syncEsqlQueryRequest().query("FROM " + REMOTE_CLUSTER_1 + ":index-1 METADATA _index"))) { + try ( + var response = run( + syncEsqlQueryRequest().query("FROM " + REMOTE_CLUSTER_1 + ":index-1 METADATA _index").includeCCSMetadata(true) + ) + ) { assertOk(response); assertResultConcreteIndices(response, REMOTE_CLUSTER_1 + ":index-1"); + assertExecutionInfo(response, new EsqlResponseExecutionInfo(REMOTE_CLUSTER_1, "index-1", Status.SUCCESSFUL)); } } @@ -40,6 +48,17 @@ public void testResolveRemoteUnknownIndex() { // This index is mixed into the resultset to test error handling of the missing concrete remote, not the empty result. indexRandom(LOCAL_CLUSTER, true, "data", 1); + expectThrows( + VerificationException.class, + containsString("Unknown index [" + REMOTE_CLUSTER_1 + ":fake]"), + () -> run(syncEsqlQueryRequest().query("FROM data," + REMOTE_CLUSTER_1 + ":fake")) + ); + expectThrows( + VerificationException.class, + containsString("Unknown index [" + REMOTE_CLUSTER_1 + ":fake]"), + () -> run(syncEsqlQueryRequest().query("FROM data," + REMOTE_CLUSTER_1 + ":fake").allowPartialResults(true)) + ); + setSkipUnavailable(REMOTE_CLUSTER_1, false); expectThrows( VerificationException.class, @@ -48,15 +67,33 @@ public void testResolveRemoteUnknownIndex() { ); setSkipUnavailable(REMOTE_CLUSTER_1, true); - try (var response = run(syncEsqlQueryRequest().query("FROM data," + REMOTE_CLUSTER_1 + ":fake METADATA _index"))) { + try ( + var response = run( + syncEsqlQueryRequest().query("FROM data," + REMOTE_CLUSTER_1 + ":fake METADATA _index").includeCCSMetadata(true) + ) + ) { assertPartial(response); assertResultConcreteIndices(response, "data"); + assertExecutionInfo( + response, + new EsqlResponseExecutionInfo(LOCAL_CLUSTER, "data", Status.SUCCESSFUL), + new EsqlResponseExecutionInfo(REMOTE_CLUSTER_1, "fake", Status.SKIPPED) + ); } setSkipUnavailable(REMOTE_CLUSTER_1, null); - try (var response = run(syncEsqlQueryRequest().query("FROM data," + REMOTE_CLUSTER_1 + ":fake METADATA _index"))) { + try ( + var response = run( + syncEsqlQueryRequest().query("FROM data," + REMOTE_CLUSTER_1 + ":fake METADATA _index").includeCCSMetadata(true) + ) + ) { assertPartial(response); assertResultConcreteIndices(response, "data"); + assertExecutionInfo( + response, + new EsqlResponseExecutionInfo(LOCAL_CLUSTER, "data", Status.SUCCESSFUL), + new EsqlResponseExecutionInfo(REMOTE_CLUSTER_1, "fake", Status.SKIPPED) + ); } } @@ -64,9 +101,18 @@ public void testResolvesLocalAndRemoteIndex() { indexRandom(LOCAL_CLUSTER, true, "index-1", 1); indexRandom(REMOTE_CLUSTER_1, true, "index-1", 1); - try (var response = run(syncEsqlQueryRequest().query("FROM index-1," + REMOTE_CLUSTER_1 + ":index-1 METADATA _index"))) { + try ( + var response = run( + syncEsqlQueryRequest().query("FROM index-1," + REMOTE_CLUSTER_1 + ":index-1 METADATA _index").includeCCSMetadata(true) + ) + ) { assertOk(response); assertResultConcreteIndices(response, "index-1", REMOTE_CLUSTER_1 + ":index-1"); + assertExecutionInfo( + response, + new EsqlResponseExecutionInfo(LOCAL_CLUSTER, "index-1", Status.SUCCESSFUL), + new EsqlResponseExecutionInfo(REMOTE_CLUSTER_1, "index-1", Status.SUCCESSFUL) + ); } } @@ -75,13 +121,19 @@ public void testResolvesRemotesWithPattern() { indexRandom(REMOTE_CLUSTER_1, true, "index-1", 1); indexRandom(REMOTE_CLUSTER_2, true, "index-1", 1); - try (var response = run(syncEsqlQueryRequest().query("FROM *:index-1 METADATA _index"))) { + try (var response = run(syncEsqlQueryRequest().query("FROM *:index-1 METADATA _index").includeCCSMetadata(true))) { assertOk(response); assertResultConcreteIndices(response, REMOTE_CLUSTER_1 + ":index-1", REMOTE_CLUSTER_2 + ":index-1"); // local is not included + assertExecutionInfo( + response, + new EsqlResponseExecutionInfo(REMOTE_CLUSTER_1, "index-1", Status.SUCCESSFUL), + new EsqlResponseExecutionInfo(REMOTE_CLUSTER_2, "index-1", Status.SUCCESSFUL) + ); } - try (var response = run(syncEsqlQueryRequest().query("FROM fake*:index-1 METADATA _index"))) { + try (var response = run(syncEsqlQueryRequest().query("FROM fake*:index-1 METADATA _index").includeCCSMetadata(true))) { assertOk(response); assertResultConcreteIndices(response); // empty + assertExecutionInfo(response); // empty } } @@ -93,14 +145,33 @@ public void testDoesNotResolvesUnknownRemote() { ); } - private EsqlQueryResponse run(EsqlQueryRequest request) { - try { - return client(LOCAL_CLUSTER).execute(EsqlQueryAction.INSTANCE, request).actionGet(30, TimeUnit.SECONDS); - } catch (ElasticsearchTimeoutException e) { - throw new AssertionError("timeout", e); + public void testResolutionWithFilter() { + indexRandom(REMOTE_CLUSTER_1, true, "index-1", 1); + + try ( + var response = run( + syncEsqlQueryRequest().query("FROM " + REMOTE_CLUSTER_1 + ":index-1 METADATA _index").filter(new MatchAllQueryBuilder()) + ) + ) { + assertOk(response); + assertResultConcreteIndices(response, REMOTE_CLUSTER_1 + ":index-1"); + assertExecutionInfo(response, new EsqlResponseExecutionInfo(REMOTE_CLUSTER_1, "index-1", Status.SUCCESSFUL)); + } + try ( + var response = run( + syncEsqlQueryRequest().query("FROM " + REMOTE_CLUSTER_1 + ":index-1 METADATA _index").filter(new MatchNoneQueryBuilder()) + ) + ) { + assertOk(response); + assertResultConcreteIndices(response); + assertExecutionInfo(response, new EsqlResponseExecutionInfo(REMOTE_CLUSTER_1, "index-1", Status.SUCCESSFUL)); } } + private EsqlQueryResponse run(EsqlQueryRequest request) { + return client(LOCAL_CLUSTER).execute(EsqlQueryAction.INSTANCE, request).actionGet(30, TimeUnit.SECONDS); + } + private void indexRandom(String clusterAlias, boolean forceRefresh, String index, int numDocs) { assert numDocs == 1; var client = client(clusterAlias); @@ -123,6 +194,10 @@ private static void assertResultConcreteIndices(EsqlQueryResponse response, Obje assertThat(() -> response.column(indexColumn), containsInAnyOrder(indices)); } + private static void assertExecutionInfo(EsqlQueryResponse response, EsqlResponseExecutionInfo... infos) { + assertThat(executionInfo(response), containsInAnyOrder(infos)); + } + private static int findIndexColumn(EsqlQueryResponse response) { for (int c = 0; c < response.columns().size(); c++) { if (Objects.equals(response.columns().get(c).name(), MetadataAttribute.INDEX)) { @@ -131,4 +206,15 @@ private static int findIndexColumn(EsqlQueryResponse response) { } throw new AssertionError("no _index column found"); } + + private static List executionInfo(EsqlQueryResponse response) { + return response.getExecutionInfo() + .getClusters() + .values() + .stream() + .map(cluster -> new EsqlResponseExecutionInfo(cluster.getClusterAlias(), cluster.getIndexExpression(), cluster.getStatus())) + .toList(); + } + + private record EsqlResponseExecutionInfo(String alias, String index, Status status) {} } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java index b106a9a2d34b7..8b71536bcf7f9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java @@ -143,16 +143,18 @@ public void profile(boolean profile) { this.profile = profile; } - public void includeCCSMetadata(Boolean include) { + public EsqlQueryRequest includeCCSMetadata(Boolean include) { this.includeCCSMetadata = include; + return this; } public Boolean includeCCSMetadata() { return includeCCSMetadata; } - public void includeExecutionMetadata(Boolean include) { + public EsqlQueryRequest includeExecutionMetadata(Boolean include) { this.includeExecutionMetadata = include; + return this; } public Boolean includeExecutionMetadata() { From 0b8a37a2cc33595684ff1e2417d63d013a650e4a Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 3 Nov 2025 14:59:02 +0100 Subject: [PATCH 17/32] Handle ._original and._ignore_malformed stored fields correctly with fls (#137442) --- docs/changelog/137442.yaml | 5 + .../mapper/IgnoreMalformedStoredValues.java | 5 +- .../index/mapper/TextFamilyFieldType.java | 3 +- .../accesscontrol/FieldSubsetReader.java | 42 ++- .../SecurityIndexReaderWrapper.java | 6 +- .../authz/permission/FieldPermissions.java | 11 +- .../accesscontrol/FieldSubsetReaderTests.java | 181 +++++++++--- ...ldDataCacheWithFieldSubsetReaderTests.java | 12 +- ..._field_level_security_synthetic_source.yml | 257 ++++++++++++++++++ .../wildcard/mapper/WildcardFieldMapper.java | 3 +- 10 files changed, 469 insertions(+), 56 deletions(-) create mode 100644 docs/changelog/137442.yaml diff --git a/docs/changelog/137442.yaml b/docs/changelog/137442.yaml new file mode 100644 index 0000000000000..b999927dff74a --- /dev/null +++ b/docs/changelog/137442.yaml @@ -0,0 +1,5 @@ +pr: 137442 +summary: Handle ._original stored fields with fls +area: "Authorization" +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java index 8544ddd0194f3..aa7b395519802 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java @@ -26,6 +26,9 @@ * {@code _source}. */ public abstract class IgnoreMalformedStoredValues { + + public static final String IGNORE_MALFORMED_FIELD_NAME_SUFFIX = "._ignore_malformed"; + /** * Creates a stored field that stores malformed data to be used in synthetic source. * Name of the stored field is original name of the field with added conventional suffix. @@ -143,6 +146,6 @@ public void reset() { } public static String name(String fieldName) { - return fieldName + "._ignore_malformed"; + return fieldName + IGNORE_MALFORMED_FIELD_NAME_SUFFIX; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldType.java index efd2bafd8e2c9..de03b116f0c19 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFamilyFieldType.java @@ -20,6 +20,7 @@ */ public abstract class TextFamilyFieldType extends StringFieldType { + public static final String FALLBACK_FIELD_NAME_SUFFIX = "._original"; private final boolean isSyntheticSourceEnabled; private final boolean isWithinMultiField; @@ -51,7 +52,7 @@ public boolean isWithinMultiField() { * stored for whatever reason. */ public String syntheticSourceFallbackFieldName() { - return name() + "._original"; + return name() + FALLBACK_FIELD_NAME_SUFFIX; } /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java index 1b25b36e1d2db..548db7e08f598 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java @@ -41,8 +41,10 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.IgnoreMalformedStoredValues; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.TextFamilyFieldType; import org.elasticsearch.transport.Transports; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -55,6 +57,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; /** * A {@link FilterLeafReader} that exposes only a subset @@ -69,15 +72,18 @@ public final class FieldSubsetReader extends SequentialStoredFieldsLeafReader { * Note that for convenience, the returned reader * can be used normally (e.g. passed to {@link DirectoryReader#openIfChanged(DirectoryReader)}) * and so on. - * @param in reader to filter - * @param filter fields to filter. + * + * @param in reader to filter + * @param filter fields to filter. + * @param isMapped whether a field is mapped or not. */ public static DirectoryReader wrap( DirectoryReader in, CharacterRunAutomaton filter, - IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat, + Function isMapped ) throws IOException { - return new FieldSubsetDirectoryReader(in, filter, ignoredSourceFormat); + return new FieldSubsetDirectoryReader(in, filter, ignoredSourceFormat, isMapped); } // wraps subreaders with fieldsubsetreaders. @@ -85,17 +91,19 @@ static class FieldSubsetDirectoryReader extends FilterDirectoryReader { private final CharacterRunAutomaton filter; private final IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat; + private final Function isMapped; FieldSubsetDirectoryReader( DirectoryReader in, final CharacterRunAutomaton filter, - final IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat + final IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat, + Function isMapped ) throws IOException { super(in, new FilterDirectoryReader.SubReaderWrapper() { @Override public LeafReader wrap(LeafReader reader) { try { - return new FieldSubsetReader(reader, filter, ignoredSourceFormat); + return new FieldSubsetReader(reader, filter, ignoredSourceFormat, isMapped); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -103,12 +111,13 @@ public LeafReader wrap(LeafReader reader) { }); this.filter = filter; this.ignoredSourceFormat = ignoredSourceFormat; + this.isMapped = isMapped; verifyNoOtherFieldSubsetDirectoryReaderIsWrapped(in); } @Override protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException { - return new FieldSubsetDirectoryReader(in, filter, ignoredSourceFormat); + return new FieldSubsetDirectoryReader(in, filter, ignoredSourceFormat, isMapped); } /** Return the automaton that is used to filter fields. */ @@ -145,12 +154,25 @@ public CacheHelper getReaderCacheHelper() { /** * Wrap a single segment, exposing a subset of its fields. */ - FieldSubsetReader(LeafReader in, CharacterRunAutomaton filter, IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat) - throws IOException { + FieldSubsetReader( + LeafReader in, + CharacterRunAutomaton filter, + IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat, + Function isMapped + ) throws IOException { super(in); ArrayList filteredInfos = new ArrayList<>(); for (FieldInfo fi : in.getFieldInfos()) { - if (filter.run(fi.name)) { + String name = fi.name; + if (fi.getName().endsWith(TextFamilyFieldType.FALLBACK_FIELD_NAME_SUFFIX) && isMapped.apply(fi.getName()) == false) { + name = fi.getName().substring(0, fi.getName().length() - TextFamilyFieldType.FALLBACK_FIELD_NAME_SUFFIX.length()); + } + if (fi.getName().endsWith(IgnoreMalformedStoredValues.IGNORE_MALFORMED_FIELD_NAME_SUFFIX) + && isMapped.apply(fi.getName()) == false) { + name = fi.getName() + .substring(0, fi.getName().length() - IgnoreMalformedStoredValues.IGNORE_MALFORMED_FIELD_NAME_SUFFIX.length()); + } + if (filter.run(name)) { filteredInfos.add(fi); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java index abb13702afb86..db8e71295f155 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapper.java @@ -96,9 +96,11 @@ public DirectoryReader apply(final DirectoryReader reader) { } } - var indexVersionCreated = searchExecutionContextProvider.apply(shardId).indexVersionCreated(); + var searchContext = searchExecutionContextProvider.apply(shardId); + var indexVersionCreated = searchContext.indexVersionCreated(); + Function isMapped = searchContext::isFieldMapped; - return permissions.getFieldPermissions().filter(wrappedReader, indexVersionCreated); + return permissions.getFieldPermissions().filter(wrappedReader, indexVersionCreated, isMapped); } catch (IOException e) { logger.error("Unable to apply field level security"); throw ExceptionsHelper.convertToElastic(e); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java index 3369774ad5c5a..828485066fca4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/FieldPermissions.java @@ -34,6 +34,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -245,11 +246,17 @@ public boolean hasFieldLevelSecurity() { } /** Return a wrapped reader that only exposes allowed fields. */ - public DirectoryReader filter(DirectoryReader reader, IndexVersion indexVersionCreated) throws IOException { + public DirectoryReader filter(DirectoryReader reader, IndexVersion indexVersionCreated, Function isMapped) + throws IOException { if (hasFieldLevelSecurity() == false) { return reader; } - return FieldSubsetReader.wrap(reader, permittedFieldsAutomaton, IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersionCreated)); + return FieldSubsetReader.wrap( + reader, + permittedFieldsAutomaton, + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersionCreated), + isMapped + ); } Automaton getIncludeAutomaton() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java index 23a2792663ec0..01d7175e9ad52 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java @@ -115,7 +115,8 @@ public void testIndexed() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -150,7 +151,8 @@ public void testPoints() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -213,7 +215,8 @@ public void testKnnVectors() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); LeafReader leafReader = ir.leaves().get(0).reader(); @@ -264,7 +267,8 @@ public void testKnnByteVectors() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); LeafReader leafReader = ir.leaves().get(0).reader(); @@ -320,7 +324,8 @@ public void testStoredFieldsString() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -351,7 +356,8 @@ public void testStoredFieldsBinary() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -382,7 +388,8 @@ public void testStoredFieldsInt() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -413,7 +420,8 @@ public void testStoredFieldsLong() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -444,7 +452,8 @@ public void testStoredFieldsFloat() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -475,7 +484,8 @@ public void testStoredFieldsDouble() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -508,7 +518,8 @@ public void testVectors() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -541,7 +552,8 @@ public void testNorms() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -571,7 +583,8 @@ public void testNumericDocValues() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -604,7 +617,8 @@ public void testBinaryDocValues() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -637,7 +651,8 @@ public void testSortedDocValues() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -670,7 +685,8 @@ public void testSortedSetDocValues() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -704,7 +720,8 @@ public void testSortedNumericDocValues() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -738,7 +755,8 @@ public void testFieldInfos() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -773,7 +791,8 @@ public void testSourceFilteringIntegration() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); // see only one field @@ -819,7 +838,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion), + (fieldName) -> true ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -837,7 +857,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion), + (fieldName) -> true ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -852,7 +873,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion), + (fieldName) -> true ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -871,7 +893,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion), + (fieldName) -> true ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -886,7 +909,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion), + (fieldName) -> true ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -905,7 +929,8 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { DirectoryReader indexReader = FieldSubsetReader.wrap( wrapInMockESDirectoryReader(DirectoryReader.open(directory)), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion) + IgnoredSourceFieldMapper.ignoredSourceFormat(indexVersion), + (fieldName) -> true ) ) { String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); @@ -916,6 +941,82 @@ public void testIgnoredSourceFilteringIntegration() throws Exception { } } + public void testVisibilityOriginalFieldNames() throws Exception { + try (Directory dir = newDirectory()) { + try (IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(null))) { + Document doc = new Document(); + doc.add(new StoredField("a._original", new BytesRef("a"))); + doc.add(new StoredField("b._ignore_malformed", new BytesRef("b"))); + doc.add(new StoredField("c", new BytesRef("c"))); + iw.addDocument(doc); + + // Field a is mapped: + + var filter = new CharacterRunAutomaton(Automatons.patterns(List.of("a", "c"))); + try ( + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + filter, + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false + ) + ) { + + Document fields = ir.storedFields().document(0); + assertEquals(2, fields.getFields().size()); + assertEquals(new BytesRef("a"), fields.getBinaryValue("a._original")); + assertEquals(new BytesRef("c"), fields.getBinaryValue("c")); + } + // Field a is not mapped: + try ( + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + filter, + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true + ) + ) { + + Document fields = ir.storedFields().document(0); + assertEquals(1, fields.getFields().size()); + assertNull(fields.getBinaryValue("a._original")); + assertEquals(new BytesRef("c"), fields.getBinaryValue("c")); + } + // Field b is mapped: + filter = new CharacterRunAutomaton(Automatons.patterns(List.of("b", "c"))); + try ( + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + filter, + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false + ) + ) { + + Document fields = ir.storedFields().document(0); + assertEquals(2, fields.getFields().size()); + assertEquals(new BytesRef("b"), fields.getBinaryValue("b._ignore_malformed")); + assertEquals(new BytesRef("c"), fields.getBinaryValue("c")); + } + // Field b is not mapped: + try ( + DirectoryReader ir = FieldSubsetReader.wrap( + DirectoryReader.open(iw), + filter, + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true + ) + ) { + + Document fields = ir.storedFields().document(0); + assertEquals(1, fields.getFields().size()); + assertNull(fields.getBinaryValue("b._ignore_malformed")); + assertEquals(new BytesRef("c"), fields.getBinaryValue("c")); + } + } + } + } + public void testSourceFiltering() { // include on top-level value Map map = new HashMap<>(); @@ -1081,7 +1182,8 @@ public void testFieldNames() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); // see only one field @@ -1143,7 +1245,8 @@ public void testFieldNamesThreeFields() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); // see only two fields @@ -1192,7 +1295,8 @@ public void testFieldNamesMissing() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); // see only one field @@ -1230,7 +1334,8 @@ public void testFieldNamesOldIndex() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); // see only one field @@ -1262,7 +1367,8 @@ public void testCoreCacheKey() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("id")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); assertEquals(2, ir.numDocs()); assertEquals(1, ir.leaves().size()); @@ -1300,7 +1406,8 @@ public void testFilterAwayAllVectors() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldB")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); // sees no fields @@ -1323,7 +1430,8 @@ public void testEmpty() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); // see no fields @@ -1356,14 +1464,16 @@ public void testWrapTwice() throws Exception { final DirectoryReader directoryReader = FieldSubsetReader.wrap( DirectoryReader.open(dir), new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, () -> FieldSubsetReader.wrap( directoryReader, new CharacterRunAutomaton(Automata.makeString("fieldA")), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ) ); assertThat( @@ -1553,7 +1663,8 @@ public void testProducesStoredFieldsReader() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( DirectoryReader.open(iw), new CharacterRunAutomaton(automaton), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> false ); TestUtil.checkReader(ir); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java index d29200fdccc41..d1f8e5096dd23 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldDataCacheWithFieldSubsetReaderTests.java @@ -120,7 +120,8 @@ public void testSortedSetDVOrdinalsIndexFieldData_global() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( this.ir, new CharacterRunAutomaton(Automata.makeEmpty()), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); global = sortedSetOrdinalsIndexFieldData.loadGlobal(ir); atomic = global.load(ir.leaves().get(0)); @@ -137,7 +138,8 @@ public void testSortedSetDVOrdinalsIndexFieldData_segment() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( this.ir, new CharacterRunAutomaton(Automata.makeEmpty()), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); for (LeafReaderContext context : ir.leaves()) { LeafOrdinalsFieldData atomic = sortedSetOrdinalsIndexFieldData.load(context); @@ -157,7 +159,8 @@ public void testPagedBytesIndexFieldData_global() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( this.ir, new CharacterRunAutomaton(Automata.makeEmpty()), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); global = pagedBytesIndexFieldData.loadGlobal(ir); atomic = global.load(ir.leaves().get(0)); @@ -176,7 +179,8 @@ public void testPagedBytesIndexFieldData_segment() throws Exception { DirectoryReader ir = FieldSubsetReader.wrap( this.ir, new CharacterRunAutomaton(Automata.makeEmpty()), - IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE + IgnoredSourceFieldMapper.IgnoredSourceFormat.NO_IGNORED_SOURCE, + (fieldName) -> true ); for (LeafReaderContext context : ir.leaves()) { LeafOrdinalsFieldData atomic = pagedBytesIndexFieldData.load(context); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml index 301cb01acd2d3..c038c33f68f5a 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz_api_keys/30_field_level_security_synthetic_source.yml @@ -24,7 +24,134 @@ Filter single field: name: type: keyword secret: + type: match_only_text + + - do: + bulk: + index: index_fls + refresh: true + body: + - '{"create": { }}' + - '{"name": "A", "secret":"squirrel"}' + - match: { errors: false } + + - do: + security.create_api_key: + body: + name: "test-fls" + expiration: "1d" + role_descriptors: + index_access: + indices: + - names: [ "index_fls" ] + privileges: [ "read" ] + field_security: + grant: [ "name" ] + - match: { name: "test-fls" } + - is_true: id + - set: + id: api_key_id + encoded: credentials + + # With superuser... + - do: + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.secret: squirrel } + + # With FLS API Key + - do: + headers: + Authorization: "ApiKey ${credentials}" + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - is_false: "hits.hits.0._source.secret" + +--- +match_only_text field type grant all except secret field: + - do: + indices.create: + index: index_fls + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: type: keyword + secret: + type: match_only_text + + - do: + bulk: + index: index_fls + refresh: true + body: + - '{"create": { }}' + - '{"name": "A", "secret":"squirrel"}' + - match: { errors: false } + + - do: + security.create_api_key: + body: + name: "test-fls" + expiration: "1d" + role_descriptors: + index_access: + indices: + - names: [ "index_fls" ] + privileges: [ "read" ] + field_security: + grant: [ "*" ] + except: [ "secret" ] + - match: { name: "test-fls" } + - is_true: id + - set: + id: api_key_id + encoded: credentials + + # With superuser... + - do: + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.secret: squirrel } + + # With FLS API Key + - do: + headers: + Authorization: "ApiKey ${credentials}" + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - is_false: "hits.hits.0._source.secret" + +--- +match_only_text field type grant name field: + - do: + indices.create: + index: index_fls + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: + type: match_only_text + secret: + type: match_only_text - do: bulk: @@ -73,6 +200,136 @@ Filter single field: - match: { hits.hits.0._source.name: A } - is_false: "hits.hits.0._source.secret" +--- +keyword field type with ignore_above: + - do: + indices.create: + index: index_fls + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: + type: keyword + secret: + type: keyword + ignore_above: 3 + + - do: + bulk: + index: index_fls + refresh: true + body: + - '{"create": { }}' + - '{"name": "A", "secret":"squirrel"}' + - match: { errors: false } + + - do: + security.create_api_key: + body: + name: "test-fls" + expiration: "1d" + role_descriptors: + index_access: + indices: + - names: [ "index_fls" ] + privileges: [ "read" ] + field_security: + grant: [ "*" ] + except: [ "secret" ] + - match: { name: "test-fls" } + - is_true: id + - set: + id: api_key_id + encoded: credentials + + # With superuser... + - do: + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.secret: squirrel } + + # With FLS API Key + - do: + headers: + Authorization: "ApiKey ${credentials}" + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - is_false: "hits.hits.0._source.secret" + +--- +long field type with ignore_malformed: + - do: + indices.create: + index: index_fls + body: + settings: + index: + mapping.source.mode: synthetic + mappings: + properties: + name: + type: keyword + secret: + type: long + ignore_malformed: true + + - do: + bulk: + index: index_fls + refresh: true + body: + - '{"create": { }}' + - '{"name": "A", "secret":"squirrel"}' + - match: { errors: false } + + - do: + security.create_api_key: + body: + name: "test-fls" + expiration: "1d" + role_descriptors: + index_access: + indices: + - names: [ "index_fls" ] + privileges: [ "read" ] + field_security: + grant: [ "*" ] + except: [ "secret" ] + - match: { name: "test-fls" } + - is_true: id + - set: + id: api_key_id + encoded: credentials + + # With superuser... + - do: + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.secret: squirrel } + + # With FLS API Key + - do: + headers: + Authorization: "ApiKey ${credentials}" + search: + index: index_fls + - match: { hits.total.value: 1 } + - match: { hits.total.relation: "eq" } + - match: { hits.hits.0._source.name: A } + - is_false: "hits.hits.0._source.secret" + --- Filter fields in object: - do: diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index f23536acba87b..ad28b0336d855 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -70,6 +70,7 @@ import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MappingParserContext; import org.elasticsearch.index.mapper.SourceValueFetcher; +import org.elasticsearch.index.mapper.TextFamilyFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.mapper.blockloader.docvalues.BytesRefsFromCustomBinaryBlockLoader; @@ -1023,7 +1024,7 @@ private WildcardFieldMapper( this.indexVersionCreated = builder.indexCreatedVersion; this.ignoreAboveDefault = builder.ignoreAboveDefault; this.ignoreAbove = new IgnoreAbove(builder.ignoreAbove.getValue(), builder.indexMode, builder.indexCreatedVersion); - this.originalName = storeIgnored ? fullPath() + "._original" : null; + this.originalName = storeIgnored ? fullPath() + TextFamilyFieldType.FALLBACK_FIELD_NAME_SUFFIX : null; } @Override From ec0efafc2fdc49adffe4f58806ea71d28b7d9f49 Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Mon, 3 Nov 2025 16:06:00 +0200 Subject: [PATCH 18/32] Support choosing the downsampling method in data stream lifecycle (#137023) Following #136813, we expose to data stream lifecycle the new sampling method config in the downsampling API. This will allow users to configure the sampling method directly in the lifecycle configuration. For example: ``` PUT _data_stream/my-ds/_lifecycle { "data_retention": "10d", "downsampling_method": "last_value", "downsampling": [ { "after": "1d", "fixed_interval": "5m } ] } ``` --- docs/changelog/137023.yaml | 5 + .../lifecycle/DataStreamLifecycleService.java | 42 ++- .../RestPutDataStreamLifecycleAction.java | 14 +- .../MetadataIndexTemplateServiceTests.java | 38 ++- .../action/GetDataStreamsResponseTests.java | 2 +- .../DataStreamLifecycleFixtures.java | 28 +- .../DataStreamLifecycleServiceTests.java | 12 +- .../test/data_stream/lifecycle/20_basic.yml | 52 ++++ .../cluster.component_template/10_basic.yml | 32 +++ .../indices.put_index_template/10_basic.yml | 34 +++ .../PutDataStreamLifecycleAction.java | 28 +- .../action/downsample/DownsampleConfig.java | 28 +- .../cluster/metadata/DataStream.java | 6 +- .../cluster/metadata/DataStreamLifecycle.java | 245 +++++++++++++----- .../RestPutComponentTemplateAction.java | 4 +- .../RestPutComposableIndexTemplateAction.java | 5 +- .../add_sample_method_downsample_dlm.csv | 1 + .../resources/transport/upper_bounds/9.3.csv | 2 +- .../datastreams/GetDataStreamActionTests.java | 2 +- .../ExplainIndexDataStreamLifecycleTests.java | 2 +- .../GetDataStreamLifecycleActionTests.java | 2 +- .../downsample/DownsampleConfigTests.java | 2 +- .../DataStreamFailureStoreTemplateTests.java | 3 +- .../DataStreamLifecycleTemplateTests.java | 129 +++++---- .../metadata/DataStreamLifecycleTests.java | 101 ++++---- ...amLifecycleWithRetentionWarningsTests.java | 14 +- .../cluster/metadata/DataStreamTests.java | 32 +-- .../MetadataIndexTemplateServiceTests.java | 100 ++++++- .../TimeSeriesUsageTransportActionIT.java | 12 +- .../TimeSeriesUsageTransportAction.java | 4 +- ...taStreamLifecycleFeatureSetUsageTests.java | 6 +- ...StreamLifecycleDownsampleDisruptionIT.java | 10 +- .../DataStreamLifecycleDownsampleIT.java | 216 ++++++++++----- .../downsample/TransportDownsampleAction.java | 11 +- ...ampleShardPersistentTaskExecutorTests.java | 2 +- ...StreamLifecycleDownsamplingSecurityIT.java | 34 ++- 36 files changed, 871 insertions(+), 389 deletions(-) create mode 100644 docs/changelog/137023.yaml create mode 100644 server/src/main/resources/transport/definitions/referable/add_sample_method_downsample_dlm.csv diff --git a/docs/changelog/137023.yaml b/docs/changelog/137023.yaml new file mode 100644 index 0000000000000..cab7620233562 --- /dev/null +++ b/docs/changelog/137023.yaml @@ -0,0 +1,5 @@ +pr: 137023 +summary: Support choosing the downsampling method in data stream lifecycle +area: "Data streams" +type: enhancement +issues: [] diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java index 776ee03928b27..5a998f43d0103 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java @@ -524,7 +524,10 @@ Set maybeExecuteDownsampling(ProjectState projectState, DataStream dataSt // - has matching downsample rounds // - is read-only // So let's wait for an in-progress downsampling operation to succeed or trigger the last matching round - affectedIndices.addAll(waitForInProgressOrTriggerDownsampling(dataStream, backingIndexMeta, downsamplingRounds, project)); + var downsamplingMethod = dataStream.getDataLifecycle().downsamplingMethod(); + affectedIndices.addAll( + waitForInProgressOrTriggerDownsampling(dataStream, backingIndexMeta, downsamplingRounds, downsamplingMethod, project) + ); } } @@ -541,6 +544,7 @@ private Set waitForInProgressOrTriggerDownsampling( DataStream dataStream, IndexMetadata backingIndex, List downsamplingRounds, + DownsampleConfig.SamplingMethod downsamplingMethod, ProjectMetadata project ) { assert dataStream.getIndices().contains(backingIndex.getIndex()) @@ -556,7 +560,7 @@ private Set waitForInProgressOrTriggerDownsampling( String downsampleIndexName = DownsampleConfig.generateDownsampleIndexName( DOWNSAMPLED_INDEX_PREFIX, backingIndex, - round.config().getFixedInterval() + round.fixedInterval() ); IndexMetadata targetDownsampleIndexMeta = project.index(downsampleIndexName); boolean targetDownsampleIndexExists = targetDownsampleIndexMeta != null; @@ -568,7 +572,8 @@ private Set waitForInProgressOrTriggerDownsampling( INDEX_DOWNSAMPLE_STATUS.get(targetDownsampleIndexMeta.getSettings()), round, lastRound, - index, + downsamplingMethod, + backingIndex, targetDownsampleIndexMeta.getIndex() ); if (downsamplingNotComplete.isEmpty() == false) { @@ -580,7 +585,7 @@ private Set waitForInProgressOrTriggerDownsampling( // no maintenance needed for previously started downsampling actions and we are on the last matching round so it's time // to kick off downsampling affectedIndices.add(index); - downsampleIndexOnce(round, project.id(), indexName, downsampleIndexName); + downsampleIndexOnce(round, downsamplingMethod, project.id(), backingIndex, downsampleIndexName); } } } @@ -592,16 +597,30 @@ private Set waitForInProgressOrTriggerDownsampling( */ private void downsampleIndexOnce( DataStreamLifecycle.DownsamplingRound round, + DownsampleConfig.SamplingMethod requestedDownsamplingMethod, ProjectId projectId, - String sourceIndex, + IndexMetadata sourceIndexMetadata, String downsampleIndexName ) { + // When an index is already downsampled with a method, we require all later downsampling rounds to use the same method. + // This is necessary to preserve the relation of the downsampled index to the raw data. For example, if an index is already + // downsampled and downsampled it again to 1 hour; we know that a document represents either the aggregated raw data of an hour + // or the last value of the raw data within this hour. If we mix the methods, we cannot derive any meaning from them. + // Furthermore, data stream lifecycle is configured on the data stream level and not on the individual index level, meaning that + // when a user changes downsampling method, some indices would not be able to be downsampled anymore. + // For this reason, when we encounter an already downsampled index, we use the source downsampling method which might be different + // from the requested one. + var sourceIndexSamplingMethod = DownsampleConfig.SamplingMethod.fromIndexMetadata(sourceIndexMetadata); + String sourceIndex = sourceIndexMetadata.getIndex().getName(); DownsampleAction.Request request = new DownsampleAction.Request( TimeValue.THIRTY_SECONDS /* TODO should this be longer/configurable? */, sourceIndex, downsampleIndexName, null, - round.config() + new DownsampleConfig( + round.fixedInterval(), + sourceIndexSamplingMethod == null ? requestedDownsamplingMethod : sourceIndexSamplingMethod + ) ); transportActionsDeduplicator.executeOnce( Tuple.tuple(projectId, request), @@ -632,11 +651,12 @@ private Set evaluateDownsampleStatus( IndexMetadata.DownsampleTaskStatus downsampleStatus, DataStreamLifecycle.DownsamplingRound currentRound, DataStreamLifecycle.DownsamplingRound lastRound, - Index backingIndex, + DownsampleConfig.SamplingMethod downsamplingMethod, + IndexMetadata backingIndex, Index downsampleIndex ) { Set affectedIndices = new HashSet<>(); - String indexName = backingIndex.getName(); + String indexName = backingIndex.getIndex().getName(); String downsampleIndexName = downsampleIndex.getName(); return switch (downsampleStatus) { case UNKNOWN -> { @@ -683,15 +703,15 @@ private Set evaluateDownsampleStatus( // NOTE that the downsample request is made through the deduplicator so it will only really be executed if // there isn't one already in-flight. This can happen if a previous request timed-out, failed, or there was a // master failover and data stream lifecycle needed to restart - downsampleIndexOnce(currentRound, projectId, indexName, downsampleIndexName); - affectedIndices.add(backingIndex); + downsampleIndexOnce(currentRound, downsamplingMethod, projectId, backingIndex, downsampleIndexName); + affectedIndices.add(backingIndex.getIndex()); yield affectedIndices; } case SUCCESS -> { if (dataStream.getIndices().contains(downsampleIndex) == false) { // at this point the source index is part of the data stream and the downsample index is complete but not // part of the data stream. we need to replace the source index with the downsample index in the data stream - affectedIndices.add(backingIndex); + affectedIndices.add(backingIndex.getIndex()); replaceBackingIndexWithDownsampleIndexOnce(projectId, dataStream, indexName, downsampleIndexName); } yield affectedIndices; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java index 2d975845fcd27..0d1cd1be1d95e 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.PUT; import static org.elasticsearch.rest.RestUtils.getAckTimeout; @@ -29,6 +30,9 @@ @ServerlessScope(Scope.PUBLIC) public class RestPutDataStreamLifecycleAction extends BaseRestHandler { + private static final String SUPPORTS_DOWNSAMPLING_METHOD = "dlm.downsampling_method"; + private static final Set CAPABILITIES = Set.of(SUPPORTS_DOWNSAMPLING_METHOD); + @Override public String getName() { return "put_data_lifecycles_action"; @@ -44,13 +48,14 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli try (XContentParser parser = request.contentParser()) { PutDataStreamLifecycleAction.Request putLifecycleRequest = PutDataStreamLifecycleAction.Request.parseRequest( parser, - (dataRetention, enabled, downsampling) -> new PutDataStreamLifecycleAction.Request( + (dataRetention, enabled, downsamplingRounds, downsamplingMethod) -> new PutDataStreamLifecycleAction.Request( getMasterNodeTimeout(request), getAckTimeout(request), Strings.splitStringByCommaToArray(request.param("name")), dataRetention, enabled, - downsampling + downsamplingRounds, + downsamplingMethod ) ); putLifecycleRequest.indicesOptions(IndicesOptions.fromRequest(request, putLifecycleRequest.indicesOptions())); @@ -61,4 +66,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli ); } } + + @Override + public Set supportedCapabilities() { + return CAPABILITIES; + } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java index fd29a2087db86..574c30376137c 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java @@ -43,6 +43,8 @@ import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.composeDataLifecycles; import static org.elasticsearch.common.settings.Settings.builder; import static org.elasticsearch.datastreams.MetadataDataStreamRolloverServiceTests.createSettingsProvider; +import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.randomResettable; +import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.randomSamplingMethod; import static org.elasticsearch.indices.ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -146,51 +148,63 @@ public void testLifecycleComposition() { List lifecycles = List.of(); assertThat(composeDataLifecycles(lifecycles), nullValue()); } - // One lifecycle results to this lifecycle as the final + // One lifecycle results in this lifecycle as the final { + ResettableValue> downsamplingRounds = randomDownsampling(); DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.createDataLifecycleTemplate( true, randomRetention(), - randomDownsampling() + downsamplingRounds, + randomResettable(() -> randomSamplingMethod(downsamplingRounds.get())) ); List lifecycles = List.of(lifecycle); DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); // Defaults to true assertThat(result.enabled(), equalTo(true)); assertThat(result.dataRetention(), equalTo(lifecycle.dataRetention().get())); - assertThat(result.downsampling(), equalTo(lifecycle.downsampling().get())); + assertThat(result.downsamplingRounds(), equalTo(lifecycle.downsamplingRounds().get())); + assertThat(result.downsamplingMethod(), equalTo(lifecycle.downsamplingMethod().get())); } // If the last lifecycle is missing a property (apart from enabled) we keep the latest from the previous ones // Enabled is always true unless it's explicitly set to false { + List downsamplingRounds = randomRounds(); DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.createDataLifecycleTemplate( false, randomPositiveTimeValue(), - randomRounds() + downsamplingRounds, + randomSamplingMethod(downsamplingRounds) ); List lifecycles = List.of(lifecycle, DataStreamLifecycle.Template.DATA_DEFAULT); DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); assertThat(result.enabled(), equalTo(true)); assertThat(result.dataRetention(), equalTo(lifecycle.dataRetention().get())); - assertThat(result.downsampling(), equalTo(lifecycle.downsampling().get())); + assertThat(result.downsamplingRounds(), equalTo(lifecycle.downsamplingRounds().get())); + assertThat(result.downsamplingMethod(), equalTo(lifecycle.downsamplingMethod().get())); } - // If both lifecycle have all properties, then the latest one overwrites all the others + // If both lifecycles have all properties, then the latest one overwrites all the others { + DownsampleConfig.SamplingMethod downsamplingMethod1 = randomFrom(DownsampleConfig.SamplingMethod.LAST_VALUE); DataStreamLifecycle.Template lifecycle1 = DataStreamLifecycle.createDataLifecycleTemplate( false, randomPositiveTimeValue(), - randomRounds() + randomRounds(), + downsamplingMethod1 ); DataStreamLifecycle.Template lifecycle2 = DataStreamLifecycle.createDataLifecycleTemplate( true, randomPositiveTimeValue(), - randomRounds() + randomRounds(), + downsamplingMethod1 == DownsampleConfig.SamplingMethod.LAST_VALUE + ? DownsampleConfig.SamplingMethod.AGGREGATE + : DownsampleConfig.SamplingMethod.LAST_VALUE ); List lifecycles = List.of(lifecycle1, lifecycle2); DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); assertThat(result.enabled(), equalTo(lifecycle2.enabled())); assertThat(result.dataRetention(), equalTo(lifecycle2.dataRetention().get())); - assertThat(result.downsampling(), equalTo(lifecycle2.downsampling().get())); + assertThat(result.downsamplingRounds(), equalTo(lifecycle2.downsamplingRounds().get())); + assertThat(result.downsamplingMethod(), equalTo(lifecycle2.downsamplingMethod().get())); } } @@ -255,7 +269,7 @@ private static List randomRounds() { List rounds = new ArrayList<>(); var previous = new DataStreamLifecycle.DownsamplingRound( TimeValue.timeValueDays(randomIntBetween(1, 365)), - new DownsampleConfig(new DateHistogramInterval(randomIntBetween(1, 24) + "h")) + new DateHistogramInterval(randomIntBetween(1, 24) + "h") ); rounds.add(previous); for (int i = 0; i < count; i++) { @@ -268,9 +282,7 @@ private static List randomRounds() { private static DataStreamLifecycle.DownsamplingRound nextRound(DataStreamLifecycle.DownsamplingRound previous) { var after = TimeValue.timeValueDays(previous.after().days() + randomIntBetween(1, 10)); - var fixedInterval = new DownsampleConfig( - new DateHistogramInterval((previous.config().getFixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms") - ); + var fixedInterval = new DateHistogramInterval((previous.fixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms"); return new DataStreamLifecycle.DownsamplingRound(after, fixedInterval); } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java index 8f6605cefae9f..10b772249e436 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java @@ -154,7 +154,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti .setGeneration(3) .setAllowCustomRouting(true) .setIndexMode(IndexMode.STANDARD) - .setLifecycle(DataStreamLifecycle.createDataLifecycle(false, null, null)) + .setLifecycle(DataStreamLifecycle.createDataLifecycle(false, null, null, null)) .setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED) .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build()) .build(); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java index 163bff3dfd2cb..c0625e34f3153 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java @@ -38,7 +38,9 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance; import static org.elasticsearch.test.ESIntegTestCase.client; +import static org.elasticsearch.test.ESTestCase.between; import static org.elasticsearch.test.ESTestCase.frequently; +import static org.elasticsearch.test.ESTestCase.randomFrom; import static org.elasticsearch.test.ESTestCase.randomIntBetween; import static org.junit.Assert.assertTrue; @@ -143,14 +145,18 @@ static void putComposableIndexTemplate( } static DataStreamLifecycle.Template randomDataLifecycleTemplate() { + ResettableValue> downsampling = randomResettable( + DataStreamLifecycleFixtures::randomDownsamplingRounds + ); return DataStreamLifecycle.createDataLifecycleTemplate( frequently(), randomResettable(ESTestCase::randomTimeValue), - randomResettable(DataStreamLifecycleFixtures::randomDownsamplingRounds) + downsampling, + randomResettable(() -> randomSamplingMethod(downsampling.get())) ); } - private static ResettableValue randomResettable(Supplier supplier) { + public static ResettableValue randomResettable(Supplier supplier) { return switch (randomIntBetween(0, 2)) { case 0 -> ResettableValue.undefined(); case 1 -> ResettableValue.reset(); @@ -164,7 +170,7 @@ private static List randomDownsamplingRou List rounds = new ArrayList<>(); var previous = new DataStreamLifecycle.DownsamplingRound( TimeValue.timeValueDays(randomIntBetween(1, 365)), - new DownsampleConfig(new DateHistogramInterval(randomIntBetween(1, 24) + "h")) + new DateHistogramInterval(randomIntBetween(1, 24) + "h") ); rounds.add(previous); for (int i = 0; i < count; i++) { @@ -177,9 +183,19 @@ private static List randomDownsamplingRou private static DataStreamLifecycle.DownsamplingRound nextRound(DataStreamLifecycle.DownsamplingRound previous) { var after = TimeValue.timeValueDays(previous.after().days() + randomIntBetween(1, 10)); - var fixedInterval = new DownsampleConfig( - new DateHistogramInterval((previous.config().getFixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms") - ); + var fixedInterval = new DateHistogramInterval((previous.fixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms"); return new DataStreamLifecycle.DownsamplingRound(after, fixedInterval); } + + /** + * In order to produce valid data stream lifecycle configurations, the sampling method can be defined only when + * the downsampling rounds are also defined. + */ + public static DownsampleConfig.SamplingMethod randomSamplingMethod(List downsamplingRounds) { + if (downsamplingRounds == null || between(0, DownsampleConfig.SamplingMethod.values().length) == 0) { + return null; + } else { + return randomFrom(DownsampleConfig.SamplingMethod.values()); + } + } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index 0ce32cf1f94ab..35c709462d5d9 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -1228,9 +1228,7 @@ public void testDownsampling() throws Exception { .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "@timestamp"), DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( - List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m")))) - ) + .downsamplingRounds(List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")))) .dataRetention(TimeValue.MAX_VALUE) .build(), now @@ -1377,9 +1375,7 @@ public void testDownsamplingWhenTargetIndexNameClashYieldsException() throws Exc .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "@timestamp"), DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( - List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m")))) - ) + .downsamplingRounds(List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")))) .dataRetention(TimeValue.MAX_VALUE) .build(), now @@ -1662,9 +1658,7 @@ private ClusterState downsampleSetup(ProjectId projectId, String dataStreamName, settings(IndexVersion.current()).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "@timestamp"), DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( - List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m")))) - ) + .downsamplingRounds(List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")))) .dataRetention(TimeValue.timeValueMillis(1)) .build(), now diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml index 4bf6ccfbfa7ce..56c8f764a3d2d 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml @@ -171,6 +171,58 @@ teardown: - match: { data_streams.1.lifecycle.downsampling.1.after: '100d'} - match: { data_streams.1.lifecycle.downsampling.1.fixed_interval: '10h'} +--- +"Put data stream lifecycle with downsampling method": + - requires: + capabilities: + - method: PUT + path: /_data_stream/{name}/_lifecycle + capabilities: [ "dlm.downsampling_method" ] + test_runner_features: [ "capabilities" ] + reason: "Downsampling method was added to data stream lifecycle was available from 9.3" + + - do: + indices.put_data_lifecycle: + name: "*" + body: > + { + "downsampling": [ + { + "after": "10d", + "fixed_interval": "1h" + }, + { + "after": "100d", + "fixed_interval": "10h" + } + ], + "downsampling_method": "aggregate", + "data_retention": "30d", + "enabled": false + } + + - is_true: acknowledged + + - do: + indices.get_data_lifecycle: + name: "*" + - length: { data_streams: 2 } + - match: { data_streams.0.name: data-stream-with-lifecycle } + - match: { data_streams.0.lifecycle.data_retention: "30d" } + - match: { data_streams.0.lifecycle.enabled: false} + - match: { data_streams.0.lifecycle.downsampling.0.after: '10d'} + - match: { data_streams.0.lifecycle.downsampling.0.fixed_interval: '1h'} + - match: { data_streams.0.lifecycle.downsampling.1.after: '100d'} + - match: { data_streams.0.lifecycle.downsampling.1.fixed_interval: '10h'} + - match: { data_streams.1.name: simple-data-stream1 } + - match: { data_streams.1.lifecycle.data_retention: "30d" } + - match: { data_streams.1.lifecycle.enabled: false} + - match: { data_streams.1.lifecycle.downsampling_method: 'aggregate'} + - match: { data_streams.1.lifecycle.downsampling.0.after: '10d'} + - match: { data_streams.1.lifecycle.downsampling.0.fixed_interval: '1h'} + - match: { data_streams.1.lifecycle.downsampling.1.after: '100d'} + - match: { data_streams.1.lifecycle.downsampling.1.fixed_interval: '10h'} + --- "Enable lifecycle": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml index 800dec2a795a4..c230371cc4003 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml @@ -142,6 +142,38 @@ - match: {component_templates.0.component_template.template.lifecycle.enabled: true} - match: {component_templates.0.component_template.template.lifecycle.data_retention: "10d"} +--- +"Add data stream lifecycle with downsampling": + - requires: + capabilities: + - method: PUT + path: /_component_template/{name} + capabilities: [ "dlm.downsampling_method" ] + test_runner_features: [ "capabilities" ] + reason: "Downsampling method was added to data stream lifecycle was available from 9.3" + + - do: + cluster.put_component_template: + name: test-lifecycle + body: + template: + lifecycle: + data_retention: "10d" + downsampling_method: last_value + downsampling: + - {"after": "1d", "fixed_interval": "5m"} + + - do: + cluster.get_component_template: + name: test-lifecycle + + - match: {component_templates.0.name: test-lifecycle} + - match: {component_templates.0.component_template.template.lifecycle.enabled: true} + - match: {component_templates.0.component_template.template.lifecycle.data_retention: "10d"} + - match: {component_templates.0.component_template.template.lifecycle.downsampling_method: "last_value"} + - match: {component_templates.0.component_template.template.lifecycle.downsampling.0.after: "1d"} + - match: {component_templates.0.component_template.template.lifecycle.downsampling.0.fixed_interval: "5m"} + --- "Get data stream lifecycle with default rollover": - requires: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml index 81068f460b2b3..b538293d668ea 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml @@ -197,3 +197,37 @@ name: baz - match: {index_templates.0.name: baz} + +--- +"Add data stream lifecycle with downsampling": + - requires: + capabilities: + - method: PUT + path: /_index_template/{name} + capabilities: [ "dlm.downsampling_method" ] + test_runner_features: [ "capabilities" ] + reason: "Downsampling method was added to data stream lifecycle was available from 9.3" + + - do: + indices.put_index_template: + name: test-lifecycle + body: + index_patterns: downsampling-* + data_stream: {} + template: + lifecycle: + data_retention: "10d" + downsampling_method: last_value + downsampling: + - {"after": "1d", "fixed_interval": "5m"} + + - do: + indices.get_index_template: + name: test-lifecycle + + - match: {index_templates.0.name: test-lifecycle} + - match: {index_templates.0.index_template.template.lifecycle.enabled: true} + - match: {index_templates.0.index_template.template.lifecycle.data_retention: "10d"} + - match: {index_templates.0.index_template.template.lifecycle.downsampling_method: "last_value"} + - match: {index_templates.0.index_template.template.lifecycle.downsampling.0.after: "1d"} + - match: {index_templates.0.index_template.template.lifecycle.downsampling.0.fixed_interval: "5m"} diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java index 45b71fd63cb16..b293f195d1864 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; @@ -33,6 +34,7 @@ import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.DATA_RETENTION_FIELD; import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.DOWNSAMPLING_FIELD; +import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.DOWNSAMPLING_METHOD_FIELD; import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.ENABLED_FIELD; /** @@ -50,7 +52,8 @@ public interface Factory { Request create( @Nullable TimeValue dataRetention, @Nullable Boolean enabled, - @Nullable List downsampling + @Nullable List downsampling, + @Nullable DownsampleConfig.SamplingMethod downsamplingMethod ); } @@ -58,7 +61,12 @@ Request create( public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "put_data_stream_lifecycle_request", false, - (args, factory) -> factory.create((TimeValue) args[0], (Boolean) args[1], (List) args[2]) + (args, factory) -> factory.create( + (TimeValue) args[0], + (Boolean) args[1], + (List) args[2], + (DownsampleConfig.SamplingMethod) args[3] + ) ); static { @@ -75,6 +83,12 @@ Request create( DOWNSAMPLING_FIELD, ObjectParser.ValueType.OBJECT_ARRAY ); + PARSER.declareField( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> DownsampleConfig.SamplingMethod.fromString(p.text()), + DOWNSAMPLING_METHOD_FIELD, + ObjectParser.ValueType.STRING + ); } public static Request parseRequest(XContentParser parser, Factory factory) { @@ -120,7 +134,7 @@ public void writeTo(StreamOutput out) throws IOException { } public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names, @Nullable TimeValue dataRetention) { - this(masterNodeTimeout, ackTimeout, names, dataRetention, null, null); + this(masterNodeTimeout, ackTimeout, names, dataRetention, null); } public Request(TimeValue masterNodeTimeout, TimeValue ackTimeout, String[] names, DataStreamLifecycle lifecycle) { @@ -136,7 +150,7 @@ public Request( @Nullable TimeValue dataRetention, @Nullable Boolean enabled ) { - this(masterNodeTimeout, ackTimeout, names, dataRetention, enabled, null); + this(masterNodeTimeout, ackTimeout, names, dataRetention, enabled, null, null); } public Request( @@ -145,14 +159,16 @@ public Request( String[] names, @Nullable TimeValue dataRetention, @Nullable Boolean enabled, - @Nullable List downsampling + @Nullable List downsamplingRounds, + @Nullable DownsampleConfig.SamplingMethod downsamplingMethod ) { super(masterNodeTimeout, ackTimeout); this.names = names; this.lifecycle = DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(dataRetention) .enabled(enabled == null || enabled) - .downsampling(downsampling) + .downsamplingRounds(downsamplingRounds) + .downsamplingMethod(downsamplingMethod) .build(); } diff --git a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java index 444785d55e3d4..ae08245ceb7c9 100644 --- a/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java +++ b/server/src/main/java/org/elasticsearch/action/downsample/DownsampleConfig.java @@ -101,17 +101,6 @@ public class DownsampleConfig implements NamedWriteable, ToXContentObject { ); } - /** - * Create a new {@link DownsampleConfig} using the given configuration parameters. - * @param fixedInterval the fixed interval to use for computing the date histogram for the rolled up documents (required). - * @deprecated please use {@link DownsampleConfig#DownsampleConfig(DateHistogramInterval, SamplingMethod)}, this method is being kept - * until the sampling method is completely integrated with ILM and DLM. - */ - @Deprecated - public DownsampleConfig(final DateHistogramInterval fixedInterval) { - this(fixedInterval, null); - } - /** * Create a new {@link DownsampleConfig} using the given configuration parameters. * @param fixedInterval the fixed interval to use for computing the date histogram for the rolled up documents (required). @@ -144,25 +133,28 @@ public DownsampleConfig(final StreamInput in) throws IOException { * - The target interval needs to be a multiple of the source interval * throws an IllegalArgumentException to signal that the target interval is not acceptable */ - public static void validateSourceAndTargetIntervals(DownsampleConfig source, DownsampleConfig target) { - long sourceMillis = source.fixedInterval.estimateMillis(); - long targetMillis = target.fixedInterval.estimateMillis(); + public static void validateSourceAndTargetIntervals( + DateHistogramInterval sourceFxedInterval, + DateHistogramInterval targetFixedInterval + ) { + long sourceMillis = sourceFxedInterval.estimateMillis(); + long targetMillis = targetFixedInterval.estimateMillis(); if (sourceMillis >= targetMillis) { // Downsampling interval must be greater than source interval throw new IllegalArgumentException( "Downsampling interval [" - + target.fixedInterval + + targetFixedInterval + "] must be greater than the source index interval [" - + source.fixedInterval + + sourceFxedInterval + "]." ); } else if (targetMillis % sourceMillis != 0) { // Downsampling interval must be a multiple of the source interval throw new IllegalArgumentException( "Downsampling interval [" - + target.fixedInterval + + targetFixedInterval + "] must be a multiple of the source index interval [" - + source.fixedInterval + + sourceFxedInterval + "]." ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 9e7fcb1e60f16..93be5259e29f6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -1247,7 +1247,7 @@ public List getDownsamplingRoundsFor( LongSupplier nowSupplier ) { assert backingIndices.indices.contains(index) : "the provided index must be a backing index for this datastream"; - if (lifecycle == null || lifecycle.downsampling() == null) { + if (lifecycle == null || lifecycle.downsamplingRounds() == null) { return List.of(); } @@ -1260,8 +1260,8 @@ public List getDownsamplingRoundsFor( if (indexGenerationTime != null) { long nowMillis = nowSupplier.getAsLong(); long indexGenerationTimeMillis = indexGenerationTime.millis(); - List orderedRoundsForIndex = new ArrayList<>(lifecycle.downsampling().size()); - for (DownsamplingRound round : lifecycle.downsampling()) { + List orderedRoundsForIndex = new ArrayList<>(lifecycle.downsamplingRounds().size()); + for (DownsamplingRound round : lifecycle.downsamplingRounds()) { if (nowMillis >= indexGenerationTimeMillis + round.after().getMillis()) { orderedRoundsForIndex.add(round); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java index 7e60ddf0818d1..bb87ab1994ace 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java @@ -52,12 +52,13 @@ * Lifecycle supports the following configurations: * - enabled, applicable to data and failures * - data retention, applicable to data and failures - * - downsampling, applicable only to data + * - downsampling and downsampling method, applicable only to data */ public class DataStreamLifecycle implements SimpleDiffable, ToXContentObject { // Versions over the wire public static final TransportVersion ADDED_ENABLED_FLAG_VERSION = TransportVersions.V_8_10_X; + public static final TransportVersion ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM = TransportVersion.fromName("add_sample_method_downsample_dlm"); public static final String EFFECTIVE_RETENTION_REST_API_CAPABILITY = "data_stream_lifecycle_effective_retention"; public static final String DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME = "data_streams.lifecycle_only.mode"; @@ -74,6 +75,8 @@ public class DataStreamLifecycle implements SimpleDiffable, "Failure store lifecycle does not support downsampling, please remove the downsampling configuration."; private static final TransportVersion INTRODUCE_LIFECYCLE_TEMPLATE = TransportVersion.fromName("introduce_lifecycle_template"); + public static final String DOWNSAMPLING_METHOD_WITHOUT_ROUNDS_ERROR = + "Downsampling method can only be set when there is at least one downsampling round."; /** * Check if {@link #DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME} is present and set to {@code true}, indicating that @@ -94,7 +97,7 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { Setting.Property.NodeScope ); - public static final DataStreamLifecycle DEFAULT_DATA_LIFECYCLE = DataStreamLifecycle.createDataLifecycle(null, null, null); + public static final DataStreamLifecycle DEFAULT_DATA_LIFECYCLE = DataStreamLifecycle.createDataLifecycle(null, null, null, null); public static final DataStreamLifecycle DEFAULT_FAILURE_LIFECYCLE = DataStreamLifecycle.createFailuresLifecycle(null, null); public static final String DATA_STREAM_LIFECYCLE_ORIGIN = "data_stream_lifecycle"; @@ -104,13 +107,20 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { public static final ParseField EFFECTIVE_RETENTION_FIELD = new ParseField("effective_retention"); public static final ParseField RETENTION_SOURCE_FIELD = new ParseField("retention_determined_by"); public static final ParseField DOWNSAMPLING_FIELD = new ParseField("downsampling"); + public static final ParseField DOWNSAMPLING_METHOD_FIELD = new ParseField("downsampling_method"); private static final ParseField ROLLOVER_FIELD = new ParseField("rollover"); @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "lifecycle", false, - (args, lt) -> new DataStreamLifecycle(lt, (Boolean) args[0], (TimeValue) args[1], (List) args[2]) + (args, lt) -> new DataStreamLifecycle( + lt, + (Boolean) args[0], + (TimeValue) args[1], + (List) args[2], + (DownsampleConfig.SamplingMethod) args[3] + ) ); static { @@ -133,6 +143,12 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { return AbstractObjectParser.parseArray(p, null, DownsamplingRound::fromXContent); } }, DOWNSAMPLING_FIELD, ObjectParser.ValueType.OBJECT_ARRAY_OR_NULL); + PARSER.declareField( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> DownsampleConfig.SamplingMethod.fromString(p.text()), + DOWNSAMPLING_METHOD_FIELD, + ObjectParser.ValueType.STRING + ); } private static final TransportVersion INTRODUCE_FAILURES_LIFECYCLE = TransportVersion.fromName("introduce_failures_lifecycle"); @@ -142,26 +158,34 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { @Nullable private final TimeValue dataRetention; @Nullable - private final List downsampling; + private final List downsamplingRounds; + @Nullable + private final DownsampleConfig.SamplingMethod downsamplingMethod; /** - * This constructor is visible for testing, please use {@link DataStreamLifecycle#createDataLifecycle(Boolean, TimeValue, List)} or + * This constructor is visible for testing, please use + * {@link DataStreamLifecycle#createDataLifecycle(Boolean, TimeValue, List, DownsampleConfig.SamplingMethod)} or * {@link DataStreamLifecycle#createFailuresLifecycle(Boolean, TimeValue)}. */ DataStreamLifecycle( LifecycleType lifecycleType, @Nullable Boolean enabled, @Nullable TimeValue dataRetention, - @Nullable List downsampling + @Nullable List downsamplingRounds, + @Nullable DownsampleConfig.SamplingMethod downsamplingMethod ) { this.lifecycleType = lifecycleType; this.enabled = enabled == null || enabled; this.dataRetention = dataRetention; - if (lifecycleType == LifecycleType.FAILURES && downsampling != null) { + if (lifecycleType == LifecycleType.FAILURES && downsamplingRounds != null) { throw new IllegalArgumentException(DOWNSAMPLING_NOT_SUPPORTED_ERROR_MESSAGE); } - DownsamplingRound.validateRounds(downsampling); - this.downsampling = downsampling; + DownsamplingRound.validateRounds(downsamplingRounds); + this.downsamplingRounds = downsamplingRounds; + if (downsamplingMethod != null && downsamplingRounds == null) { + throw new IllegalArgumentException(DOWNSAMPLING_METHOD_WITHOUT_ROUNDS_ERROR); + } + this.downsamplingMethod = downsamplingMethod; } /** @@ -171,9 +195,10 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { public static DataStreamLifecycle createDataLifecycle( @Nullable Boolean enabled, @Nullable TimeValue dataRetention, - @Nullable List downsampling + @Nullable List downsamplingRounds, + @Nullable DownsampleConfig.SamplingMethod downsamplingMethod ) { - return new DataStreamLifecycle(LifecycleType.DATA, enabled, dataRetention, downsampling); + return new DataStreamLifecycle(LifecycleType.DATA, enabled, dataRetention, downsamplingRounds, downsamplingMethod); } /** @@ -181,7 +206,7 @@ public static DataStreamLifecycle createDataLifecycle( * means it supports only enabling and retention. */ public static DataStreamLifecycle createFailuresLifecycle(@Nullable Boolean enabled, @Nullable TimeValue dataRetention) { - return new DataStreamLifecycle(LifecycleType.FAILURES, enabled, dataRetention, null); + return new DataStreamLifecycle(LifecycleType.FAILURES, enabled, dataRetention, null, null); } /** @@ -304,11 +329,19 @@ public void addWarningHeaderIfDataRetentionNotEffective( /** * The configured downsampling rounds with the `after` and the `fixed_interval` per round. If downsampling is - * not configured then it returns null. + * not configured, then it returns null. + */ + @Nullable + public List downsamplingRounds() { + return downsamplingRounds; + } + + /** + * The configured downsampling method. If downsampling is not configured, then it returns null. */ @Nullable - public List downsampling() { - return downsampling; + public DownsampleConfig.SamplingMethod downsamplingMethod() { + return downsamplingMethod; } @Override @@ -319,13 +352,14 @@ public boolean equals(Object o) { final DataStreamLifecycle that = (DataStreamLifecycle) o; return lifecycleType == that.lifecycleType && Objects.equals(dataRetention, that.dataRetention) - && Objects.equals(downsampling, that.downsampling) + && Objects.equals(downsamplingRounds, that.downsamplingRounds) + && Objects.equals(downsamplingMethod, that.downsamplingMethod) && enabled == that.enabled; } @Override public int hashCode() { - return Objects.hash(lifecycleType, enabled, dataRetention, downsampling); + return Objects.hash(lifecycleType, enabled, dataRetention, downsamplingRounds, downsamplingMethod); } @Override @@ -340,15 +374,18 @@ public void writeTo(StreamOutput out) throws IOException { } if (out.getTransportVersion().onOrAfter(ADDED_ENABLED_FLAG_VERSION)) { if (out.getTransportVersion().supports(INTRODUCE_LIFECYCLE_TEMPLATE)) { - out.writeOptionalCollection(downsampling); + out.writeOptionalCollection(downsamplingRounds); } else { - writeLegacyOptionalValue(downsampling, out, StreamOutput::writeCollection); + writeLegacyOptionalValue(downsamplingRounds, out, StreamOutput::writeCollection); } out.writeBoolean(enabled()); } if (out.getTransportVersion().supports(INTRODUCE_FAILURES_LIFECYCLE)) { lifecycleType.writeTo(out); } + if (out.getTransportVersion().supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM)) { + out.writeOptionalWriteable(downsamplingMethod); + } } public DataStreamLifecycle(StreamInput in) throws IOException { @@ -363,16 +400,19 @@ public DataStreamLifecycle(StreamInput in) throws IOException { } if (in.getTransportVersion().onOrAfter(ADDED_ENABLED_FLAG_VERSION)) { if (in.getTransportVersion().supports(INTRODUCE_LIFECYCLE_TEMPLATE)) { - downsampling = in.readOptionalCollectionAsList(DownsamplingRound::read); + downsamplingRounds = in.readOptionalCollectionAsList(DownsamplingRound::read); } else { - downsampling = readLegacyOptionalValue(in, is -> is.readCollectionAsList(DownsamplingRound::read)); + downsamplingRounds = readLegacyOptionalValue(in, is -> is.readCollectionAsList(DownsamplingRound::read)); } enabled = in.readBoolean(); } else { - downsampling = null; + downsamplingRounds = null; enabled = true; } lifecycleType = in.getTransportVersion().supports(INTRODUCE_FAILURES_LIFECYCLE) ? LifecycleType.read(in) : LifecycleType.DATA; + downsamplingMethod = in.getTransportVersion().supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM) + ? in.readOptionalWriteable(DownsampleConfig.SamplingMethod::read) + : null; } /** @@ -425,8 +465,10 @@ public String toString() { + enabled + ", dataRetention=" + dataRetention - + ", downsampling=" - + downsampling + + ", downsamplingRounds=" + + downsamplingRounds + + ", downsamplingMethod=" + + downsamplingMethod + '}'; } @@ -465,8 +507,11 @@ public XContentBuilder toXContent( } } - if (downsampling != null) { - builder.field(DOWNSAMPLING_FIELD.getPreferredName(), downsampling); + if (downsamplingRounds != null) { + builder.field(DOWNSAMPLING_FIELD.getPreferredName(), downsamplingRounds); + } + if (downsamplingMethod != null) { + builder.field(DOWNSAMPLING_METHOD_FIELD.getPreferredName(), downsamplingMethod.toString()); } if (rolloverConfiguration != null) { builder.field(ROLLOVER_FIELD.getPreferredName()); @@ -522,9 +567,9 @@ public String displayName() { /** * A round represents the configuration for when and how elasticsearch will downsample a backing index. * @param after is a TimeValue configuring how old (based on generation age) should a backing index be before downsampling - * @param config contains the interval that the backing index is going to be downsampled. + * @param fixedInterval contains the interval that the backing index is going to be downsampled. */ - public record DownsamplingRound(TimeValue after, DownsampleConfig config) implements Writeable, ToXContentObject { + public record DownsamplingRound(TimeValue after, DateHistogramInterval fixedInterval) implements Writeable, ToXContentObject { public static final ParseField AFTER_FIELD = new ParseField("after"); public static final ParseField FIXED_INTERVAL_FIELD = new ParseField("fixed_interval"); @@ -533,7 +578,7 @@ public record DownsamplingRound(TimeValue after, DownsampleConfig config) implem private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "downsampling_round", false, - (args, unused) -> new DownsamplingRound((TimeValue) args[0], new DownsampleConfig((DateHistogramInterval) args[1])) + (args, unused) -> new DownsamplingRound((TimeValue) args[0], (DateHistogramInterval) args[1]) ); static { @@ -545,7 +590,7 @@ public record DownsamplingRound(TimeValue after, DownsampleConfig config) implem PARSER.declareField( constructorArg(), p -> new DateHistogramInterval(p.text()), - new ParseField(FIXED_INTERVAL_FIELD.getPreferredName()), + FIXED_INTERVAL_FIELD, ObjectParser.ValueType.STRING ); } @@ -576,19 +621,23 @@ static void validateRounds(List rounds) { + "." ); } - DownsampleConfig.validateSourceAndTargetIntervals(previous.config(), round.config()); + DownsampleConfig.validateSourceAndTargetIntervals(previous.fixedInterval(), round.fixedInterval()); } } } public static DownsamplingRound read(StreamInput in) throws IOException { - return new DownsamplingRound(in.readTimeValue(), new DownsampleConfig(in)); + TimeValue after = in.readTimeValue(); + DateHistogramInterval fixedInterval = in.getTransportVersion().supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM) + ? new DateHistogramInterval(in) + : new DownsampleConfig(in).getFixedInterval(); + return new DownsamplingRound(after, fixedInterval); } public DownsamplingRound { - if (config.getFixedInterval().estimateMillis() < FIVE_MINUTES_MILLIS) { + if (fixedInterval.estimateMillis() < FIVE_MINUTES_MILLIS) { throw new IllegalArgumentException( - "A downsampling round must have a fixed interval of at least five minutes but found: " + config.getFixedInterval() + "A downsampling round must have a fixed interval of at least five minutes but found: " + fixedInterval ); } } @@ -596,14 +645,18 @@ public static DownsamplingRound read(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { out.writeTimeValue(after); - out.writeWriteable(config); + if (out.getTransportVersion().supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM)) { + out.writeWriteable(fixedInterval); + } else { + out.writeWriteable(new DownsampleConfig(fixedInterval, null)); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(AFTER_FIELD.getPreferredName(), after.getStringRep()); - config.toXContentFragment(builder); + builder.field(FIXED_INTERVAL_FIELD.getPreferredName(), fixedInterval().toString()); builder.endObject(); return builder; } @@ -625,9 +678,16 @@ public String toString() { public static Template createDataLifecycleTemplate( boolean enabled, TimeValue dataRetention, - List downsampling + List downsamplingRounds, + DownsampleConfig.SamplingMethod downsamplingMethod ) { - return new Template(LifecycleType.DATA, enabled, ResettableValue.create(dataRetention), ResettableValue.create(downsampling)); + return new Template( + LifecycleType.DATA, + enabled, + ResettableValue.create(dataRetention), + ResettableValue.create(downsamplingRounds), + ResettableValue.create(downsamplingMethod) + ); } /** @@ -637,9 +697,10 @@ public static Template createDataLifecycleTemplate( public static Template createDataLifecycleTemplate( boolean enabled, ResettableValue dataRetention, - ResettableValue> downsampling + ResettableValue> downsamplingRounds, + ResettableValue downsamplingMethod ) { - return new Template(LifecycleType.DATA, enabled, dataRetention, downsampling); + return new Template(LifecycleType.DATA, enabled, dataRetention, downsamplingRounds, downsamplingMethod); } /** @@ -647,7 +708,13 @@ public static Template createDataLifecycleTemplate( * means it supports only setting the enabled and the retention. */ public static Template createFailuresLifecycleTemplate(boolean enabled, TimeValue dataRetention) { - return new Template(LifecycleType.FAILURES, enabled, ResettableValue.create(dataRetention), ResettableValue.undefined()); + return new Template( + LifecycleType.FAILURES, + enabled, + ResettableValue.create(dataRetention), + ResettableValue.undefined(), + ResettableValue.undefined() + ); } /** @@ -658,24 +725,34 @@ public record Template( LifecycleType lifecycleType, boolean enabled, ResettableValue dataRetention, - ResettableValue> downsampling + ResettableValue> downsamplingRounds, + ResettableValue downsamplingMethod ) implements ToXContentObject, Writeable { Template( LifecycleType lifecycleType, boolean enabled, TimeValue dataRetention, - List downsampling + List downsamplingRounds, + DownsampleConfig.SamplingMethod downsamplingMethod ) { - this(lifecycleType, enabled, ResettableValue.create(dataRetention), ResettableValue.create(downsampling)); + this( + lifecycleType, + enabled, + ResettableValue.create(dataRetention), + ResettableValue.create(downsamplingRounds), + ResettableValue.create(downsamplingMethod) + ); } public Template { - if (lifecycleType == LifecycleType.FAILURES && downsampling.get() != null) { + if (lifecycleType == LifecycleType.FAILURES && downsamplingRounds.get() != null) { throw new IllegalArgumentException(DOWNSAMPLING_NOT_SUPPORTED_ERROR_MESSAGE); } - if (downsampling.isDefined() && downsampling.get() != null) { - DownsamplingRound.validateRounds(downsampling.get()); + if (downsamplingRounds.isDefined() && downsamplingRounds.get() != null) { + DownsamplingRound.validateRounds(downsamplingRounds.get()); + } else if (downsamplingMethod.isDefined() && downsamplingMethod.get() != null) { + throw new IllegalArgumentException(DOWNSAMPLING_METHOD_WITHOUT_ROUNDS_ERROR); } } @@ -683,6 +760,7 @@ public record Template( LifecycleType.DATA, true, ResettableValue.undefined(), + ResettableValue.undefined(), ResettableValue.undefined() ); @@ -694,7 +772,8 @@ public record Template( lt, args[0] == null || (boolean) args[0], args[1] == null ? ResettableValue.undefined() : (ResettableValue) args[1], - args[2] == null ? ResettableValue.undefined() : (ResettableValue>) args[2] + args[2] == null ? ResettableValue.undefined() : (ResettableValue>) args[2], + args[3] == null ? ResettableValue.undefined() : (ResettableValue) args[3] ) ); @@ -713,6 +792,10 @@ public record Template( return ResettableValue.create(AbstractObjectParser.parseArray(p, null, DownsamplingRound::fromXContent)); } }, DOWNSAMPLING_FIELD, ObjectParser.ValueType.OBJECT_ARRAY_OR_NULL); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> { + String value = p.textOrNull(); + return value == null ? ResettableValue.reset() : ResettableValue.create(DownsampleConfig.SamplingMethod.fromString(value)); + }, DOWNSAMPLING_METHOD_FIELD, ObjectParser.ValueType.STRING_OR_NULL); } @Override @@ -727,15 +810,18 @@ public void writeTo(StreamOutput out) throws IOException { } if (out.getTransportVersion().onOrAfter(ADDED_ENABLED_FLAG_VERSION)) { if (out.getTransportVersion().supports(INTRODUCE_LIFECYCLE_TEMPLATE)) { - ResettableValue.write(out, downsampling, StreamOutput::writeCollection); + ResettableValue.write(out, downsamplingRounds, StreamOutput::writeCollection); } else { - writeLegacyValue(out, downsampling, StreamOutput::writeCollection); + writeLegacyValue(out, downsamplingRounds, StreamOutput::writeCollection); } out.writeBoolean(enabled); } if (out.getTransportVersion().supports(INTRODUCE_FAILURES_LIFECYCLE)) { lifecycleType.writeTo(out); } + if (out.getTransportVersion().supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM)) { + ResettableValue.write(out, downsamplingMethod, StreamOutput::writeWriteable); + } } /** @@ -775,7 +861,7 @@ static ResettableValue readLegacyValues(StreamInput in, Writeable.Reader< public static Template read(StreamInput in) throws IOException { boolean enabled = true; ResettableValue dataRetention = ResettableValue.undefined(); - ResettableValue> downsampling = ResettableValue.undefined(); + ResettableValue> downsamplingRounds = ResettableValue.undefined(); // The order of the fields is like this for bwc reasons if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { @@ -787,16 +873,20 @@ public static Template read(StreamInput in) throws IOException { } if (in.getTransportVersion().onOrAfter(ADDED_ENABLED_FLAG_VERSION)) { if (in.getTransportVersion().supports(INTRODUCE_LIFECYCLE_TEMPLATE)) { - downsampling = ResettableValue.read(in, i -> i.readCollectionAsList(DownsamplingRound::read)); + downsamplingRounds = ResettableValue.read(in, i -> i.readCollectionAsList(DownsamplingRound::read)); } else { - downsampling = readLegacyValues(in, i -> i.readCollectionAsList(DownsamplingRound::read)); + downsamplingRounds = readLegacyValues(in, i -> i.readCollectionAsList(DownsamplingRound::read)); } enabled = in.readBoolean(); } var lifecycleTarget = in.getTransportVersion().supports(INTRODUCE_FAILURES_LIFECYCLE) ? LifecycleType.read(in) : LifecycleType.DATA; - return new Template(lifecycleTarget, enabled, dataRetention, downsampling); + ResettableValue downsamplingMethod = in.getTransportVersion() + .supports(ADD_SAMPLE_METHOD_DOWNSAMPLE_DLM) + ? ResettableValue.read(in, DownsampleConfig.SamplingMethod::read) + : ResettableValue.undefined(); + return new Template(lifecycleTarget, enabled, dataRetention, downsamplingRounds, downsamplingMethod); } public static Template dataLifecycleTemplatefromXContent(XContentParser parser) throws IOException { @@ -830,7 +920,13 @@ public XContentBuilder toXContent( builder.startObject(); builder.field(ENABLED_FIELD.getPreferredName(), enabled); dataRetention.toXContent(builder, params, DATA_RETENTION_FIELD.getPreferredName(), TimeValue::getStringRep); - downsampling.toXContent(builder, params, DOWNSAMPLING_FIELD.getPreferredName()); + downsamplingRounds.toXContent(builder, params, DOWNSAMPLING_FIELD.getPreferredName()); + downsamplingMethod.toXContent( + builder, + params, + DOWNSAMPLING_METHOD_FIELD.getPreferredName(), + DownsampleConfig.SamplingMethod::toString + ); if (rolloverConfiguration != null) { builder.field(ROLLOVER_FIELD.getPreferredName()); rolloverConfiguration.evaluateAndConvertToXContent( @@ -844,7 +940,7 @@ public XContentBuilder toXContent( } public DataStreamLifecycle toDataStreamLifecycle() { - return new DataStreamLifecycle(lifecycleType, enabled, dataRetention.get(), downsampling.get()); + return new DataStreamLifecycle(lifecycleType, enabled, dataRetention.get(), downsamplingRounds.get(), downsamplingMethod.get()); } } @@ -881,7 +977,9 @@ public static class Builder { @Nullable private TimeValue dataRetention = null; @Nullable - private List downsampling = null; + private List downsamplingRounds = null; + @Nullable + private DownsampleConfig.SamplingMethod downsamplingMethod = null; private Builder(LifecycleType lifecycleType) { this.lifecycleType = lifecycleType; @@ -891,21 +989,24 @@ private Builder(DataStreamLifecycle.Template template) { lifecycleType = template.lifecycleType(); enabled = template.enabled(); dataRetention = template.dataRetention().get(); - downsampling = template.downsampling().get(); + downsamplingRounds = template.downsamplingRounds().get(); + downsamplingMethod = template.downsamplingMethod().get(); } private Builder(DataStreamLifecycle lifecycle) { lifecycleType = lifecycle.lifecycleType; enabled = lifecycle.enabled(); dataRetention = lifecycle.dataRetention(); - downsampling = lifecycle.downsampling(); + downsamplingRounds = lifecycle.downsamplingRounds(); + downsamplingMethod = lifecycle.downsamplingMethod(); } public Builder composeTemplate(DataStreamLifecycle.Template template) { assert lifecycleType == template.lifecycleType() : "Trying to compose templates with different lifecycle types"; enabled(template.enabled()); dataRetention(template.dataRetention()); - downsampling(template.downsampling()); + downsamplingRounds(template.downsamplingRounds()); + downsamplingMethod(template.downsamplingMethod()); return this; } @@ -926,24 +1027,36 @@ public Builder dataRetention(@Nullable TimeValue dataRetention) { return this; } - public Builder downsampling(ResettableValue> downsampling) { + public Builder downsamplingRounds(ResettableValue> downsampling) { if (downsampling.isDefined()) { - this.downsampling = downsampling.get(); + this.downsamplingRounds = downsampling.get(); + } + return this; + } + + public Builder downsamplingRounds(@Nullable List downsampling) { + this.downsamplingRounds = downsampling; + return this; + } + + public Builder downsamplingMethod(ResettableValue downsamplingMethod) { + if (downsamplingMethod.isDefined()) { + this.downsamplingMethod = downsamplingMethod.get(); } return this; } - public Builder downsampling(@Nullable List downsampling) { - this.downsampling = downsampling; + public Builder downsamplingMethod(@Nullable DownsampleConfig.SamplingMethod downsamplingMethod) { + this.downsamplingMethod = downsamplingMethod; return this; } public DataStreamLifecycle build() { - return new DataStreamLifecycle(lifecycleType, enabled, dataRetention, downsampling); + return new DataStreamLifecycle(lifecycleType, enabled, dataRetention, downsamplingRounds, downsamplingMethod); } public Template buildTemplate() { - return new Template(lifecycleType, enabled, dataRetention, downsampling); + return new Template(lifecycleType, enabled, dataRetention, downsamplingRounds, downsamplingMethod); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java index aff4287ecbc58..d790b5ddb70db 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java @@ -32,10 +32,12 @@ public class RestPutComponentTemplateAction extends BaseRestHandler { public static final String SUPPORTS_FAILURE_STORE_LIFECYCLE = "data_stream_options.failure_store.lifecycle"; public static final String SUPPORTS_FAILURE_STORE = "data_stream_options.failure_store"; private static final String COMPONENT_TEMPLATE_TRACKING_INFO = "component_template_tracking_info"; + static final String SUPPORTS_DOWNSAMPLING_METHOD = "dlm.downsampling_method"; private static final Set CAPABILITIES = Set.of( SUPPORTS_FAILURE_STORE, SUPPORTS_FAILURE_STORE_LIFECYCLE, - COMPONENT_TEMPLATE_TRACKING_INFO + COMPONENT_TEMPLATE_TRACKING_INFO, + SUPPORTS_DOWNSAMPLING_METHOD ); @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComposableIndexTemplateAction.java index 28f5430775004..15f713ea583d7 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComposableIndexTemplateAction.java @@ -25,6 +25,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.rest.RestRequest.Method.PUT; import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; +import static org.elasticsearch.rest.action.admin.indices.RestPutComponentTemplateAction.SUPPORTS_DOWNSAMPLING_METHOD; import static org.elasticsearch.rest.action.admin.indices.RestPutComponentTemplateAction.SUPPORTS_FAILURE_STORE; import static org.elasticsearch.rest.action.admin.indices.RestPutComponentTemplateAction.SUPPORTS_FAILURE_STORE_LIFECYCLE; @@ -32,10 +33,12 @@ public class RestPutComposableIndexTemplateAction extends BaseRestHandler { private static final String INDEX_TEMPLATE_TRACKING_INFO = "index_template_tracking_info"; + private static final Set CAPABILITIES = Set.of( SUPPORTS_FAILURE_STORE, SUPPORTS_FAILURE_STORE_LIFECYCLE, - INDEX_TEMPLATE_TRACKING_INFO + INDEX_TEMPLATE_TRACKING_INFO, + SUPPORTS_DOWNSAMPLING_METHOD ); @Override diff --git a/server/src/main/resources/transport/definitions/referable/add_sample_method_downsample_dlm.csv b/server/src/main/resources/transport/definitions/referable/add_sample_method_downsample_dlm.csv new file mode 100644 index 0000000000000..9b32a5cf8b643 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/add_sample_method_downsample_dlm.csv @@ -0,0 +1 @@ +9211000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index 88b7f13fd8f9f..a70b776735734 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -add_sample_method_downsample_ilm,9210000 +add_sample_method_downsample_dlm,9211000 diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java index c90669650a949..c9027799a57c6 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java @@ -107,7 +107,7 @@ private static GetDataStreamAction.Response.DataStreamInfo newDataStreamInfo(boo private static DataStream newDataStreamInstance(boolean isSystem, TimeValue retention) { List indices = List.of(new Index(randomAlphaOfLength(10), randomAlphaOfLength(10))); - DataStreamLifecycle lifecycle = DataStreamLifecycle.createDataLifecycle(true, retention, null); + DataStreamLifecycle lifecycle = DataStreamLifecycle.createDataLifecycle(true, retention, null, null); Settings settings = randomSettings(); CompressedXContent mappings = randomMappings(); return DataStream.builder(randomAlphaOfLength(50), indices) diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycleTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycleTests.java index 43a39f762ce69..2fb4a1c0ced4f 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycleTests.java @@ -200,7 +200,7 @@ public void testToXContent() throws Exception { TimeValue configuredRetention = TimeValue.timeValueDays(100); TimeValue globalDefaultRetention = TimeValue.timeValueDays(10); TimeValue globalMaxRetention = TimeValue.timeValueDays(50); - DataStreamLifecycle dataStreamLifecycle = DataStreamLifecycle.createDataLifecycle(true, configuredRetention, null); + DataStreamLifecycle dataStreamLifecycle = DataStreamLifecycle.createDataLifecycle(true, configuredRetention, null, null); { boolean isSystemDataStream = true; ExplainIndexDataStreamLifecycle explainIndexDataStreamLifecycle = createManagedIndexDataStreamLifecycleExplanation( diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java index 32269d0bc5113..7c72764901505 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleActionTests.java @@ -96,7 +96,7 @@ public void testDataStreamLifecycleToXContent() throws Exception { TimeValue globalDefaultRetention = TimeValue.timeValueDays(10); TimeValue globalMaxRetention = TimeValue.timeValueDays(50); DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(globalDefaultRetention, globalMaxRetention); - DataStreamLifecycle lifecycle = DataStreamLifecycle.createDataLifecycle(true, configuredRetention, null); + DataStreamLifecycle lifecycle = DataStreamLifecycle.createDataLifecycle(true, configuredRetention, null, null); { boolean isInternalDataStream = true; GetDataStreamLifecycleAction.Response.DataStreamLifecycle explainIndexDataStreamLifecycle = createDataStreamLifecycle( diff --git a/server/src/test/java/org/elasticsearch/action/downsample/DownsampleConfigTests.java b/server/src/test/java/org/elasticsearch/action/downsample/DownsampleConfigTests.java index 95611f319d6ed..3469564b28729 100644 --- a/server/src/test/java/org/elasticsearch/action/downsample/DownsampleConfigTests.java +++ b/server/src/test/java/org/elasticsearch/action/downsample/DownsampleConfigTests.java @@ -49,7 +49,7 @@ public static DownsampleConfig randomConfig() { return new DownsampleConfig(randomInterval(), randomSamplingMethod()); } - private static DownsampleConfig.SamplingMethod randomSamplingMethod() { + public static DownsampleConfig.SamplingMethod randomSamplingMethod() { if (between(0, DownsampleConfig.SamplingMethod.values().length) == 0) { return null; } else { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTemplateTests.java index 3b7834b67d7a1..950eece0793b7 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamFailureStoreTemplateTests.java @@ -146,7 +146,8 @@ private static DataStreamFailureStore.Template normalise(DataStreamFailureStore. template.lifecycleType(), template.enabled(), template.dataRetention().get(), - template.downsampling().get() + template.downsamplingRounds().get(), + template.downsamplingMethod().get() ) ) ); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java index 1d1c352788346..e17d6891b71d6 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTemplateTests.java @@ -10,6 +10,7 @@ package org.elasticsearch.cluster.metadata; import org.elasticsearch.action.downsample.DownsampleConfig; +import org.elasticsearch.action.downsample.DownsampleConfigTests; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.TimeValue; @@ -44,27 +45,38 @@ protected DataStreamLifecycle.Template mutateInstance(DataStreamLifecycle.Templa var lifecycleTarget = instance.lifecycleType(); var enabled = instance.enabled(); var retention = instance.dataRetention(); - var downsampling = instance.downsampling(); - switch (randomInt(3)) { + var downsamplingRounds = instance.downsamplingRounds(); + var downsamplingMethod = instance.downsamplingMethod(); + switch (randomInt(4)) { case 0 -> { lifecycleTarget = lifecycleTarget == DataStreamLifecycle.LifecycleType.DATA ? DataStreamLifecycle.LifecycleType.FAILURES : DataStreamLifecycle.LifecycleType.DATA; if (lifecycleTarget == DataStreamLifecycle.LifecycleType.FAILURES) { - downsampling = ResettableValue.undefined(); + downsamplingRounds = ResettableValue.undefined(); + downsamplingMethod = ResettableValue.undefined(); } } case 1 -> enabled = enabled == false; case 2 -> retention = randomValueOtherThan(retention, DataStreamLifecycleTemplateTests::randomRetention); case 3 -> { - downsampling = randomValueOtherThan(downsampling, DataStreamLifecycleTemplateTests::randomDownsampling); - if (downsampling.get() != null) { + downsamplingRounds = randomValueOtherThan(downsamplingRounds, DataStreamLifecycleTemplateTests::randomDownsamplingRounds); + if (downsamplingRounds.get() != null) { + lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; + } else { + downsamplingMethod = ResettableValue.undefined(); + } + } + case 4 -> { + downsamplingMethod = randomValueOtherThan(downsamplingMethod, DataStreamLifecycleTemplateTests::randomDownsamplingMethod); + if (downsamplingMethod.get() != null && downsamplingRounds.get() == null) { + downsamplingRounds = ResettableValue.create(DataStreamLifecycleTests.randomDownsamplingRounds()); lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; } } default -> throw new AssertionError("Illegal randomisation branch"); } - return new DataStreamLifecycle.Template(lifecycleTarget, enabled, retention, downsampling); + return new DataStreamLifecycle.Template(lifecycleTarget, enabled, retention, downsamplingRounds, downsamplingMethod); } public void testDataLifecycleXContentSerialization() throws IOException { @@ -104,16 +116,10 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2h")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(3), - new DownsampleConfig(new DateHistogramInterval("2h")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(3), new DateHistogramInterval("2h")) ) ) .buildTemplate() @@ -127,16 +133,10 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2h")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(30), - new DownsampleConfig(new DateHistogramInterval("2h")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("2h")) ) ) .buildTemplate() @@ -147,16 +147,10 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2h")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(30), - new DownsampleConfig(new DateHistogramInterval("3h")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("3h")) ) ) .buildTemplate() @@ -166,7 +160,7 @@ public void testInvalidDownsamplingConfiguration() { { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> DataStreamLifecycle.dataLifecycleBuilder().downsampling((List.of())).buildTemplate() + () -> DataStreamLifecycle.dataLifecycleBuilder().downsamplingRounds((List.of())).buildTemplate() ); assertThat(exception.getMessage(), equalTo("Downsampling configuration should have at least one round configured.")); } @@ -174,13 +168,13 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( Stream.iterate(1, i -> i * 2) .limit(12) .map( i -> new DataStreamLifecycle.DownsamplingRound( TimeValue.timeValueDays(i), - new DownsampleConfig(new DateHistogramInterval(i + "h")) + new DateHistogramInterval(i + "h") ) ) .toList() @@ -194,13 +188,8 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( - List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2m")) - ) - ) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2m"))) ) .buildTemplate() ); @@ -209,10 +198,31 @@ public void testInvalidDownsamplingConfiguration() { equalTo("A downsampling round must have a fixed interval of at least five minutes but found: 2m") ); } + + { + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingMethod(randomFrom(DownsampleConfig.SamplingMethod.values())) + .buildTemplate() + ); + assertThat( + exception.getMessage(), + equalTo("Downsampling method can only be set when there is at least one downsampling round.") + ); + } } public static DataStreamLifecycle.Template randomDataLifecycleTemplate() { - return DataStreamLifecycle.createDataLifecycleTemplate(randomBoolean(), randomRetention(), randomDownsampling()); + ResettableValue> downsamplingRounds = randomDownsamplingRounds(); + return DataStreamLifecycle.createDataLifecycleTemplate( + randomBoolean(), + randomRetention(), + downsamplingRounds, + downsamplingRounds.get() == null + ? randomBoolean() ? ResettableValue.undefined() : ResettableValue.reset() + : randomDownsamplingMethod() + ); } public void testInvalidLifecycleConfiguration() { @@ -222,13 +232,29 @@ public void testInvalidLifecycleConfiguration() { DataStreamLifecycle.LifecycleType.FAILURES, randomBoolean(), randomBoolean() ? null : DataStreamLifecycleTests.randomPositiveTimeValue(), - DataStreamLifecycleTests.randomDownsampling() + DataStreamLifecycleTests.randomDownsamplingRounds(), + null ) ); assertThat( exception.getMessage(), containsString("Failure store lifecycle does not support downsampling, please remove the downsampling configuration.") ); + + exception = expectThrows( + IllegalArgumentException.class, + () -> new DataStreamLifecycle.Template( + DataStreamLifecycle.LifecycleType.DATA, + randomBoolean(), + randomBoolean() ? null : DataStreamLifecycleTests.randomPositiveTimeValue(), + null, + randomFrom(DownsampleConfig.SamplingMethod.values()) + ) + ); + assertThat( + exception.getMessage(), + containsString("Downsampling method can only be set when there is at least one downsampling round.") + ); } /** @@ -240,6 +266,7 @@ public static DataStreamLifecycle.Template randomFailuresLifecycleTemplate() { DataStreamLifecycle.LifecycleType.FAILURES, randomBoolean(), randomRetention(), + ResettableValue.undefined(), ResettableValue.undefined() ); } @@ -253,10 +280,18 @@ private static ResettableValue randomRetention() { }; } - private static ResettableValue> randomDownsampling() { + private static ResettableValue> randomDownsamplingRounds() { + return switch (randomIntBetween(0, 1)) { + case 0 -> ResettableValue.reset(); + case 1 -> ResettableValue.create(DataStreamLifecycleTests.randomDownsamplingRounds()); + default -> throw new IllegalStateException("Unknown randomisation path"); + }; + } + + private static ResettableValue randomDownsamplingMethod() { return switch (randomIntBetween(0, 1)) { case 0 -> ResettableValue.reset(); - case 1 -> ResettableValue.create(DataStreamLifecycleTests.randomDownsampling()); + case 1 -> ResettableValue.create(DownsampleConfigTests.randomSamplingMethod()); default -> throw new IllegalStateException("Unknown randomisation path"); }; } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java index 198163645c83b..64bc13cb0dc98 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleTests.java @@ -12,7 +12,7 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverConditions; import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; import org.elasticsearch.action.admin.indices.rollover.RolloverConfigurationTests; -import org.elasticsearch.action.downsample.DownsampleConfig; +import org.elasticsearch.action.downsample.DownsampleConfigTests; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.ClusterSettings; @@ -64,14 +64,16 @@ protected DataStreamLifecycle mutateInstance(DataStreamLifecycle instance) throw : DataStreamLifecycle.LifecycleType.DATA; var enabled = instance.enabled(); var retention = instance.dataRetention(); - var downsampling = instance.downsampling(); - switch (randomInt(3)) { + var downsamplingRounds = instance.downsamplingRounds(); + var downsamplingMethod = instance.downsamplingMethod(); + switch (randomInt(4)) { case 0 -> { if (instance.targetsFailureStore()) { lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; } else { lifecycleTarget = DataStreamLifecycle.LifecycleType.FAILURES; - downsampling = null; + downsamplingRounds = null; + downsamplingMethod = null; } } case 1 -> { @@ -82,20 +84,31 @@ protected DataStreamLifecycle mutateInstance(DataStreamLifecycle instance) throw } } case 2 -> { - if (downsampling == null) { - downsampling = randomDownsampling(); + if (downsamplingRounds == null) { + downsamplingRounds = randomDownsamplingRounds(); if (lifecycleTarget == DataStreamLifecycle.LifecycleType.FAILURES) { lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; } } else { - downsampling = randomBoolean() + downsamplingRounds = randomBoolean() ? null - : randomValueOtherThan(downsampling, DataStreamLifecycleTests::randomDownsampling); + : randomValueOtherThan(downsamplingRounds, DataStreamLifecycleTests::randomDownsamplingRounds); + if (downsamplingRounds == null) { + downsamplingMethod = null; + } + } + } + case 3 -> { + // We need to enable downsampling in order to add a non-value downsampling method + downsamplingMethod = randomValueOtherThan(downsamplingMethod, DownsampleConfigTests::randomSamplingMethod); + if (downsamplingMethod != null && downsamplingRounds == null) { + downsamplingRounds = randomDownsamplingRounds(); + lifecycleTarget = DataStreamLifecycle.LifecycleType.DATA; } } default -> enabled = enabled == false; } - return new DataStreamLifecycle(lifecycleTarget, enabled, retention, downsampling); + return new DataStreamLifecycle(lifecycleTarget, enabled, retention, downsamplingRounds, downsamplingMethod); } public void testDataLifecycleXContentSerialization() throws IOException { @@ -215,16 +228,10 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2h")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(3), - new DownsampleConfig(new DateHistogramInterval("2h")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(3), new DateHistogramInterval("2h")) ) ) .build() @@ -238,16 +245,10 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2h")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(30), - new DownsampleConfig(new DateHistogramInterval("2h")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("2h")) ) ) .build() @@ -258,16 +259,10 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2h")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(30), - new DownsampleConfig(new DateHistogramInterval("3h")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2h")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("3h")) ) ) .build() @@ -277,7 +272,7 @@ public void testInvalidDownsamplingConfiguration() { { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> DataStreamLifecycle.dataLifecycleBuilder().downsampling((List.of())).build() + () -> DataStreamLifecycle.dataLifecycleBuilder().downsamplingRounds((List.of())).build() ); assertThat(exception.getMessage(), equalTo("Downsampling configuration should have at least one round configured.")); } @@ -285,13 +280,13 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( Stream.iterate(1, i -> i * 2) .limit(12) .map( i -> new DataStreamLifecycle.DownsamplingRound( TimeValue.timeValueDays(i), - new DownsampleConfig(new DateHistogramInterval(i + "h")) + new DateHistogramInterval(i + "h") ) ) .toList() @@ -305,13 +300,8 @@ public void testInvalidDownsamplingConfiguration() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( - List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(10), - new DownsampleConfig(new DateHistogramInterval("2m")) - ) - ) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(10), new DateHistogramInterval("2m"))) ) .build() ); @@ -326,7 +316,7 @@ public void testEffectiveRetention() { // No retention in the data stream lifecycle { DataStreamLifecycle noDataRetentionLifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling(randomDownsampling()) + .downsamplingRounds(randomDownsamplingRounds()) .build(); DataStreamLifecycle noFailuresRetentionLifecycle = DataStreamLifecycle.failuresLifecycleBuilder().build(); TimeValue maxRetention = TimeValue.timeValueDays(randomIntBetween(50, 100)); @@ -381,7 +371,7 @@ public void testEffectiveRetention() { TimeValue dataStreamRetention = TimeValue.timeValueDays(randomIntBetween(5, 100)); DataStreamLifecycle dataLifecycleRetention = DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(dataStreamRetention) - .downsampling(randomDownsampling()) + .downsamplingRounds(randomDownsamplingRounds()) .build(); DataStreamLifecycle failuresLifecycleRetention = DataStreamLifecycle.failuresLifecycleBuilder() .dataRetention(dataStreamRetention) @@ -510,9 +500,11 @@ public void testEffectiveRetentionParams() { } public static DataStreamLifecycle randomDataLifecycle() { + List downsamplingRounds = randomBoolean() ? null : randomDownsamplingRounds(); return DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(randomBoolean() ? null : randomTimeValue(1, 365, TimeUnit.DAYS)) - .downsampling(randomBoolean() ? null : randomDownsampling()) + .downsamplingRounds(downsamplingRounds) + .downsamplingMethod(downsamplingRounds == null ? null : DownsampleConfigTests.randomSamplingMethod()) .enabled(randomBoolean()) .build(); } @@ -528,12 +520,12 @@ public static DataStreamLifecycle randomFailuresLifecycle() { .build(); } - static List randomDownsampling() { + static List randomDownsamplingRounds() { var count = randomIntBetween(0, 9); List rounds = new ArrayList<>(); var previous = new DataStreamLifecycle.DownsamplingRound( randomTimeValue(1, 365, TimeUnit.DAYS), - new DownsampleConfig(new DateHistogramInterval(randomIntBetween(1, 24) + "h")) + new DateHistogramInterval(randomIntBetween(1, 24) + "h") ); rounds.add(previous); for (int i = 0; i < count; i++) { @@ -546,9 +538,7 @@ static List randomDownsampling() { private static DataStreamLifecycle.DownsamplingRound nextRound(DataStreamLifecycle.DownsamplingRound previous) { var after = TimeValue.timeValueDays(previous.after().days() + randomIntBetween(1, 10)); - var fixedInterval = new DownsampleConfig( - new DateHistogramInterval((previous.config().getFixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms") - ); + var fixedInterval = new DateHistogramInterval((previous.fixedInterval().estimateMillis() * randomIntBetween(2, 5)) + "ms"); return new DataStreamLifecycle.DownsamplingRound(after, fixedInterval); } @@ -561,7 +551,8 @@ public void testInvalidLifecycleConfiguration() { DataStreamLifecycle.LifecycleType.FAILURES, null, null, - DataStreamLifecycleTests.randomDownsampling() + DataStreamLifecycleTests.randomDownsamplingRounds(), + DownsampleConfigTests.randomSamplingMethod() ) ) ); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java index 3b7a28a894156..9e1a06126fa96 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java @@ -34,7 +34,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import static org.elasticsearch.cluster.metadata.DataStreamLifecycleTests.randomDownsampling; +import static org.elasticsearch.cluster.metadata.DataStreamLifecycleTests.randomDownsamplingRounds; import static org.elasticsearch.common.settings.Settings.builder; import static org.elasticsearch.indices.ShardLimitValidatorTests.createTestShardLimitService; import static org.hamcrest.Matchers.containsString; @@ -60,7 +60,9 @@ public void testNoHeaderWarning() { ThreadContext threadContext = new ThreadContext(Settings.EMPTY); HeaderWarning.setThreadContext(threadContext); - DataStreamLifecycle noRetentionLifecycle = DataStreamLifecycle.dataLifecycleBuilder().downsampling(randomDownsampling()).build(); + DataStreamLifecycle noRetentionLifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingRounds(randomDownsamplingRounds()) + .build(); noRetentionLifecycle.addWarningHeaderIfDataRetentionNotEffective(null, randomBoolean()); Map> responseHeaders = threadContext.getResponseHeaders(); assertThat(responseHeaders.isEmpty(), is(true)); @@ -68,7 +70,7 @@ public void testNoHeaderWarning() { TimeValue dataStreamRetention = TimeValue.timeValueDays(randomIntBetween(5, 100)); DataStreamLifecycle lifecycleWithRetention = DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(dataStreamRetention) - .downsampling(randomDownsampling()) + .downsamplingRounds(randomDownsamplingRounds()) .build(); DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention( TimeValue.timeValueDays(2), @@ -83,7 +85,9 @@ public void testDefaultRetentionHeaderWarning() { ThreadContext threadContext = new ThreadContext(Settings.EMPTY); HeaderWarning.setThreadContext(threadContext); - DataStreamLifecycle noRetentionLifecycle = DataStreamLifecycle.dataLifecycleBuilder().downsampling(randomDownsampling()).build(); + DataStreamLifecycle noRetentionLifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingRounds(randomDownsamplingRounds()) + .build(); DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention( randomTimeValue(2, 10, TimeUnit.DAYS), randomBoolean() ? null : TimeValue.timeValueDays(20) @@ -107,7 +111,7 @@ public void testMaxRetentionHeaderWarning() { TimeValue maxRetention = randomTimeValue(2, 100, TimeUnit.DAYS); DataStreamLifecycle lifecycle = DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(randomBoolean() ? null : TimeValue.timeValueDays(maxRetention.days() + 1)) - .downsampling(randomDownsampling()) + .downsamplingRounds(randomDownsamplingRounds()) .build(); DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(null, maxRetention); lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention, false); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index c822b066629f8..e3a2147991ac4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; import org.elasticsearch.action.admin.indices.rollover.RolloverConfigurationTests; import org.elasticsearch.action.admin.indices.rollover.RolloverInfo; -import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; @@ -1590,19 +1589,13 @@ public void testGetDownsampleRounds() { settings(IndexVersion.current()).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "@timestamp"), DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(2000), - new DownsampleConfig(new DateHistogramInterval("10m")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(3200), - new DownsampleConfig(new DateHistogramInterval("100m")) - ), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(2000), new DateHistogramInterval("10m")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(3200), new DateHistogramInterval("100m")), new DataStreamLifecycle.DownsamplingRound( TimeValue.timeValueMillis(3500), - new DownsampleConfig(new DateHistogramInterval("1000m")) + new DateHistogramInterval("1000m") ) ) @@ -1648,20 +1641,11 @@ public void testGetDownsampleRounds() { // no TSDB settings settings(IndexVersion.current()), DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(2000), - new DownsampleConfig(new DateHistogramInterval("10m")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(3200), - new DownsampleConfig(new DateHistogramInterval("100m")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(3500), - new DownsampleConfig(new DateHistogramInterval("1000m")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(2000), new DateHistogramInterval("10m")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(3200), new DateHistogramInterval("100m")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(3500), new DateHistogramInterval("1000m")) ) ) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index a94b92527bc53..4677ca1aed9b3 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -1104,26 +1104,40 @@ public void testResolveLifecycle() throws Exception { DataStreamLifecycle.Template lifecycle45d = DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(TimeValue.timeValueDays(45)) - .downsampling( - List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(30), - new DownsampleConfig(new DateHistogramInterval("3h")) - ) - ) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(30), new DateHistogramInterval("3h"))) ) .buildTemplate(); String ct45d = "ct_45d"; project = addComponentTemplate(service, project, ct45d, lifecycle45d); + DataStreamLifecycle.Template lifecycle60d = DataStreamLifecycle.dataLifecycleBuilder() + .dataRetention(TimeValue.timeValueDays(60)) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(7), new DateHistogramInterval("3h"))) + ) + .downsamplingMethod(DownsampleConfig.SamplingMethod.LAST_VALUE) + .buildTemplate(); + String ct60d = "ct_60d"; + project = addComponentTemplate(service, project, ct60d, lifecycle60d); DataStreamLifecycle.Template lifecycleNullRetention = DataStreamLifecycle.createDataLifecycleTemplate( true, ResettableValue.reset(), + ResettableValue.undefined(), ResettableValue.undefined() ); String ctNullRetention = "ct_null_retention"; project = addComponentTemplate(service, project, ctNullRetention, lifecycleNullRetention); + DataStreamLifecycle.Template lifecycleNullDownsampling = DataStreamLifecycle.createDataLifecycleTemplate( + true, + ResettableValue.undefined(), + ResettableValue.reset(), + ResettableValue.reset() + ); + String ctNullDownsampling = "ct_null_downsampling"; + project = addComponentTemplate(service, project, ctNullDownsampling, lifecycleNullDownsampling); + String ctEmptyLifecycle = "ct_empty_lifecycle"; project = addComponentTemplate(service, project, ctEmptyLifecycle, emptyLifecycle); @@ -1167,7 +1181,7 @@ public void testResolveLifecycle() throws Exception { lifecycle30d, DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(lifecycle30d.dataRetention()) - .downsampling(lifecycle45d.downsampling()) + .downsamplingRounds(lifecycle45d.downsamplingRounds()) .buildTemplate() ); @@ -1189,7 +1203,7 @@ public void testResolveLifecycle() throws Exception { project, List.of(ctEmptyLifecycle, ct45d), lifecycleNullRetention, - DataStreamLifecycle.dataLifecycleBuilder().downsampling(lifecycle45d.downsampling()).buildTemplate() + DataStreamLifecycle.dataLifecycleBuilder().downsamplingRounds(lifecycle45d.downsamplingRounds()).buildTemplate() ); // Component A: "lifecycle": {"retention": "30d"} @@ -1203,7 +1217,7 @@ public void testResolveLifecycle() throws Exception { DataStreamLifecycle.dataLifecycleBuilder().enabled(false).buildTemplate(), DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(lifecycle45d.dataRetention()) - .downsampling(lifecycle45d.downsampling()) + .downsamplingRounds(lifecycle45d.downsamplingRounds()) .enabled(false) .buildTemplate() ); @@ -1225,6 +1239,72 @@ public void testResolveLifecycle() throws Exception { // Composable Z: "lifecycle": {"retention": "45d", "downsampling": [{"after": "30d", "fixed_interval": "3h"}]} // Result: "lifecycle": {"retention": "45d", "downsampling": [{"after": "30d", "fixed_interval": "3h"}]} assertLifecycleResolution(service, project, List.of(ct30d, ctDisabledLifecycle), lifecycle45d, lifecycle45d); + + // Component A: "lifecycle": { + // "retention": "60d", + // "downsampling_method": "last_value", + // "downsampling": [{"after": "3d", "fixed_interval": "3h"}] + // } + // Composable Z: "lifecycle": {"retention": "45d", "downsampling": [{"after": "30d", "fixed_interval": "3h"}]} + // Result: "lifecycle": { + // "retention": "45d", + // "downsampling": [{"after": "30d", "fixed_interval": "3h"}], + // "downsampling_method": "last_value" + // } + assertLifecycleResolution( + service, + project, + List.of(ct60d), + lifecycle45d, + DataStreamLifecycle.dataLifecycleBuilder() + .dataRetention(lifecycle45d.dataRetention()) + .downsamplingMethod(lifecycle60d.downsamplingMethod()) + .downsamplingRounds(lifecycle45d.downsamplingRounds()) + .buildTemplate() + ); + + // Component A: "lifecycle": { + // "retention": "60d", + // "downsampling_method": "last_value", + // "downsampling": [{"after": "3d", "fixed_interval": "3h"}] + // } + // Component B: "lifecycle": {"retention": "45d", "downsampling": [{"after": "30d", "fixed_interval": "3h"}]} + // Composable Z: "lifecycle": {"downsampling": null, "downsampling_method": null} + // Result: "lifecycle": {"retention": "45d"} + assertLifecycleResolution( + service, + project, + List.of(ct60d, ct45d), + lifecycleNullDownsampling, + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(lifecycle45d.dataRetention()).buildTemplate() + ); + + // Component A: "lifecycle": { + // "retention": "60d", + // "downsampling_method": "last_value", + // "downsampling": [{"after": "3d", "fixed_interval": "3h"}] + // } + // Composable Z: "lifecycle": {"retention": "45d", "downsampling": [{"after": "30d", "fixed_interval": "3h"}]} + // Result: "lifecycle": { + // "retention": "45d", + // "downsampling": [{"after": "30d", "fixed_interval": "3h"}], + // "downsampling_method": "last_value" + // } + assertLifecycleResolution( + service, + project, + List.of(ct60d), + DataStreamLifecycle.createDataLifecycleTemplate( + true, + ResettableValue.undefined(), + ResettableValue.undefined(), + ResettableValue.reset() + ), + DataStreamLifecycle.dataLifecycleBuilder() + .dataRetention(lifecycle60d.dataRetention()) + .downsamplingRounds(lifecycle60d.downsamplingRounds()) + .buildTemplate() + ); } public void testResolveFailureStore() throws Exception { diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java index f9c326c1ac854..8330f7c92a8f9 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.core.action; -import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; @@ -164,7 +163,7 @@ public void testAction() throws Exception { timeSeriesDataStreamCount.incrementAndGet(); if (downsamplingConfiguredBy == DownsampledBy.DLM) { dlmDownsampledDataStreamCount.incrementAndGet(); - updateRounds(lifecycle.downsampling().size(), dlmRoundsCount, dlmRoundsSum, dlmRoundsMin, dlmRoundsMax); + updateRounds(lifecycle.downsamplingRounds().size(), dlmRoundsCount, dlmRoundsSum, dlmRoundsMin, dlmRoundsMax); } else if (downsamplingConfiguredBy == DownsampledBy.ILM) { ilmDownsampledDataStreamCount.incrementAndGet(); } @@ -410,7 +409,7 @@ private DataStreamLifecycle maybeCreateLifecycle(boolean isDownsampled, boolean } var builder = DataStreamLifecycle.dataLifecycleBuilder(); if (isDownsampled) { - builder.downsampling(randomDownsamplingRounds()); + builder.downsamplingRounds(randomDownsamplingRounds()); } return builder.build(); } @@ -471,12 +470,7 @@ private List randomDownsamplingRounds() { int minutes = 5; int days = 1; for (int i = 0; i < randomIntBetween(1, 10); i++) { - rounds.add( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueDays(days), - new DownsampleConfig(new DateHistogramInterval(minutes + "m")) - ) - ); + rounds.add(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueDays(days), new DateHistogramInterval(minutes + "m"))); minutes *= randomIntBetween(2, 5); days += randomIntBetween(1, 5); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java index 77245a396e5db..5e980387bc4f8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java @@ -89,9 +89,9 @@ protected void localClusterStateOperation( continue; } tsDataStreamCount++; - Integer dlmRounds = ds.getDataLifecycle() == null || ds.getDataLifecycle().downsampling() == null + Integer dlmRounds = ds.getDataLifecycle() == null || ds.getDataLifecycle().downsamplingRounds() == null ? null - : ds.getDataLifecycle().downsampling().size(); + : ds.getDataLifecycle().downsamplingRounds().size(); for (Index backingIndex : ds.getIndices()) { IndexMetadata indexMetadata = projectMetadata.index(backingIndex); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datastreams/DataStreamLifecycleFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datastreams/DataStreamLifecycleFeatureSetUsageTests.java index fa4e7ead7eaf9..1516353731b56 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datastreams/DataStreamLifecycleFeatureSetUsageTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datastreams/DataStreamLifecycleFeatureSetUsageTests.java @@ -117,7 +117,7 @@ public void testLifecycleStats() { 1L, null, false, - DataStreamLifecycle.createDataLifecycle(true, TimeValue.timeValueSeconds(50), null) + DataStreamLifecycle.createDataLifecycle(true, TimeValue.timeValueSeconds(50), null, null) ), DataStreamTestHelper.newInstance( randomAlphaOfLength(10), @@ -125,7 +125,7 @@ public void testLifecycleStats() { 1L, null, false, - DataStreamLifecycle.createDataLifecycle(true, TimeValue.timeValueMillis(150), null) + DataStreamLifecycle.createDataLifecycle(true, TimeValue.timeValueMillis(150), null, null) ), DataStreamTestHelper.newInstance( randomAlphaOfLength(10), @@ -133,7 +133,7 @@ public void testLifecycleStats() { 1L, null, false, - DataStreamLifecycle.createDataLifecycle(false, TimeValue.timeValueSeconds(5), null) + DataStreamLifecycle.createDataLifecycle(false, TimeValue.timeValueSeconds(5), null, null) ), DataStreamTestHelper.newInstance( randomAlphaOfLength(10), diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java index b07a00c13903c..8fff5bb52a75f 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleDisruptionIT.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.rollover.RolloverAction; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; -import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; @@ -52,13 +51,8 @@ public void testDataStreamLifecycleDownsampleRollingRestart() throws Exception { final String dataStreamName = "metrics-foo"; DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( - List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ) - ) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m"))) ) .buildTemplate(); setupTSDBDataStreamAndIngestDocs( diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java index 114bdeeff7e98..7cfbdef90a2bd 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DataStreamLifecycleDownsampleIT.java @@ -9,13 +9,19 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverAction; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction; import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.junit.annotations.TestLogging; import java.util.HashSet; @@ -26,6 +32,7 @@ import static org.elasticsearch.cluster.metadata.ClusterChangedEventUtils.indicesCreated; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; public class DataStreamLifecycleDownsampleIT extends DownsamplingIntegTestCase { @@ -42,17 +49,13 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { public void testDownsampling() throws Exception { String dataStreamName = "metrics-foo"; + DownsampleConfig.SamplingMethod downsamplingMethod = randomSamplingMethod(); DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingMethod(downsamplingMethod) + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueSeconds(10), - new DownsampleConfig(new DateHistogramInterval("10m")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueSeconds(10), new DateHistogramInterval("10m")) ) ) .buildTemplate(); @@ -68,34 +71,34 @@ public void testDownsampling() throws Exception { List backingIndices = getDataStreamBackingIndexNames(dataStreamName); String firstGenerationBackingIndex = backingIndices.get(0); - String oneSecondDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex; - String tenSecondsDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex; + String fiveMinuteDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex; + String tenMinuteDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex; Set witnessedDownsamplingIndices = new HashSet<>(); clusterService().addListener(event -> { - if (indicesCreated(event).contains(oneSecondDownsampleIndex) - || event.indicesDeleted().stream().anyMatch(index -> index.getName().equals(oneSecondDownsampleIndex))) { - witnessedDownsamplingIndices.add(oneSecondDownsampleIndex); + if (indicesCreated(event).contains(fiveMinuteDownsampleIndex) + || event.indicesDeleted().stream().anyMatch(index -> index.getName().equals(fiveMinuteDownsampleIndex))) { + witnessedDownsamplingIndices.add(fiveMinuteDownsampleIndex); } - if (indicesCreated(event).contains(tenSecondsDownsampleIndex)) { - witnessedDownsamplingIndices.add(tenSecondsDownsampleIndex); + if (indicesCreated(event).contains(tenMinuteDownsampleIndex)) { + witnessedDownsamplingIndices.add(tenMinuteDownsampleIndex); } }); // before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with - // testing so DSL doesn't have to wait for the end_time to lapse) + // testing, so DSL doesn't have to wait for the end_time to lapse) putTSDBIndexTemplate(dataStreamName, null, null, lifecycle); client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet(); assertBusy(() -> { // first downsampling round - assertThat(witnessedDownsamplingIndices.contains(oneSecondDownsampleIndex), is(true)); + assertThat(witnessedDownsamplingIndices.contains(fiveMinuteDownsampleIndex), is(true)); }, 30, TimeUnit.SECONDS); assertBusy(() -> { assertThat(witnessedDownsamplingIndices.size(), is(2)); - assertThat(witnessedDownsamplingIndices.contains(oneSecondDownsampleIndex), is(true)); - assertThat(witnessedDownsamplingIndices.contains(tenSecondsDownsampleIndex), is(true)); + assertThat(witnessedDownsamplingIndices.contains(fiveMinuteDownsampleIndex), is(true)); + assertThat(witnessedDownsamplingIndices.contains(tenMinuteDownsampleIndex), is(true)); }, 30, TimeUnit.SECONDS); assertBusy(() -> { @@ -105,28 +108,25 @@ public void testDownsampling() throws Exception { String writeIndex = dsBackingIndices.get(1); assertThat(writeIndex, backingIndexEqualTo(dataStreamName, 2)); // the last downsampling round must remain in the data stream - assertThat(dsBackingIndices.get(0), is(tenSecondsDownsampleIndex)); - assertThat(indexExists(oneSecondDownsampleIndex), is(false)); + assertThat(dsBackingIndices.get(0), is(tenMinuteDownsampleIndex)); + assertThat(indexExists(fiveMinuteDownsampleIndex), is(false)); }, 30, TimeUnit.SECONDS); + assertDownsamplingMethod(downsamplingMethod, tenMinuteDownsampleIndex); } @TestLogging(value = "org.elasticsearch.datastreams.lifecycle:TRACE", reason = "debugging") public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception { String dataStreamName = "metrics-bar"; + DownsampleConfig.SamplingMethod downsamplingMethod = randomSamplingMethod(); DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingMethod(downsamplingMethod) + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), // data stream lifecycle runs every 1 second, so by the time we forcemerge the backing index it would've been at // least 2 seconds since rollover. only the 10 seconds round should be executed. - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(10), - new DownsampleConfig(new DateHistogramInterval("10m")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(10), new DateHistogramInterval("10m")) ) ) .buildTemplate(); @@ -141,28 +141,28 @@ public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception List backingIndices = getDataStreamBackingIndexNames(dataStreamName); String firstGenerationBackingIndex = backingIndices.get(0); - String oneSecondDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex; - String tenSecondsDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex; + String fiveMinuteDownsampleIndex = "downsample-5m-" + firstGenerationBackingIndex; + String tenMinuteDownsampleIndex = "downsample-10m-" + firstGenerationBackingIndex; Set witnessedDownsamplingIndices = new HashSet<>(); clusterService().addListener(event -> { - if (indicesCreated(event).contains(oneSecondDownsampleIndex) - || event.indicesDeleted().stream().anyMatch(index -> index.getName().equals(oneSecondDownsampleIndex))) { - witnessedDownsamplingIndices.add(oneSecondDownsampleIndex); + if (indicesCreated(event).contains(fiveMinuteDownsampleIndex) + || event.indicesDeleted().stream().anyMatch(index -> index.getName().equals(fiveMinuteDownsampleIndex))) { + witnessedDownsamplingIndices.add(fiveMinuteDownsampleIndex); } - if (indicesCreated(event).contains(tenSecondsDownsampleIndex)) { - witnessedDownsamplingIndices.add(tenSecondsDownsampleIndex); + if (indicesCreated(event).contains(tenMinuteDownsampleIndex)) { + witnessedDownsamplingIndices.add(tenMinuteDownsampleIndex); } }); // before we rollover we update the index template to remove the start/end time boundaries (they're there just to ease with - // testing so DSL doesn't have to wait for the end_time to lapse) + // testing, so DSL doesn't have to wait for the end_time to lapse) putTSDBIndexTemplate(dataStreamName, null, null, lifecycle); client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null)).actionGet(); assertBusy(() -> { assertThat(witnessedDownsamplingIndices.size(), is(1)); // only the ten seconds downsample round should've been executed - assertThat(witnessedDownsamplingIndices.contains(tenSecondsDownsampleIndex), is(true)); + assertThat(witnessedDownsamplingIndices.contains(tenMinuteDownsampleIndex), is(true)); }, 30, TimeUnit.SECONDS); assertBusy(() -> { @@ -171,8 +171,9 @@ public void testDownsamplingOnlyExecutesTheLastMatchingRound() throws Exception assertThat(dsBackingIndices.size(), is(2)); String writeIndex = dsBackingIndices.get(1); assertThat(writeIndex, backingIndexEqualTo(dataStreamName, 2)); - assertThat(dsBackingIndices.get(0), is(tenSecondsDownsampleIndex)); + assertThat(dsBackingIndices.get(0), is(tenMinuteDownsampleIndex)); }, 30, TimeUnit.SECONDS); + assertDownsamplingMethod(downsamplingMethod, tenMinuteDownsampleIndex); } @TestLogging(value = "org.elasticsearch.datastreams.lifecycle:TRACE", reason = "debugging") @@ -181,19 +182,15 @@ public void testUpdateDownsampleRound() throws Exception { // we expect the earlier round to be ignored String dataStreamName = "metrics-baz"; + DownsampleConfig.SamplingMethod downsamplingMethod = randomSamplingMethod(); DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingMethod(downsamplingMethod) + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), // data stream lifecycle runs every 1 second, so by the time we forcemerge the backing index it would've been at - // least 2 seconds since rollover. only the 10 seconds round should be executed. - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(10), - new DownsampleConfig(new DateHistogramInterval("10m")) - ) + // least 2 seconds since rollover. Only the 10 seconds round should be executed. + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(10), new DateHistogramInterval("10m")) ) ) .buildTemplate(); @@ -243,15 +240,10 @@ public void testUpdateDownsampleRound() throws Exception { // update the lifecycle so that it only has one round, for the same `after` parameter as before, but a different interval // the different interval should yield a different downsample index name so we expect the data stream lifecycle to get the previous - // `10s` interval downsample index, downsample it to `20m` and replace it in the data stream instead of the `10s` one. + // `10m` interval downsample index, downsample it to `20m` and replace it in the data stream instead of the `10m` one. DataStreamLifecycle updatedLifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( - List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(10), - new DownsampleConfig(new DateHistogramInterval("20m")) - ) - ) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(10), new DateHistogramInterval("20m"))) ) .build(); assertAcked( @@ -277,5 +269,111 @@ public void testUpdateDownsampleRound() throws Exception { assertThat(writeIndex, backingIndexEqualTo(dataStreamName, 2)); assertThat(dsBackingIndices.get(0), is(thirtySecondsDownsampleIndex)); }, 30, TimeUnit.SECONDS); + assertDownsamplingMethod(downsamplingMethod, thirtySecondsDownsampleIndex); + } + + /** + * This test ensures that when we change the sampling method, the already downsampled indices will use the original sampling method, + * while the raw data ones will be downsampled with the most recent configuration. + * To achieve that, we set the following test: + * 1. Create a data stream that is downsampled with a sampling method. + * 2. Rollover and wait for the downsampling to occur + * 3. Double the downsample interval (so it can downsample the first index as well) and change the sampling method. + * 4. Rollover and wait for both indices to be downsampled with the new interval + * 5. Check that the indices have been downsampled with the correct method. + */ + public void testUpdateDownsampleSamplingMode() throws Exception { + String dataStreamName = "metrics-baz"; + DownsampleConfig.SamplingMethod initialSamplingMethod = randomBoolean() + ? null + : randomFrom(DownsampleConfig.SamplingMethod.values()); + DownsampleConfig.SamplingMethod updatedSamplingMethod = initialSamplingMethod == DownsampleConfig.SamplingMethod.LAST_VALUE + ? (randomBoolean() ? null : DownsampleConfig.SamplingMethod.AGGREGATE) + : DownsampleConfig.SamplingMethod.LAST_VALUE; + + DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingMethod(initialSamplingMethod) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(10), new DateHistogramInterval("5m"))) + ) + .buildTemplate(); + + // Start and end time there just to ease with testing, so DLM doesn't have to wait for the end_time to lapse + // Creating the first backing index. + setupTSDBDataStreamAndIngestDocs( + dataStreamName, + "1986-01-08T23:40:53.384Z", + "2022-01-08T23:40:53.384Z", + lifecycle, + DOC_COUNT, + "1990-09-09T18:00:00" + ); + + // before we roll over, we update the index template to have new start/end time boundaries + // Creating the second backing index. + putTSDBIndexTemplate(dataStreamName, "2022-01-08T23:40:53.384Z", "2023-01-08T23:40:53.384Z", lifecycle); + RolloverResponse rolloverResponse = safeGet(client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null))); + assertTrue(rolloverResponse.isRolledOver()); + String firstBackingIndex = rolloverResponse.getOldIndex(); + String secondBackingIndex = rolloverResponse.getNewIndex(); + indexDocuments(dataStreamName, randomIntBetween(1, 1000), "2022-01-08T23:50:00"); + + // Ensure that the first backing index has been downsampled + awaitClusterState(clusterState -> { + final var dataStream = clusterState.metadata().getProject().dataStreams().get(dataStreamName); + if (dataStream == null) { + return false; + } + return dataStream.getIndices().size() > 1 && dataStream.getIndices().getFirst().getName().startsWith("downsample-"); + }); + assertDownsamplingMethod(initialSamplingMethod, "downsample-5m-" + firstBackingIndex); + // We change the sampling method, but also we double the downsampling interval. We expect the data stream lifecycle to get the + // previous `5m` interval downsampled index, downsample it to `10m` and replace it in the data stream with the `5m` one. + DataStreamLifecycle updatedLifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .downsamplingMethod(updatedSamplingMethod) + .downsamplingRounds( + List.of(new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(5), new DateHistogramInterval("10m"))) + ) + .build(); + assertAcked( + client().execute( + PutDataStreamLifecycleAction.INSTANCE, + new PutDataStreamLifecycleAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + new String[] { dataStreamName }, + updatedLifecycle + ) + ) + ); + + // We roll over one more time, so the second backing index will be eligible for downsampling + putTSDBIndexTemplate(dataStreamName, null, null, lifecycle); + rolloverResponse = safeGet(client().execute(RolloverAction.INSTANCE, new RolloverRequest(dataStreamName, null))); + assertTrue(rolloverResponse.isRolledOver()); + String downsampledPrefix = "downsample-10m-"; + final var waitForUpdatedDownsamplingRound = ClusterServiceUtils.addMasterTemporaryStateListener(clusterState -> { + ProjectMetadata projectMetadata = clusterState.metadata().getProject(); + final var dataStream = projectMetadata.dataStreams().get(dataStreamName); + if (dataStream == null) { + return false; + } + + return dataStream.getIndices().size() > 2 + && dataStream.getIndices().stream().filter(index -> index.getName().startsWith(downsampledPrefix)).count() == 2; + }); + safeAwait(waitForUpdatedDownsamplingRound); + assertDownsamplingMethod(initialSamplingMethod, downsampledPrefix + firstBackingIndex); + assertDownsamplingMethod(updatedSamplingMethod, downsampledPrefix + secondBackingIndex); + } + + private void assertDownsamplingMethod(DownsampleConfig.SamplingMethod downsamplingMethod, String... indexNames) { + String expected = DownsampleConfig.SamplingMethod.getOrDefault(downsamplingMethod).toString(); + GetSettingsResponse response = safeGet( + client().admin().indices().getSettings(new GetSettingsRequest(TimeValue.THIRTY_SECONDS).indices(indexNames)) + ); + for (String indexName : indexNames) { + assertThat(response.getSetting(indexName, IndexMetadata.INDEX_DOWNSAMPLE_METHOD_KEY), equalTo(expected)); + } } } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index 50589bd80946d..b8e03a295e75d 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -868,15 +868,16 @@ private static void validateDownsamplingConfiguration( Map meta = timestampFieldType.meta(); if (meta.isEmpty() == false) { - String interval = meta.get(config.getIntervalType()); - DownsampleConfig.SamplingMethod sourceSamplingMethod = DownsampleConfig.SamplingMethod.fromIndexMetadata(sourceIndexMetadata); - if (interval != null) { + String sourceInterval = meta.get(config.getIntervalType()); + if (sourceInterval != null) { try { - DownsampleConfig sourceConfig = new DownsampleConfig(new DateHistogramInterval(interval), sourceSamplingMethod); - DownsampleConfig.validateSourceAndTargetIntervals(sourceConfig, config); + DownsampleConfig.validateSourceAndTargetIntervals(new DateHistogramInterval(sourceInterval), config.getFixedInterval()); } catch (IllegalArgumentException exception) { e.addValidationError("Source index is a downsampled index. " + exception.getMessage()); } + DownsampleConfig.SamplingMethod sourceSamplingMethod = DownsampleConfig.SamplingMethod.fromIndexMetadata( + sourceIndexMetadata + ); if (Objects.equals(sourceSamplingMethod, config.getSamplingMethodOrDefault()) == false) { e.addValidationError( "Source index is a downsampled index. Downsampling method [" diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutorTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutorTests.java index 8f0254cdf586b..40bcfdac5b00a 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutorTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutorTests.java @@ -157,7 +157,7 @@ public void testGetStatelessAssignment() { .build(); var params = new DownsampleShardTaskParams( - new DownsampleConfig(new DateHistogramInterval("1h")), + new DownsampleConfig(new DateHistogramInterval("1h"), randomSamplingMethod()), shardId.getIndexName(), 1, 1, diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java index 9def00c964a7d..c37cd36dacb64 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java @@ -121,16 +121,11 @@ public void testDownsamplingAuthorized() throws Exception { String dataStreamName = "metrics-foo"; DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingMethod(randomSamplingMethod()) + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueSeconds(10), - new DownsampleConfig(new DateHistogramInterval("10m")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueSeconds(10), new DateHistogramInterval("10m")) ) ) .buildTemplate(); @@ -408,19 +403,22 @@ private void bulkIndex(Client client, String dataStreamName, Supplier Indexed [{}] documents. Dropped [{}] duplicates.", docsIndexed, duplicates); } + private static DownsampleConfig.SamplingMethod randomSamplingMethod() { + if (between(0, DownsampleConfig.SamplingMethod.values().length) == 0) { + return null; + } else { + return randomFrom(DownsampleConfig.SamplingMethod.values()); + } + } + public static class SystemDataStreamWithDownsamplingConfigurationPlugin extends Plugin implements SystemIndexPlugin { public static final DataStreamLifecycle.Template LIFECYCLE = DataStreamLifecycle.dataLifecycleBuilder() - .downsampling( + .downsamplingMethod(randomSamplingMethod()) + .downsamplingRounds( List.of( - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueMillis(0), - new DownsampleConfig(new DateHistogramInterval("5m")) - ), - new DataStreamLifecycle.DownsamplingRound( - TimeValue.timeValueSeconds(10), - new DownsampleConfig(new DateHistogramInterval("10m")) - ) + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueMillis(0), new DateHistogramInterval("5m")), + new DataStreamLifecycle.DownsamplingRound(TimeValue.timeValueSeconds(10), new DateHistogramInterval("10m")) ) ) .buildTemplate(); From 60b89a88c570133555eacd745b06b71b732fd04a Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Mon, 3 Nov 2025 15:35:20 +0100 Subject: [PATCH 19/32] Increase timeout for searchable snapshots in ILM tests (#137514) As of #133954, we clone indices before performing the force-merge step in the `searchable_snapshot` action. On slow CI servers, 10 seconds for the index to go through the whole `searchable_snapshot` action isn't enough, so we bump the timeout to 20 seconds. I looked at the logs of a few test failures, and ILM was clearly still progressing when the test timed out. I didn't identify any particular step that was taking extraordinarily long; there were always just a few steps that took a bit longer. I would love to make these tests faster rather than bumping the timeout, but the `searchable_snapshot` action is simply one of the largest ILM actions and ILM itself isn't particularly fast. That being said, if a timeout of 20 seconds proves to be insufficient (i.e. test failures come back), I do think it's worth having a look at reducing the runtime of the tests somehow first before we increase the timeout further. Closes #137149 Closes #137151 Closes #137152 Closes #137153 Closes #137156 Closes #137166 Closes #137167 Closes #137192 --- muted-tests.yml | 3 -- .../xpack/ilm/TimeSeriesDataStreamsIT.java | 4 +-- .../actions/SearchableSnapshotActionIT.java | 34 +++++++++---------- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 4a97af263c9d5..f2ed2c6a4a5a7 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -492,9 +492,6 @@ tests: - class: org.elasticsearch.readiness.ReadinessClusterIT method: testReadinessDuringRestartsNormalOrder issue: https://github.com/elastic/elasticsearch/issues/136955 -- class: org.elasticsearch.xpack.ilm.TimeSeriesDataStreamsIT - method: testSearchableSnapshotAction - issue: https://github.com/elastic/elasticsearch/issues/137167 - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT method: test {yaml=indices.validate_query/20_query_string/validate_query with query_string parameters} issue: https://github.com/elastic/elasticsearch/issues/137391 diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java index 366fcc2f821df..ffd2b2b681824 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java @@ -184,8 +184,8 @@ public void testSearchableSnapshotAction() throws Exception { // Manual rollover the original index such that it's not the write index in the data stream anymore rolloverMaxOneDocCondition(client(), dataStream); - awaitIndexExists(restoredIndexName); - awaitIndexDoesNotExist(backingIndexName, TimeValue.timeValueSeconds(60)); + awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20)); + awaitIndexDoesNotExist(backingIndexName); assertBusy( () -> assertThat(explainIndex(client(), restoredIndexName).get("step"), is(PhaseCompleteStep.NAME)), 30, diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java index 735f64821045d..ce70f73d7c28f 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java @@ -111,7 +111,7 @@ public void testSearchableSnapshotAction() throws Exception { assertThat(backingIndices.size(), equalTo(2)); String backingIndexName = backingIndices.getFirst(); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; - awaitIndexExists(restoredIndexName); + awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), restoredIndexName, null, null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished @@ -233,7 +233,7 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception { Map coldActions = Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo)); Map phases = new HashMap<>(); phases.put("cold", new Phase("cold", TimeValue.ZERO, coldActions)); - phases.put("delete", new Phase("delete", TimeValue.timeValueMillis(10000), Map.of(DeleteAction.NAME, WITH_SNAPSHOT_DELETE))); + phases.put("delete", new Phase("delete", TimeValue.ZERO, Map.of(DeleteAction.NAME, WITH_SNAPSHOT_DELETE))); LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, phases); // PUT policy XContentBuilder builder = jsonBuilder(); @@ -261,7 +261,7 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception { String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; // let's wait for ILM to finish - awaitIndexDoesNotExist(backingIndexName); + awaitIndexDoesNotExist(backingIndexName, TimeValue.timeValueSeconds(20)); awaitIndexDoesNotExist(restoredIndexName); List> snapshots = getSnapshots(); @@ -347,7 +347,7 @@ public void testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped() throws String backingIndexName = backingIndices.getFirst(); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; - awaitIndexExists(restoredIndexName); + awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), restoredIndexName, "hot", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(backingIndexName); @@ -417,7 +417,7 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex String backingIndexName = backingIndices.getFirst(); String searchableSnapMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; - awaitIndexExists(searchableSnapMountedIndexName); + awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "hot", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(backingIndexName); @@ -459,7 +459,7 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex restoreSnapshot.setJsonEntity("{\"indices\": \"" + dataStream + "\", \"include_global_state\": false}"); assertOK(client().performRequest(restoreSnapshot)); - assertThat(indexExists(searchableSnapMountedIndexName), is(true)); + awaitIndexExists(searchableSnapMountedIndexName); ensureGreen(searchableSnapMountedIndexName); // the restored index is now managed by the now updated ILM policy and needs to go through the warm and cold phase @@ -523,7 +523,7 @@ public void testIdenticalSearchableSnapshotActionIsNoop() throws Exception { final String searchableSnapMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index; logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName); - awaitIndexExists(searchableSnapMountedIndexName); + awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "cold", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(index); @@ -569,7 +569,7 @@ public void testConvertingSearchableSnapshotFromFullToPartial() throws Exception + SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index; logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName); - awaitIndexExists(searchableSnapMountedIndexName); + awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "frozen", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(index); @@ -635,7 +635,7 @@ public void testResumingSearchableSnapshotFromFullToPartial() throws Exception { final String fullMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index; logger.info("--> waiting for [{}] to exist...", fullMountedIndexName); - awaitIndexExists(fullMountedIndexName); + awaitIndexExists(fullMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), fullMountedIndexName, "cold", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(index); @@ -651,7 +651,7 @@ public void testResumingSearchableSnapshotFromFullToPartial() throws Exception { String partiallyMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX + fullMountedIndexName; logger.info("--> waiting for [{}] to exist...", partiallyMountedIndexName); - awaitIndexExists(partiallyMountedIndexName); + awaitIndexExists(partiallyMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), partiallyMountedIndexName, "frozen", null, PhaseCompleteStep.NAME); // Ensure the searchable snapshot is not deleted when the index was deleted because it was not created by this @@ -727,7 +727,7 @@ public void testResumingSearchableSnapshotFromPartialToFull() throws Exception { final String fullMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index; final String partialMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX + fullMountedIndexName; logger.info("--> waiting for [{}] to exist...", partialMountedIndexName); - awaitIndexExists(partialMountedIndexName); + awaitIndexExists(partialMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), partialMountedIndexName, "frozen", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(index); @@ -743,7 +743,7 @@ public void testResumingSearchableSnapshotFromPartialToFull() throws Exception { String restoredPartiallyMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + partialMountedIndexName; logger.info("--> waiting for [{}] to exist...", restoredPartiallyMountedIndexName); - awaitIndexExists(restoredPartiallyMountedIndexName); + awaitIndexExists(restoredPartiallyMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), restoredPartiallyMountedIndexName, "cold", null, PhaseCompleteStep.NAME); // Ensure the searchable snapshot is not deleted when the index was deleted because it was not created by this @@ -848,7 +848,7 @@ public void testSearchableSnapshotsInHotPhasePinnedToHotNodes() throws Exception final String restoredIndex = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + firstGenIndex; logger.info("--> waiting for [{}] to exist...", restoredIndex); - awaitIndexExists(restoredIndex); + awaitIndexExists(restoredIndex, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), restoredIndex, "hot", PhaseCompleteStep.NAME, PhaseCompleteStep.NAME); Map hotIndexSettings = getIndexSettingsAsMap(restoredIndex); @@ -887,7 +887,7 @@ public void testSearchableSnapshotInvokesAsyncActionOnNewIndex() throws Exceptio assertThat(backingIndices.size(), equalTo(2)); String backingIndexName = backingIndices.getFirst(); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; - awaitIndexExists(restoredIndexName); + awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), restoredIndexName, null, null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished @@ -931,7 +931,7 @@ public void testSearchableSnapshotTotalShardsPerNode() throws Exception { final String searchableSnapMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX + SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index; logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName); - awaitIndexExists(searchableSnapMountedIndexName); + awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "frozen", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(index); @@ -984,7 +984,7 @@ public void testSearchableSnapshotReplicateFor() throws Exception { assertThat(backingIndices.size(), equalTo(2)); String backingIndexName = backingIndices.getFirst(); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; - awaitIndexExists(restoredIndexName); + awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20)); // check that the index is in the expected step and has the expected step_info.message assertBusy(() -> { @@ -1133,7 +1133,7 @@ private void assertForceMergedSnapshotDone(String phase, String backingIndexName ? SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX : SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX; final String restoredIndexName = prefix + backingIndexName; - awaitIndexExists(restoredIndexName); + awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20)); assertBusy(() -> assertThat(explainIndex(client(), restoredIndexName).get("step"), is(PhaseCompleteStep.NAME))); // Wait for the original index to be deleted, to ensure ILM has finished From ebca69f1cbbd8f26ca38f5140424b7873aad8582 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Mon, 3 Nov 2025 15:43:55 +0100 Subject: [PATCH 20/32] Revert "Re-enable some performance updates to ES|QL" (#136781) (#137520) This reverts commit 4d3e27f1958b915a0b512309be4966a4cb08586e. And then re-applies the revert to https://github.com/elastic/elasticsearch/pull/134738, so we only get an effective revert of https://github.com/elastic/elasticsearch/pull/134511. --- .../xpack/esql/CsvTestsDataLoader.java | 52 ++++--------------- .../xpack/esql/CsvTestsDataLoaderTests.java | 10 +--- 2 files changed, 13 insertions(+), 49 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 369a5ea61d9e0..d056050f8fe97 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -17,7 +17,6 @@ import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.logging.log4j.core.config.plugins.util.PluginManager; -import org.apache.lucene.util.IOConsumer; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; @@ -31,7 +30,6 @@ import org.elasticsearch.inference.TaskType; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; -import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentType; @@ -45,7 +43,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.Semaphore; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -58,7 +55,6 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.reader; public class CsvTestsDataLoader { - private static final int PARALLEL_THREADS = 10; private static final int BULK_DATA_SIZE = 100_000; private static final TestDataset EMPLOYEES = new TestDataset("employees", "mapping-default.json", "employees.csv").noSubfields(); private static final TestDataset EMPLOYEES_INCOMPATIBLE = new TestDataset( @@ -433,42 +429,18 @@ private static void loadDataSetIntoEs( IndexCreator indexCreator ) throws IOException { Logger logger = LogManager.getLogger(CsvTestsDataLoader.class); - List datasets = availableDatasetsForEs( - supportsIndexModeLookup, - supportsSourceFieldMapping, - inferenceEnabled, - timeSeriesOnly - ).stream().toList(); - - logger.info("Creating test indices"); - executeInParallel(datasets, dataset -> createIndex(client, dataset, indexCreator), "Failed to create indices in parallel"); + Set loadedDatasets = new HashSet<>(); logger.info("Loading test datasets"); - executeInParallel(datasets, dataset -> loadData(client, dataset, logger), "Failed to load data in parallel"); - - forceMerge(client, datasets.stream().map(d -> d.indexName).collect(Collectors.toSet()), logger); - + for (var dataset : availableDatasetsForEs(supportsIndexModeLookup, supportsSourceFieldMapping, inferenceEnabled, timeSeriesOnly)) { + load(client, dataset, logger, indexCreator); + loadedDatasets.add(dataset.indexName); + } + forceMerge(client, loadedDatasets, logger); logger.info("Loading enrich policies"); - executeInParallel( - ENRICH_POLICIES, - policy -> loadEnrichPolicy(client, policy.policyName, policy.policyFileName, logger), - "Failed to load enrich policies in parallel" - ); - - } - - private static void executeInParallel(List items, IOConsumer consumer, String errorMessage) { - Semaphore semaphore = new Semaphore(PARALLEL_THREADS); - ESTestCase.runInParallel(items.size(), i -> { - try { - semaphore.acquire(); - consumer.accept(items.get(i)); - } catch (IOException | InterruptedException e) { - throw new RuntimeException(errorMessage, e); - } finally { - semaphore.release(); - } - }); + for (var policy : ENRICH_POLICIES) { + loadEnrichPolicy(client, policy.policyName, policy.policyFileName, logger); + } } public static void createInferenceEndpoints(RestClient client) throws IOException { @@ -626,14 +598,12 @@ private static URL getResource(String name) { return result; } - private static void createIndex(RestClient client, TestDataset dataset, IndexCreator indexCreator) throws IOException { + private static void load(RestClient client, TestDataset dataset, Logger logger, IndexCreator indexCreator) throws IOException { + logger.info("Loading dataset [{}] into ES index [{}]", dataset.dataFileName, dataset.indexName); URL mapping = getResource("/" + dataset.mappingFileName); Settings indexSettings = dataset.readSettingsFile(); indexCreator.createIndex(client, dataset.indexName, readMappingFile(mapping, dataset.typeMapping), indexSettings); - } - private static void loadData(RestClient client, TestDataset dataset, Logger logger) throws IOException { - logger.info("Loading dataset [{}] into ES index [{}]", dataset.dataFileName, dataset.indexName); // Some examples only test that the query and mappings are valid, and don't need example data. Use .noData() for those if (dataset.dataFileName != null) { URL data = getResource("/data/" + dataset.dataFileName); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/test/java/org/elasticsearch/xpack/esql/CsvTestsDataLoaderTests.java b/x-pack/plugin/esql/qa/testFixtures/src/test/java/org/elasticsearch/xpack/esql/CsvTestsDataLoaderTests.java index 612e007c4b13f..5b40e1d03e92f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/test/java/org/elasticsearch/xpack/esql/CsvTestsDataLoaderTests.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/test/java/org/elasticsearch/xpack/esql/CsvTestsDataLoaderTests.java @@ -11,18 +11,12 @@ import java.net.ConnectException; -import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.startsWith; public class CsvTestsDataLoaderTests extends ESTestCase { public void testCsvTestsDataLoaderExecution() { - Throwable cause = expectThrows(AssertionError.class, () -> CsvTestsDataLoader.main(new String[] {})); - // find the root cause - while (cause.getCause() != null) { - cause = cause.getCause(); - } - assertThat(cause, instanceOf(ConnectException.class)); - assertThat(cause.getMessage(), startsWith("Connection refused")); + ConnectException ce = expectThrows(ConnectException.class, () -> CsvTestsDataLoader.main(new String[] {})); + assertThat(ce.getMessage(), startsWith("Connection refused")); } } From 846593cf411eca339bce09a77e5932536beed52c Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Mon, 3 Nov 2025 16:01:28 +0100 Subject: [PATCH 21/32] DiskBBQ - Panama support for 4 bits symmetric quantization (#137510) --- .../MSBitToInt4ESNextOSQVectorsScorer.java | 18 - ...MSInt4SymmetricESNextOSQVectorsScorer.java | 419 ++++++++++++++++++ .../MemorySegmentESNextOSQVectorsScorer.java | 37 +- .../PanamaESVectorizationProvider.java | 2 +- 4 files changed, 442 insertions(+), 34 deletions(-) create mode 100644 libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MSInt4SymmetricESNextOSQVectorsScorer.java diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MSBitToInt4ESNextOSQVectorsScorer.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MSBitToInt4ESNextOSQVectorsScorer.java index 14b0cc3499ae5..888388ce2caa6 100644 --- a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MSBitToInt4ESNextOSQVectorsScorer.java +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MSBitToInt4ESNextOSQVectorsScorer.java @@ -14,7 +14,6 @@ import jdk.incubator.vector.LongVector; import jdk.incubator.vector.ShortVector; import jdk.incubator.vector.VectorOperators; -import jdk.incubator.vector.VectorSpecies; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.IndexInput; @@ -31,23 +30,6 @@ /** Panamized scorer for quantized vectors stored as a {@link MemorySegment}. */ final class MSBitToInt4ESNextOSQVectorsScorer extends MemorySegmentESNextOSQVectorsScorer.MemorySegmentScorer { - private static final int BULK_SIZE = MemorySegmentESNextOSQVectorsScorer.BULK_SIZE; - private static final float FOUR_BIT_SCALE = 1f / ((1 << 4) - 1); - - private static final VectorSpecies INT_SPECIES_128 = IntVector.SPECIES_128; - - private static final VectorSpecies LONG_SPECIES_128 = LongVector.SPECIES_128; - private static final VectorSpecies LONG_SPECIES_256 = LongVector.SPECIES_256; - - private static final VectorSpecies BYTE_SPECIES_128 = ByteVector.SPECIES_128; - private static final VectorSpecies BYTE_SPECIES_256 = ByteVector.SPECIES_256; - - private static final VectorSpecies SHORT_SPECIES_128 = ShortVector.SPECIES_128; - private static final VectorSpecies SHORT_SPECIES_256 = ShortVector.SPECIES_256; - - private static final VectorSpecies FLOAT_SPECIES_128 = FloatVector.SPECIES_128; - private static final VectorSpecies FLOAT_SPECIES_256 = FloatVector.SPECIES_256; - MSBitToInt4ESNextOSQVectorsScorer(IndexInput in, int dimensions, int dataLength, MemorySegment memorySegment) { super(in, dimensions, dataLength, memorySegment); } diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MSInt4SymmetricESNextOSQVectorsScorer.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MSInt4SymmetricESNextOSQVectorsScorer.java new file mode 100644 index 0000000000000..9263c4ce56282 --- /dev/null +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MSInt4SymmetricESNextOSQVectorsScorer.java @@ -0,0 +1,419 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +package org.elasticsearch.simdvec.internal.vectorization; + +import jdk.incubator.vector.ByteVector; +import jdk.incubator.vector.FloatVector; +import jdk.incubator.vector.IntVector; +import jdk.incubator.vector.LongVector; +import jdk.incubator.vector.ShortVector; +import jdk.incubator.vector.VectorOperators; + +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.BitUtil; +import org.apache.lucene.util.VectorUtil; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.nio.ByteOrder; + +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; +import static org.apache.lucene.index.VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT; + +/** Panamized scorer for quantized vectors stored as a {@link MemorySegment}. */ +final class MSInt4SymmetricESNextOSQVectorsScorer extends MemorySegmentESNextOSQVectorsScorer.MemorySegmentScorer { + + MSInt4SymmetricESNextOSQVectorsScorer(IndexInput in, int dimensions, int dataLength, MemorySegment memorySegment) { + super(in, dimensions, dataLength, memorySegment); + } + + @Override + public long quantizeScore(byte[] q) throws IOException { + assert q.length == length; + // 128 / 8 == 16 + if (length >= 16 && PanamaESVectorUtilSupport.HAS_FAST_INTEGER_VECTORS) { + if (PanamaESVectorUtilSupport.VECTOR_BITSIZE >= 256) { + return quantizeScoreSymmetric256(q); + } else if (PanamaESVectorUtilSupport.VECTOR_BITSIZE == 128) { + return quantizeScoreSymmetric128(q); + } + } + return Long.MIN_VALUE; + } + + private long quantizeScoreSymmetric128(byte[] q) throws IOException { + int stripe0 = (int) quantizeScore128(q); + int stripe1 = (int) quantizeScore128(q); + int stripe2 = (int) quantizeScore128(q); + int stripe3 = (int) quantizeScore128(q); + return stripe0 + ((long) stripe1 << 1) + ((long) stripe2 << 2) + ((long) stripe3 << 3); + } + + private long quantizeScoreSymmetric256(byte[] q) throws IOException { + int stripe0 = (int) quantizeScore256(q); + int stripe1 = (int) quantizeScore256(q); + int stripe2 = (int) quantizeScore256(q); + int stripe3 = (int) quantizeScore256(q); + return stripe0 + ((long) stripe1 << 1) + ((long) stripe2 << 2) + ((long) stripe3 << 3); + } + + private long quantizeScore256(byte[] q) throws IOException { + long subRet0 = 0; + long subRet1 = 0; + long subRet2 = 0; + long subRet3 = 0; + int i = 0; + long offset = in.getFilePointer(); + int size = length / 4; + if (size >= ByteVector.SPECIES_256.vectorByteSize() * 2) { + int limit = ByteVector.SPECIES_256.loopBound(size); + var sum0 = LongVector.zero(LONG_SPECIES_256); + var sum1 = LongVector.zero(LONG_SPECIES_256); + var sum2 = LongVector.zero(LONG_SPECIES_256); + var sum3 = LongVector.zero(LONG_SPECIES_256); + for (; i < limit; i += ByteVector.SPECIES_256.length(), offset += LONG_SPECIES_256.vectorByteSize()) { + var vq0 = ByteVector.fromArray(BYTE_SPECIES_256, q, i).reinterpretAsLongs(); + var vq1 = ByteVector.fromArray(BYTE_SPECIES_256, q, i + size).reinterpretAsLongs(); + var vq2 = ByteVector.fromArray(BYTE_SPECIES_256, q, i + size * 2).reinterpretAsLongs(); + var vq3 = ByteVector.fromArray(BYTE_SPECIES_256, q, i + size * 3).reinterpretAsLongs(); + var vd = LongVector.fromMemorySegment(LONG_SPECIES_256, memorySegment, offset, ByteOrder.LITTLE_ENDIAN); + sum0 = sum0.add(vq0.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum1 = sum1.add(vq1.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum2 = sum2.add(vq2.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum3 = sum3.add(vq3.and(vd).lanewise(VectorOperators.BIT_COUNT)); + } + subRet0 += sum0.reduceLanes(VectorOperators.ADD); + subRet1 += sum1.reduceLanes(VectorOperators.ADD); + subRet2 += sum2.reduceLanes(VectorOperators.ADD); + subRet3 += sum3.reduceLanes(VectorOperators.ADD); + } + + if (size - i >= ByteVector.SPECIES_128.vectorByteSize()) { + var sum0 = LongVector.zero(LONG_SPECIES_128); + var sum1 = LongVector.zero(LONG_SPECIES_128); + var sum2 = LongVector.zero(LONG_SPECIES_128); + var sum3 = LongVector.zero(LONG_SPECIES_128); + int limit = ByteVector.SPECIES_128.loopBound(size); + for (; i < limit; i += ByteVector.SPECIES_128.length(), offset += LONG_SPECIES_128.vectorByteSize()) { + var vq0 = ByteVector.fromArray(BYTE_SPECIES_128, q, i).reinterpretAsLongs(); + var vq1 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + size).reinterpretAsLongs(); + var vq2 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + size * 2).reinterpretAsLongs(); + var vq3 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + size * 3).reinterpretAsLongs(); + var vd = LongVector.fromMemorySegment(LONG_SPECIES_128, memorySegment, offset, ByteOrder.LITTLE_ENDIAN); + sum0 = sum0.add(vq0.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum1 = sum1.add(vq1.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum2 = sum2.add(vq2.and(vd).lanewise(VectorOperators.BIT_COUNT)); + sum3 = sum3.add(vq3.and(vd).lanewise(VectorOperators.BIT_COUNT)); + } + subRet0 += sum0.reduceLanes(VectorOperators.ADD); + subRet1 += sum1.reduceLanes(VectorOperators.ADD); + subRet2 += sum2.reduceLanes(VectorOperators.ADD); + subRet3 += sum3.reduceLanes(VectorOperators.ADD); + } + // process scalar tail + in.seek(offset); + for (final int upperBound = size & -Long.BYTES; i < upperBound; i += Long.BYTES) { + final long value = in.readLong(); + subRet0 += Long.bitCount((long) BitUtil.VH_LE_LONG.get(q, i) & value); + subRet1 += Long.bitCount((long) BitUtil.VH_LE_LONG.get(q, i + size) & value); + subRet2 += Long.bitCount((long) BitUtil.VH_LE_LONG.get(q, i + 2 * size) & value); + subRet3 += Long.bitCount((long) BitUtil.VH_LE_LONG.get(q, i + 3 * size) & value); + } + for (final int upperBound = size & -Integer.BYTES; i < upperBound; i += Integer.BYTES) { + final int value = in.readInt(); + subRet0 += Integer.bitCount((int) BitUtil.VH_LE_INT.get(q, i) & value); + subRet1 += Integer.bitCount((int) BitUtil.VH_LE_INT.get(q, i + size) & value); + subRet2 += Integer.bitCount((int) BitUtil.VH_LE_INT.get(q, i + 2 * size) & value); + subRet3 += Integer.bitCount((int) BitUtil.VH_LE_INT.get(q, i + 3 * size) & value); + } + for (; i < size; i++) { + int dValue = in.readByte() & 0xFF; + subRet0 += Integer.bitCount((q[i] & dValue) & 0xFF); + subRet1 += Integer.bitCount((q[i + size] & dValue) & 0xFF); + subRet2 += Integer.bitCount((q[i + 2 * size] & dValue) & 0xFF); + subRet3 += Integer.bitCount((q[i + 3 * size] & dValue) & 0xFF); + } + return subRet0 + (subRet1 << 1) + (subRet2 << 2) + (subRet3 << 3); + } + + private long quantizeScore128(byte[] q) throws IOException { + long subRet0 = 0; + long subRet1 = 0; + long subRet2 = 0; + long subRet3 = 0; + int i = 0; + long offset = in.getFilePointer(); + + var sum0 = IntVector.zero(INT_SPECIES_128); + var sum1 = IntVector.zero(INT_SPECIES_128); + var sum2 = IntVector.zero(INT_SPECIES_128); + var sum3 = IntVector.zero(INT_SPECIES_128); + int size = length / 4; + int limit = ByteVector.SPECIES_128.loopBound(size); + for (; i < limit; i += ByteVector.SPECIES_128.length(), offset += INT_SPECIES_128.vectorByteSize()) { + var vd = IntVector.fromMemorySegment(INT_SPECIES_128, memorySegment, offset, ByteOrder.LITTLE_ENDIAN); + var vq0 = ByteVector.fromArray(BYTE_SPECIES_128, q, i).reinterpretAsInts(); + var vq1 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + size).reinterpretAsInts(); + var vq2 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + size * 2).reinterpretAsInts(); + var vq3 = ByteVector.fromArray(BYTE_SPECIES_128, q, i + size * 3).reinterpretAsInts(); + sum0 = sum0.add(vd.and(vq0).lanewise(VectorOperators.BIT_COUNT)); + sum1 = sum1.add(vd.and(vq1).lanewise(VectorOperators.BIT_COUNT)); + sum2 = sum2.add(vd.and(vq2).lanewise(VectorOperators.BIT_COUNT)); + sum3 = sum3.add(vd.and(vq3).lanewise(VectorOperators.BIT_COUNT)); + } + subRet0 += sum0.reduceLanes(VectorOperators.ADD); + subRet1 += sum1.reduceLanes(VectorOperators.ADD); + subRet2 += sum2.reduceLanes(VectorOperators.ADD); + subRet3 += sum3.reduceLanes(VectorOperators.ADD); + // process scalar tail + in.seek(offset); + for (final int upperBound = size & -Long.BYTES; i < upperBound; i += Long.BYTES) { + final long value = in.readLong(); + subRet0 += Long.bitCount((long) BitUtil.VH_LE_LONG.get(q, i) & value); + subRet1 += Long.bitCount((long) BitUtil.VH_LE_LONG.get(q, i + size) & value); + subRet2 += Long.bitCount((long) BitUtil.VH_LE_LONG.get(q, i + 2 * size) & value); + subRet3 += Long.bitCount((long) BitUtil.VH_LE_LONG.get(q, i + 3 * size) & value); + } + for (final int upperBound = size & -Integer.BYTES; i < upperBound; i += Integer.BYTES) { + final int value = in.readInt(); + subRet0 += Integer.bitCount((int) BitUtil.VH_LE_INT.get(q, i) & value); + subRet1 += Integer.bitCount((int) BitUtil.VH_LE_INT.get(q, i + size) & value); + subRet2 += Integer.bitCount((int) BitUtil.VH_LE_INT.get(q, i + 2 * size) & value); + subRet3 += Integer.bitCount((int) BitUtil.VH_LE_INT.get(q, i + 3 * size) & value); + } + for (; i < size; i++) { + int dValue = in.readByte() & 0xFF; + subRet0 += Integer.bitCount((q[i] & dValue) & 0xFF); + subRet1 += Integer.bitCount((q[i + size] & dValue) & 0xFF); + subRet2 += Integer.bitCount((q[i + 2 * size] & dValue) & 0xFF); + subRet3 += Integer.bitCount((q[i + 3 * size] & dValue) & 0xFF); + } + return subRet0 + (subRet1 << 1) + (subRet2 << 2) + (subRet3 << 3); + } + + @Override + public boolean quantizeScoreBulk(byte[] q, int count, float[] scores) throws IOException { + assert q.length == length; + // 128 / 8 == 16 + if (length >= 16 && PanamaESVectorUtilSupport.HAS_FAST_INTEGER_VECTORS) { + if (PanamaESVectorUtilSupport.VECTOR_BITSIZE >= 256) { + quantizeScore256Bulk(q, count, scores); + return true; + } else if (PanamaESVectorUtilSupport.VECTOR_BITSIZE == 128) { + quantizeScore128Bulk(q, count, scores); + return true; + } + } + return false; + } + + private void quantizeScore128Bulk(byte[] q, int count, float[] scores) throws IOException { + for (int iter = 0; iter < count; iter++) { + scores[iter] = quantizeScoreSymmetric128(q); + } + } + + private void quantizeScore256Bulk(byte[] q, int count, float[] scores) throws IOException { + for (int iter = 0; iter < count; iter++) { + scores[iter] = quantizeScoreSymmetric256(q); + } + } + + @Override + public float scoreBulk( + byte[] q, + float queryLowerInterval, + float queryUpperInterval, + int queryComponentSum, + float queryAdditionalCorrection, + VectorSimilarityFunction similarityFunction, + float centroidDp, + float[] scores + ) throws IOException { + assert q.length == length; + // 128 / 8 == 16 + if (length >= 16 && PanamaESVectorUtilSupport.HAS_FAST_INTEGER_VECTORS) { + if (PanamaESVectorUtilSupport.VECTOR_BITSIZE >= 256) { + return score256Bulk( + q, + queryLowerInterval, + queryUpperInterval, + queryComponentSum, + queryAdditionalCorrection, + similarityFunction, + centroidDp, + scores + ); + } else if (PanamaESVectorUtilSupport.VECTOR_BITSIZE == 128) { + return score128Bulk( + q, + queryLowerInterval, + queryUpperInterval, + queryComponentSum, + queryAdditionalCorrection, + similarityFunction, + centroidDp, + scores + ); + } + } + return Float.NEGATIVE_INFINITY; + } + + private float score128Bulk( + byte[] q, + float queryLowerInterval, + float queryUpperInterval, + int queryComponentSum, + float queryAdditionalCorrection, + VectorSimilarityFunction similarityFunction, + float centroidDp, + float[] scores + ) throws IOException { + quantizeScore128Bulk(q, BULK_SIZE, scores); + int limit = FLOAT_SPECIES_128.loopBound(BULK_SIZE); + int i = 0; + long offset = in.getFilePointer(); + float ay = queryLowerInterval; + float ly = (queryUpperInterval - ay) * FOUR_BIT_SCALE; + float y1 = queryComponentSum; + float maxScore = Float.NEGATIVE_INFINITY; + for (; i < limit; i += FLOAT_SPECIES_128.length()) { + var ax = FloatVector.fromMemorySegment(FLOAT_SPECIES_128, memorySegment, offset + i * Float.BYTES, ByteOrder.LITTLE_ENDIAN); + var lx = FloatVector.fromMemorySegment( + FLOAT_SPECIES_128, + memorySegment, + offset + 4 * BULK_SIZE + i * Float.BYTES, + ByteOrder.LITTLE_ENDIAN + ).sub(ax).mul(FOUR_BIT_SCALE); + var targetComponentSums = ShortVector.fromMemorySegment( + SHORT_SPECIES_128, + memorySegment, + offset + 8 * BULK_SIZE + i * Short.BYTES, + ByteOrder.LITTLE_ENDIAN + ).convert(VectorOperators.S2I, 0).reinterpretAsInts().and(0xffff).convert(VectorOperators.I2F, 0); + var additionalCorrections = FloatVector.fromMemorySegment( + FLOAT_SPECIES_128, + memorySegment, + offset + 10 * BULK_SIZE + i * Float.BYTES, + ByteOrder.LITTLE_ENDIAN + ); + var qcDist = FloatVector.fromArray(FLOAT_SPECIES_128, scores, i); + // ax * ay * dimensions + ay * lx * (float) targetComponentSum + ax * ly * y1 + lx * ly * + // qcDist; + var res1 = ax.mul(ay).mul(dimensions); + var res2 = lx.mul(ay).mul(targetComponentSums); + var res3 = ax.mul(ly).mul(y1); + var res4 = lx.mul(ly).mul(qcDist); + var res = res1.add(res2).add(res3).add(res4); + // For euclidean, we need to invert the score and apply the additional correction, which is + // assumed to be the squared l2norm of the centroid centered vectors. + if (similarityFunction == EUCLIDEAN) { + res = res.mul(-2).add(additionalCorrections).add(queryAdditionalCorrection).add(1f); + res = FloatVector.broadcast(FLOAT_SPECIES_128, 1).div(res).max(0); + maxScore = Math.max(maxScore, res.reduceLanes(VectorOperators.MAX)); + res.intoArray(scores, i); + } else { + // For cosine and max inner product, we need to apply the additional correction, which is + // assumed to be the non-centered dot-product between the vector and the centroid + res = res.add(queryAdditionalCorrection).add(additionalCorrections).sub(centroidDp); + if (similarityFunction == MAXIMUM_INNER_PRODUCT) { + res.intoArray(scores, i); + // not sure how to do it better + for (int j = 0; j < FLOAT_SPECIES_128.length(); j++) { + scores[i + j] = VectorUtil.scaleMaxInnerProductScore(scores[i + j]); + maxScore = Math.max(maxScore, scores[i + j]); + } + } else { + res = res.add(1f).mul(0.5f).max(0); + res.intoArray(scores, i); + maxScore = Math.max(maxScore, res.reduceLanes(VectorOperators.MAX)); + } + } + } + in.seek(offset + 14L * BULK_SIZE); + return maxScore; + } + + private float score256Bulk( + byte[] q, + float queryLowerInterval, + float queryUpperInterval, + int queryComponentSum, + float queryAdditionalCorrection, + VectorSimilarityFunction similarityFunction, + float centroidDp, + float[] scores + ) throws IOException { + quantizeScore256Bulk(q, BULK_SIZE, scores); + int limit = FLOAT_SPECIES_256.loopBound(BULK_SIZE); + int i = 0; + long offset = in.getFilePointer(); + float ay = queryLowerInterval; + float ly = (queryUpperInterval - ay) * FOUR_BIT_SCALE; + float y1 = queryComponentSum; + float maxScore = Float.NEGATIVE_INFINITY; + for (; i < limit; i += FLOAT_SPECIES_256.length()) { + var ax = FloatVector.fromMemorySegment(FLOAT_SPECIES_256, memorySegment, offset + i * Float.BYTES, ByteOrder.LITTLE_ENDIAN); + var lx = FloatVector.fromMemorySegment( + FLOAT_SPECIES_256, + memorySegment, + offset + 4 * BULK_SIZE + i * Float.BYTES, + ByteOrder.LITTLE_ENDIAN + ).sub(ax).mul(FOUR_BIT_SCALE); + var targetComponentSums = ShortVector.fromMemorySegment( + SHORT_SPECIES_256, + memorySegment, + offset + 8 * BULK_SIZE + i * Short.BYTES, + ByteOrder.LITTLE_ENDIAN + ).convert(VectorOperators.S2I, 0).reinterpretAsInts().and(0xffff).convert(VectorOperators.I2F, 0); + var additionalCorrections = FloatVector.fromMemorySegment( + FLOAT_SPECIES_256, + memorySegment, + offset + 10 * BULK_SIZE + i * Float.BYTES, + ByteOrder.LITTLE_ENDIAN + ); + var qcDist = FloatVector.fromArray(FLOAT_SPECIES_256, scores, i); + // ax * ay * dimensions + ay * lx * (float) targetComponentSum + ax * ly * y1 + lx * ly * + // qcDist; + var res1 = ax.mul(ay).mul(dimensions); + var res2 = lx.mul(ay).mul(targetComponentSums); + var res3 = ax.mul(ly).mul(y1); + var res4 = lx.mul(ly).mul(qcDist); + var res = res1.add(res2).add(res3).add(res4); + // For euclidean, we need to invert the score and apply the additional correction, which is + // assumed to be the squared l2norm of the centroid centered vectors. + if (similarityFunction == EUCLIDEAN) { + res = res.mul(-2).add(additionalCorrections).add(queryAdditionalCorrection).add(1f); + res = FloatVector.broadcast(FLOAT_SPECIES_256, 1).div(res).max(0); + maxScore = Math.max(maxScore, res.reduceLanes(VectorOperators.MAX)); + res.intoArray(scores, i); + } else { + // For cosine and max inner product, we need to apply the additional correction, which is + // assumed to be the non-centered dot-product between the vector and the centroid + res = res.add(queryAdditionalCorrection).add(additionalCorrections).sub(centroidDp); + if (similarityFunction == MAXIMUM_INNER_PRODUCT) { + res.intoArray(scores, i); + // not sure how to do it better + for (int j = 0; j < FLOAT_SPECIES_256.length(); j++) { + scores[i + j] = VectorUtil.scaleMaxInnerProductScore(scores[i + j]); + maxScore = Math.max(maxScore, scores[i + j]); + } + } else { + res = res.add(1f).mul(0.5f).max(0); + maxScore = Math.max(maxScore, res.reduceLanes(VectorOperators.MAX)); + res.intoArray(scores, i); + } + } + } + in.seek(offset + 14L * BULK_SIZE); + return maxScore; + } +} diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MemorySegmentESNextOSQVectorsScorer.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MemorySegmentESNextOSQVectorsScorer.java index 52563fa959f27..492e19dd649ad 100644 --- a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MemorySegmentESNextOSQVectorsScorer.java +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MemorySegmentESNextOSQVectorsScorer.java @@ -25,20 +25,6 @@ /** Panamized scorer for quantized vectors stored as a {@link MemorySegment}. */ public final class MemorySegmentESNextOSQVectorsScorer extends ESNextOSQVectorsScorer { - private static final VectorSpecies INT_SPECIES_128 = IntVector.SPECIES_128; - - private static final VectorSpecies LONG_SPECIES_128 = LongVector.SPECIES_128; - private static final VectorSpecies LONG_SPECIES_256 = LongVector.SPECIES_256; - - private static final VectorSpecies BYTE_SPECIES_128 = ByteVector.SPECIES_128; - private static final VectorSpecies BYTE_SPECIES_256 = ByteVector.SPECIES_256; - - private static final VectorSpecies SHORT_SPECIES_128 = ShortVector.SPECIES_128; - private static final VectorSpecies SHORT_SPECIES_256 = ShortVector.SPECIES_256; - - private static final VectorSpecies FLOAT_SPECIES_128 = FloatVector.SPECIES_128; - private static final VectorSpecies FLOAT_SPECIES_256 = FloatVector.SPECIES_256; - private final MemorySegment memorySegment; private final MemorySegmentScorer scorer; @@ -54,6 +40,10 @@ public MemorySegmentESNextOSQVectorsScorer( this.memorySegment = memorySegment; if (queryBits == 4 && indexBits == 1) { this.scorer = new MSBitToInt4ESNextOSQVectorsScorer(in, dimensions, dataLength, memorySegment); + } else if (queryBits == 4 && indexBits == 4) { + this.scorer = new MSInt4SymmetricESNextOSQVectorsScorer(in, dimensions, dataLength, memorySegment); + } else if (queryBits == 4 && indexBits == 2) { + throw new IllegalArgumentException("Only symmetric 4-bit query and 1-bit index supported"); } else { throw new IllegalArgumentException("Only asymmetric 4-bit query and 1-bit index supported"); } @@ -112,7 +102,24 @@ public float scoreBulk( ); } - abstract static sealed class MemorySegmentScorer permits MSBitToInt4ESNextOSQVectorsScorer { + abstract static sealed class MemorySegmentScorer permits MSBitToInt4ESNextOSQVectorsScorer, MSInt4SymmetricESNextOSQVectorsScorer { + + static final int BULK_SIZE = MemorySegmentESNextOSQVectorsScorer.BULK_SIZE; + static final float FOUR_BIT_SCALE = 1f / ((1 << 4) - 1); + static final VectorSpecies INT_SPECIES_128 = IntVector.SPECIES_128; + + static final VectorSpecies LONG_SPECIES_128 = LongVector.SPECIES_128; + static final VectorSpecies LONG_SPECIES_256 = LongVector.SPECIES_256; + + static final VectorSpecies BYTE_SPECIES_128 = ByteVector.SPECIES_128; + static final VectorSpecies BYTE_SPECIES_256 = ByteVector.SPECIES_256; + + static final VectorSpecies SHORT_SPECIES_128 = ShortVector.SPECIES_128; + static final VectorSpecies SHORT_SPECIES_256 = ShortVector.SPECIES_256; + + static final VectorSpecies FLOAT_SPECIES_128 = FloatVector.SPECIES_128; + static final VectorSpecies FLOAT_SPECIES_256 = FloatVector.SPECIES_256; + protected final MemorySegment memorySegment; protected final IndexInput in; protected final int length; diff --git a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java index 17838c5bef05b..44e6937b5b23e 100644 --- a/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java @@ -40,7 +40,7 @@ public ESNextOSQVectorsScorer newESNextOSQVectorsScorer(IndexInput input, byte q if (PanamaESVectorUtilSupport.HAS_FAST_INTEGER_VECTORS && input instanceof MemorySegmentAccessInput msai && queryBits == 4 - && indexBits == 1) { + && (indexBits == 1 || indexBits == 4)) { MemorySegment ms = msai.segmentSliceOrNull(0, input.length()); if (ms != null) { return new MemorySegmentESNextOSQVectorsScorer(input, queryBits, indexBits, dimension, dataLength, ms); From da8b9c63ed1b60492fca13454dc362317baa5555 Mon Sep 17 00:00:00 2001 From: Pat Whelan Date: Mon, 3 Nov 2025 10:18:18 -0500 Subject: [PATCH 22/32] [ML] Skip dataframes when disabled (#137220) When Dataframes are disabled, do not try to call the Action and instead assume there are no trained models associated with a dataframe job. This avoids a transport exception for the disabled action. --- docs/changelog/137220.yaml | 5 ++++ .../xpack/ml/MachineLearning.java | 2 +- .../rest/cat/RestCatTrainedModelsAction.java | 26 ++++++++++++----- .../cat/RestCatTrainedModelsActionTests.java | 29 ++++++++++++------- 4 files changed, 42 insertions(+), 20 deletions(-) create mode 100644 docs/changelog/137220.yaml diff --git a/docs/changelog/137220.yaml b/docs/changelog/137220.yaml new file mode 100644 index 0000000000000..895a634d91501 --- /dev/null +++ b/docs/changelog/137220.yaml @@ -0,0 +1,5 @@ +pr: 137220 +summary: Skip dataframes when disabled +area: Machine Learning +type: bug +issues: [] diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 532f9cf5c71d8..3c6e6d685f377 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -1527,7 +1527,7 @@ public List getRestHandlers( restHandlers.add(new RestDeleteTrainedModelAliasAction()); restHandlers.add(new RestPutTrainedModelDefinitionPartAction()); restHandlers.add(new RestInferTrainedModelAction()); - restHandlers.add(new RestCatTrainedModelsAction()); + restHandlers.add(new RestCatTrainedModelsAction(machineLearningExtension.get().isDataFrameAnalyticsEnabled())); if (machineLearningExtension.get().isDataFrameAnalyticsEnabled()) { restHandlers.add(new RestGetDataFrameAnalyticsAction()); restHandlers.add(new RestGetDataFrameAnalyticsStatsAction()); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java index d4a2b602759ad..6299aa4bca6af 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java @@ -49,6 +49,12 @@ @ServerlessScope(Scope.PUBLIC) public class RestCatTrainedModelsAction extends AbstractCatAction { + private final boolean areDataFrameAnalyticsEnabled; + + public RestCatTrainedModelsAction(boolean areDataFrameAnalyticsEnabled) { + this.areDataFrameAnalyticsEnabled = areDataFrameAnalyticsEnabled; + } + @Override public List routes() { return List.of( @@ -122,14 +128,18 @@ private void getDerivedData( listeners.acquire(response -> trainedModelsStats = response.getResources().results()) ); - final var dataFrameAnalyticsRequest = new GetDataFrameAnalyticsAction.Request(requestIdPattern); - dataFrameAnalyticsRequest.setAllowNoResources(true); - dataFrameAnalyticsRequest.setPageParams(new PageParams(0, potentialAnalyticsIds.size())); - client.execute( - GetDataFrameAnalyticsAction.INSTANCE, - dataFrameAnalyticsRequest, - listeners.acquire(response -> dataFrameAnalytics = response.getResources().results()) - ); + if (areDataFrameAnalyticsEnabled) { + final var dataFrameAnalyticsRequest = new GetDataFrameAnalyticsAction.Request(requestIdPattern); + dataFrameAnalyticsRequest.setAllowNoResources(true); + dataFrameAnalyticsRequest.setPageParams(new PageParams(0, potentialAnalyticsIds.size())); + client.execute( + GetDataFrameAnalyticsAction.INSTANCE, + dataFrameAnalyticsRequest, + listeners.acquire(response -> dataFrameAnalytics = response.getResources().results()) + ); + } else { + dataFrameAnalytics = List.of(); + } } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsActionTests.java index f7cfba048e722..079cd5d9c4d0a 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsActionTests.java @@ -7,28 +7,25 @@ package org.elasticsearch.xpack.ml.rest.cat; +import org.elasticsearch.common.Strings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsStatsAction; import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsStatsActionResponseTests; +import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfigTests; import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfigTests; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TrainedModelSizeStats; -import org.junit.Before; import java.util.List; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; public class RestCatTrainedModelsActionTests extends ESTestCase { - private RestCatTrainedModelsAction action; - - @Before - public void setUpAction() { - action = new RestCatTrainedModelsAction(); - } - public void testBuildTableAccumulatedStats() { + var action = new RestCatTrainedModelsAction(true); + // GetTrainedModelsStatsActionResponseTests var deployment1 = new GetTrainedModelsStatsAction.Response.TrainedModelStats( "id1", @@ -48,10 +45,13 @@ public void testBuildTableAccumulatedStats() { null ); - var configs = List.of(TrainedModelConfigTests.createTestInstance("id1").build()); + var dataframeConfig = DataFrameAnalyticsConfigTests.createRandom("dataframe1"); + var configs = List.of( + TrainedModelConfigTests.createTestInstance(deployment1.getModelId()).setTags(List.of(dataframeConfig.getId())).build() + ); - var table = action.buildTable(new FakeRestRequest(), List.of(deployment1, deployment2), configs, List.of()); - assertThat(table.getRows().get(0).get(0).value, is("id1")); + var table = action.buildTable(new FakeRestRequest(), List.of(deployment1, deployment2), configs, List.of(dataframeConfig)); + assertThat(table.getRows().get(0).get(0).value, is(deployment1.getModelId())); // pipeline count assertThat(table.getRows().get(0).get(9).value, is(4)); // ingest count @@ -82,5 +82,12 @@ public void testBuildTableAccumulatedStats() { .ingestFailedCount() ) ); + assertThat(table.getRows().get(0).get(14).value, is(dataframeConfig.getId())); + assertThat(table.getRows().get(0).get(15).value, is(dataframeConfig.getCreateTime())); + assertThat(table.getRows().get(0).get(16).value, is(Strings.arrayToCommaDelimitedString(dataframeConfig.getSource().getIndex()))); + assertThat( + table.getRows().get(0).get(17).value, + dataframeConfig.getAnalysis() == null ? nullValue() : is(dataframeConfig.getAnalysis().getWriteableName()) + ); } } From b54cd8e05f66df61b10aadb9b521eedf09cd4851 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Mon, 3 Nov 2025 16:29:07 +0100 Subject: [PATCH 23/32] Mute org.elasticsearch.xpack.ilm.CCRIndexLifecycleIT testBasicCCRAndILMIntegration {targetCluster=FOLLOWER} #137494 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index f2ed2c6a4a5a7..1d7193fd8214c 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -504,6 +504,9 @@ tests: - class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTests method: testUpdates issue: https://github.com/elastic/elasticsearch/issues/137512 +- class: org.elasticsearch.xpack.ilm.CCRIndexLifecycleIT + method: testBasicCCRAndILMIntegration {targetCluster=FOLLOWER} + issue: https://github.com/elastic/elasticsearch/issues/137494 # Examples: # From b29af0f290fdf1b27431b20fec12f561f37cdad7 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Mon, 3 Nov 2025 16:29:15 +0100 Subject: [PATCH 24/32] Mute org.elasticsearch.xpack.ilm.CCRIndexLifecycleIT testCcrAndIlmWithRollover {targetCluster=FOLLOWER} #137528 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 1d7193fd8214c..b5e2fb03e2f6c 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -507,6 +507,9 @@ tests: - class: org.elasticsearch.xpack.ilm.CCRIndexLifecycleIT method: testBasicCCRAndILMIntegration {targetCluster=FOLLOWER} issue: https://github.com/elastic/elasticsearch/issues/137494 +- class: org.elasticsearch.xpack.ilm.CCRIndexLifecycleIT + method: testCcrAndIlmWithRollover {targetCluster=FOLLOWER} + issue: https://github.com/elastic/elasticsearch/issues/137528 # Examples: # From 4f3ff8191bd2b7027a05073193d493b982275e4b Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Mon, 3 Nov 2025 15:32:29 +0000 Subject: [PATCH 25/32] Fix the test dimensions for bit element_type (#137523) --- muted-tests.yml | 3 --- .../vectors/DenseVectorFieldMapperTests.java | 14 ++++++-------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index b5e2fb03e2f6c..3b420b5e51563 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -501,9 +501,6 @@ tests: - class: org.elasticsearch.xpack.esql.qa.single_node.PushQueriesIT method: testEqualityAndOther {SEMANTIC_TEXT_WITH_KEYWORD} issue: https://github.com/elastic/elasticsearch/issues/137491 -- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTests - method: testUpdates - issue: https://github.com/elastic/elasticsearch/issues/137512 - class: org.elasticsearch.xpack.ilm.CCRIndexLifecycleIT method: testBasicCCRAndILMIntegration {targetCluster=FOLLOWER} issue: https://github.com/elastic/elasticsearch/issues/137494 diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 3702af0ff33ed..645c61ee45e2b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -264,18 +264,16 @@ protected void registerParameters(ParameterChecker checker) throws IOException { registerConflict( checker, "element_type", - b -> b.field("type", "dense_vector").field("dims", dims).field("index", true).field("similarity", "l2_norm"), - "element_type", - "float", - "bit" + b -> b.field("type", "dense_vector").field("index", true).field("similarity", "l2_norm"), + b -> b.field("dims", dims).field("element_type", "float"), + b -> b.field("dims", dims * 8).field("element_type", "bit") ); registerConflict( checker, "element_type", - b -> b.field("type", "dense_vector").field("dims", dims).field("index", true).field("similarity", "l2_norm"), - "element_type", - "byte", - "bit" + b -> b.field("type", "dense_vector").field("index", true).field("similarity", "l2_norm"), + b -> b.field("dims", dims).field("element_type", "float"), + b -> b.field("dims", dims * 8).field("element_type", "bit") ); // update for flat From 1b74ec03be73972af94ab24153c0b4fb31dad5aa Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Mon, 3 Nov 2025 15:39:05 +0000 Subject: [PATCH 26/32] Add competitive sort tests to MapperTestCase (#137437) This adds a new abstract method to MapperTestCase for checking whether a particular field type configuration supports competitive iterators during sorting. --- .../LegacyGeoShapeFieldMapperTests.java | 5 ++ .../extras/MatchOnlyTextFieldMapperTests.java | 5 ++ .../extras/RankFeatureFieldMapperTests.java | 5 ++ .../extras/RankFeaturesFieldMapperTests.java | 5 ++ .../extras/ScaledFloatFieldMapperTests.java | 8 +++ .../SearchAsYouTypeFieldMapperTests.java | 5 ++ .../extras/TokenCountFieldMapperTests.java | 5 ++ .../ICUCollationKeywordFieldMapperTests.java | 5 ++ .../AnnotatedTextFieldMapperTests.java | 5 ++ .../murmur3/Murmur3FieldMapperTests.java | 5 ++ .../index/mapper/BinaryFieldMapperTests.java | 5 ++ .../index/mapper/BooleanFieldMapperTests.java | 9 +++ .../mapper/CompletionFieldMapperTests.java | 5 ++ .../index/mapper/DateFieldMapperTests.java | 20 ++++++ .../mapper/GeoPointFieldMapperTests.java | 5 ++ .../index/mapper/IpFieldMapperTests.java | 9 +++ .../index/mapper/KeywordFieldMapperTests.java | 5 ++ .../index/mapper/RangeFieldMapperTests.java | 5 ++ .../index/mapper/TextFieldMapperTests.java | 4 ++ .../flattened/FlattenedFieldMapperTests.java | 5 ++ .../SyntheticVectorsMapperTestCase.java | 6 ++ .../index/mapper/MapperTestCase.java | 66 +++++++++++++++++++ .../index/mapper/NumberFieldMapperTests.java | 9 +++ .../mapper/HistogramFieldMapperTests.java | 5 ++ .../mapper/OffsetSourceFieldMapperTests.java | 5 ++ .../mapper/SemanticTextFieldMapperTests.java | 5 ++ .../PatternTextFieldMapperTests.java | 5 ++ ...AggregateMetricDoubleFieldMapperTests.java | 5 ++ .../ConstantKeywordFieldMapperTests.java | 5 ++ .../CountedKeywordFieldMapperTests.java | 5 ++ .../ExponentialHistogramFieldMapperTests.java | 5 ++ .../UnsignedLongFieldMapperTests.java | 5 ++ .../VersionStringFieldMapperTests.java | 5 ++ ...GeoShapeWithDocValuesFieldMapperTests.java | 5 ++ .../index/mapper/PointFieldMapperTests.java | 5 ++ .../index/mapper/ShapeFieldMapperTests.java | 5 ++ .../mapper/WildcardFieldMapperTests.java | 5 ++ 37 files changed, 276 insertions(+) diff --git a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java index c97b0a28d22de..dcbf5bcb87e41 100644 --- a/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java +++ b/modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java @@ -678,4 +678,9 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java index ef72e234f8d6b..3a567490fc218 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java @@ -385,4 +385,9 @@ public void testLoadSyntheticSourceFromStringOrBytesRef() throws IOException { } } } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java index 917941b39e888..174bc44cbaecc 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java @@ -215,4 +215,9 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java index 6818b7936d124..c229752917560 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java @@ -209,4 +209,9 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean syntheticSource) protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java index dce105f339a52..7f988ebcb04f4 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java @@ -649,4 +649,12 @@ private double encodeDecode(double value, double scalingFactor) { private static double randomValue() { return randomBoolean() ? randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true) : randomFloat(); } + + @Override + protected List getSortShortcutSupport() { + return List.of( + // TODO doubles currently disable pruning, can we re-enable? + new SortShortcutSupport(this::minimalMapping, this::writeField, false) + ); + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java index 3e2bb422c2062..9965259e294da 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java @@ -877,4 +877,9 @@ protected RandomIndexWriter indexWriterForSyntheticSource(Directory directory) t protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java index 8105eee707cc3..b77ab875926e8 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java @@ -264,4 +264,9 @@ public void testAggregationsDocValuesDisabled() throws IOException { })); assertAggregatableConsistency(mapperService.fieldType("field")); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java index 034a25e854298..275a70eb43f57 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapperTests.java @@ -318,4 +318,9 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index ae3d03c3a503f..b808cffd75c7a 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -704,4 +704,9 @@ protected void validateRoundTripReader(String syntheticSource, DirectoryReader r protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index d04f2067d8aed..2978ee458dda9 100644 --- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -148,4 +148,9 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java index 7f257172638c2..8c3800bdf6132 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java @@ -263,4 +263,9 @@ public int compareTo(BytesCompareUnsigned o) { protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 47161239a363d..76df9b6303109 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -409,4 +409,13 @@ public void execute() { } }; } + + @Override + protected List getSortShortcutSupport() { + return List.of( + // TODO: boolean field mapper uses a numeric comparator but is indexed with Terms + // so skipping doesn't work here. + new SortShortcutSupport(this::minimalMapping, this::writeField, false) + ); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 995127e2b3593..079521b26b666 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -985,4 +985,9 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index 4d6368b712f98..f3e2633717a4b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -914,4 +914,24 @@ public void testSingletonLongBulkBlockReadingManyValues() throws Exception { } } } + + @Override + protected List getSortShortcutSupport() { + return List.of( + new SortShortcutSupport(b -> b.field("type", "date"), b -> b.field("field", "2025-10-30T00:00:00"), true), + new SortShortcutSupport(b -> b.field("type", "date_nanos"), b -> b.field("field", "2025-10-30T00:00:00"), true), + new SortShortcutSupport( + IndexVersion.fromId(5000099), + b -> b.field("type", "date"), + b -> b.field("field", "2025-10-30T00:00:00"), + false + ), + new SortShortcutSupport( + IndexVersion.fromId(5000099), + b -> b.field("type", "date_nanos"), + b -> b.field("field", "2025-10-30T00:00:00"), + false + ) + ); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java index 9ffe1ba394b0b..c560448f6bb33 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java @@ -719,4 +719,9 @@ public void testSyntheticSourceKeepArrays() { protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 2ca2a7f8eb62d..e0308b5fd8bbf 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -438,4 +438,13 @@ public void execute() { protected String randomSyntheticSourceKeep() { return "all"; } + + @Override + protected List getSortShortcutSupport() { + return List.of( + // TODO - shortcuts are disabled here, can we enable them? + new SortShortcutSupport(this::minimalMapping, this::writeField, false), + new SortShortcutSupport(IndexVersion.fromId(5000099), this::minimalMapping, this::writeField, false) + ); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 520ec8a8c3400..641ddeb170f77 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -1219,4 +1219,9 @@ protected String randomSyntheticSourceKeep() { // Only option all keeps array source in ignored source. return randomFrom("all"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(new SortShortcutSupport(this::minimalMapping, this::writeField, true)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index 7555d1cd7fb48..6dd85a3f8c5e6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -426,4 +426,9 @@ protected Object generateRandomInputValue(MappedFieldType ft) { assumeFalse("DocValuesFetcher doesn't work", true); return null; } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 970dd8886b93f..4947af5eeba11 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1776,4 +1776,8 @@ public void testNormsEnabledWhenIndexModeIsTsdb_bwcCheck() throws IOException { assertThat(fieldType.omitNorms(), is(false)); } + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index d9223a157b2ff..3939fcaa6e0e8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -1016,4 +1016,9 @@ public void assertStoredFieldsEquals(String info, IndexReader leftReader, IndexR assertFalse(info, rightIterator.hasNext()); } } + + @Override + protected List getSortShortcutSupport() { + return List.of(new SortShortcutSupport(this::minimalMapping, this::writeField, true)); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SyntheticVectorsMapperTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SyntheticVectorsMapperTestCase.java index f7a23383f4e92..dbf4bd9846165 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SyntheticVectorsMapperTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SyntheticVectorsMapperTestCase.java @@ -25,6 +25,7 @@ import org.elasticsearch.xcontent.XContentType; import java.io.IOException; +import java.util.List; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; @@ -262,4 +263,9 @@ private void assertSyntheticVectors(String mapping, BytesReference source, XCont } } } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 3ef66cebc1f14..f7ee3dcdbda43 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -22,6 +22,7 @@ import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.search.FieldExistsQuery; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Pruning; import org.apache.lucene.search.Query; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; @@ -64,6 +65,7 @@ import org.elasticsearch.script.ScriptFactory; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.lookup.LeafStoredFieldsLookup; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.Source; @@ -1697,4 +1699,68 @@ protected T compileOtherScript(Script script, ScriptContext context) { */ abstract ScriptFactory nonEmptyFieldScript(); } + + /** + * Return a list of scenarios where sorts on this field should or should not produce + * competitive iterators. Field types that do not support sorting should return an + * empty List. + */ + protected abstract List getSortShortcutSupport(); + + public record SortShortcutSupport( + IndexVersion indexVersion, + Settings settings, + CheckedConsumer mappings, + CheckedConsumer document, + boolean supportsShortcut + ) { + public SortShortcutSupport( + CheckedConsumer mappings, + CheckedConsumer document, + boolean supportsShortcut + ) { + this(IndexVersion.current(), SETTINGS, mappings, document, supportsShortcut); + } + + public SortShortcutSupport( + IndexVersion indexVersion, + CheckedConsumer mappings, + CheckedConsumer document, + boolean supportsShortcut + ) { + this(indexVersion, SETTINGS, mappings, document, supportsShortcut); + } + } + + public final void testSortShortcuts() throws IOException { + List tests = getSortShortcutSupport(); + assumeTrue("Sort shortcuts not supported", tests != null && tests.isEmpty() == false); + + for (SortShortcutSupport sortShortcutSupport : tests) { + MapperService mapperService = createMapperService(sortShortcutSupport.indexVersion(), sortShortcutSupport.settings, () -> true); + merge(mapperService, fieldMapping(sortShortcutSupport.mappings)); + withLuceneIndex(mapperService, iw -> { + iw.addDocument( + mapperService.documentParser() + .parseDocument(source(sortShortcutSupport.document()), mapperService.mappingLookup()) + .rootDoc() + ); + }, reader -> { + IndexSearcher searcher = newSearcher(reader); + MappedFieldType ft = mapperService.fieldType("field"); + SortField sortField = ft.fielddataBuilder(new FieldDataContext("", mapperService.getIndexSettings(), () -> { + throw new UnsupportedOperationException(); + }, Set::of, MappedFieldType.FielddataOperation.SEARCH)) + .build(null, null) + .sortField(getVersion(), null, MultiValueMode.MIN, null, false); + var comparator = sortField.getComparator(10, Pruning.GREATER_THAN_OR_EQUAL_TO); + var leafComparator = comparator.getLeafComparator(searcher.getLeafContexts().getFirst()); + if (sortShortcutSupport.supportsShortcut) { + assertNotNull(leafComparator.competitiveIterator()); + } else { + assertNull(leafComparator.competitiveIterator()); + } + }); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 18e60c008db96..2f16534908b2e 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.script.DoubleFieldScript; import org.elasticsearch.script.LongFieldScript; import org.elasticsearch.script.Script; @@ -496,4 +497,12 @@ public List invalidExample() throws IOException { return List.of(); } } + + @Override + protected List getSortShortcutSupport() { + return List.of( + new SortShortcutSupport(this::minimalMapping, this::writeField, true), + new SortShortcutSupport(IndexVersion.fromId(5000099), this::minimalMapping, this::writeField, false) + ); + } } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java index ef096f344f352..b26b33cc01544 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java @@ -523,4 +523,9 @@ public List invalidExample() throws IOException { public void testSyntheticSourceKeepArrays() { // The mapper expects to parse an array of values by default, it's not compatible with array of arrays. } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java index b3ee52c8dede5..384a321d524be 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/OffsetSourceFieldMapperTests.java @@ -214,4 +214,9 @@ public void testInvalidOffsets() throws IOException { }))); assertThat(exc.getCause().getCause().getCause().getMessage(), containsString("Illegal offsets")); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 273a202baa7d6..2c9c404fcb275 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -2369,4 +2369,9 @@ private static void assertSparseFeatures(LuceneDocument doc, String fieldName, i private void givenModelSettings(String inferenceId, MinimalServiceSettings modelSettings) { when(globalModelRegistry.getMinimalServiceSettings(inferenceId)).thenReturn(modelSettings); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java index 1b3ccb376febf..8a684122cb658 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextFieldMapperTests.java @@ -470,4 +470,9 @@ public void testSyntheticSourceKeepArrays() { protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java index 874ced5436f69..b1e31882fbf1a 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapperTests.java @@ -618,4 +618,9 @@ public void testSyntheticSourceKeepArrays() { protected boolean supportsCopyTo() { return false; } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index d5405018dd64c..cdbfc6c3b9582 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -346,4 +346,9 @@ protected boolean supportsEmptyInputArray() { protected boolean addsValueWhenNotSupplied() { return true; } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapperTests.java b/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapperTests.java index d6b762ab4b79e..5c26222ff1618 100644 --- a/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-counted-keyword/src/test/java/org/elasticsearch/xpack/countedkeyword/CountedKeywordFieldMapperTests.java @@ -219,4 +219,9 @@ public void testDisableIndex() throws IOException { assertEquals(IndexOptions.NONE, fields.get(0).fieldType().indexOptions()); assertEquals(DocValuesType.SORTED_SET, fields.get(0).fieldType().docValuesType()); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java b/x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java index 730d0b591ec7a..f782e7f30b7e3 100644 --- a/x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java +++ b/x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapperTests.java @@ -605,4 +605,9 @@ public void testSyntheticSourceKeepArrays() { protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not yet implemented"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index cb28a6168c561..ea3e4df43d615 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -520,4 +520,9 @@ protected Object[] getThreeEncodedSampleValues() { protected boolean supportsBulkLongBlockReading() { return true; } + + @Override + protected List getSortShortcutSupport() { + return List.of(new SortShortcutSupport(this::minimalMapping, this::writeField, true)); + } } diff --git a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index 710b6652ed05d..61d6021005f56 100644 --- a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -198,4 +198,9 @@ public List invalidExample() throws IOException { return List.of(); } } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java index 9482f45063b64..d656f407b3243 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java @@ -553,4 +553,9 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java index 70df2763e30c6..3cb9f531b23dc 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java @@ -553,4 +553,9 @@ protected Object[] getThreeEncodedSampleValues() { protected boolean supportsBulkLongBlockReading() { return true; } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java index ee6f6bc8b70c0..9530f24a153a1 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java @@ -372,4 +372,9 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index be216c55d49cf..0690d38fe79ce 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -1272,4 +1272,9 @@ public List invalidExample() throws IOException { return List.of(); } } + + @Override + protected List getSortShortcutSupport() { + return List.of(); + } } From 9795d0f15bc9151c6b49c8c66711701201acd078 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Mon, 3 Nov 2025 15:43:02 +0000 Subject: [PATCH 27/32] We don't need a separate search impl here --- .../ES93ScalarQuantizedVectorsFormat.java | 72 +------------------ 1 file changed, 1 insertion(+), 71 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java index 90cb3aec59ed8..075c1728f1029 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es93/ES93ScalarQuantizedVectorsFormat.java @@ -14,24 +14,14 @@ import org.apache.lucene.codecs.KnnVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; -import org.apache.lucene.codecs.hnsw.FlatVectorsReader; import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorScorer; import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat; import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsReader; import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsWriter; -import org.apache.lucene.index.ByteVectorValues; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FloatVectorValues; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.search.AcceptDocs; -import org.apache.lucene.search.KnnCollector; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector; -import org.apache.lucene.util.hnsw.RandomVectorScorer; import java.io.IOException; -import java.util.Map; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MAX_DIMS_COUNT; @@ -71,9 +61,7 @@ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException @Override public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { - return new ES93FlatVectorsReader( - new Lucene104ScalarQuantizedVectorsReader(state, rawVectorFormat.fieldsReader(state), flatVectorScorer) - ); + return new Lucene104ScalarQuantizedVectorsReader(state, rawVectorFormat.fieldsReader(state), flatVectorScorer); } @Override @@ -94,62 +82,4 @@ public String toString() { + rawVectorFormat + ")"; } - - public static class ES93FlatVectorsReader extends KnnVectorsReader { - - private final FlatVectorsReader reader; - - public ES93FlatVectorsReader(FlatVectorsReader reader) { - super(); - this.reader = reader; - } - - @Override - public void checkIntegrity() throws IOException { - reader.checkIntegrity(); - } - - @Override - public FloatVectorValues getFloatVectorValues(String field) throws IOException { - return reader.getFloatVectorValues(field); - } - - @Override - public ByteVectorValues getByteVectorValues(String field) throws IOException { - return reader.getByteVectorValues(field); - } - - @Override - public void search(String field, float[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { - collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); - } - - private void collectAllMatchingDocs(KnnCollector knnCollector, AcceptDocs acceptDocs, RandomVectorScorer scorer) - throws IOException { - OrdinalTranslatedKnnCollector collector = new OrdinalTranslatedKnnCollector(knnCollector, scorer::ordToDoc); - Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs.bits()); - for (int i = 0; i < scorer.maxOrd(); i++) { - if (acceptedOrds == null || acceptedOrds.get(i)) { - collector.collect(i, scorer.score(i)); - collector.incVisitedCount(1); - } - } - assert collector.earlyTerminated() == false; - } - - @Override - public void search(String field, byte[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { - collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); - } - - @Override - public Map getOffHeapByteSize(FieldInfo fieldInfo) { - return reader.getOffHeapByteSize(fieldInfo); - } - - @Override - public void close() throws IOException { - reader.close(); - } - } } From 94732160f9ca0a7e6f3cee9d1601c9336e4320fc Mon Sep 17 00:00:00 2001 From: Matteo Piergiovanni <134913285+piergm@users.noreply.github.com> Date: Mon, 3 Nov 2025 16:44:12 +0100 Subject: [PATCH 28/32] Field caps transport changes to return for each original expression what it was resolved to (#136632) * alexey-ivanov-es work * iter * field caps: local resolution working and tested * CCS impl * using CPS SAF resolution if present * some TODOs to check later on * [CI] Update transport version definitions * iter * iter * iter * iter * Update docs/changelog/136632.yaml * [CI] Auto commit changes from spotless * iter * iter * iter * iter * only return resolved to if minTransportVersion supports it * iter * updated comment based on feedback --------- Co-authored-by: elasticsearchmachine --- docs/changelog/136632.yaml | 6 + .../fieldcaps/CCSFieldCapabilitiesIT.java | 138 +++++++++++++ .../search/fieldcaps/FieldCapabilitiesIT.java | 167 +++++++++++++++ .../action/ResolvedIndexExpression.java | 13 ++ .../action/ResolvedIndexExpressions.java | 8 + .../fieldcaps/FieldCapabilitiesRequest.java | 20 ++ .../FieldCapabilitiesRequestBuilder.java | 5 + .../fieldcaps/FieldCapabilitiesResponse.java | 110 +++++++++- .../TransportFieldCapabilitiesAction.java | 194 ++++++++++++++++-- .../referable/resolved_fields_caps.csv | 1 + .../resources/transport/upper_bounds/9.3.csv | 2 +- .../FieldCapabilitiesResponseTests.java | 8 +- .../MergedFieldCapabilitiesResponseTests.java | 10 +- .../action/fieldcaps/FieldCapsUtils.java | 7 +- .../analyses/ClassificationTests.java | 16 +- .../xpack/esql/analysis/AnalyzerTests.java | 26 +-- .../enrich/EnrichPolicyResolverTests.java | 4 +- .../esql/type/EsqlDataTypeRegistryTests.java | 2 +- .../ml/dataframe/DestinationIndexTests.java | 44 ++-- .../ExtractedFieldsDetectorTests.java | 2 +- .../analysis/index/IndexResolverTests.java | 4 +- .../common/DocumentConversionUtilsTests.java | 18 +- .../transforms/pivot/SchemaUtilTests.java | 5 +- 23 files changed, 720 insertions(+), 90 deletions(-) create mode 100644 docs/changelog/136632.yaml create mode 100644 server/src/main/resources/transport/definitions/referable/resolved_fields_caps.csv diff --git a/docs/changelog/136632.yaml b/docs/changelog/136632.yaml new file mode 100644 index 0000000000000..1a63e135fefcc --- /dev/null +++ b/docs/changelog/136632.yaml @@ -0,0 +1,6 @@ +pr: 136632 +summary: Field caps transport changes to return for each original expression what + it was resolved to +area: Search +type: enhancement +issues: [] diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java index 650fad8a496ad..25fe5bdb222cb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java @@ -11,8 +11,11 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.TransportVersion; +import org.elasticsearch.action.ResolvedIndexExpression; +import org.elasticsearch.action.ResolvedIndexExpressions; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.shard.IllegalIndexShardStateException; @@ -22,11 +25,13 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Map; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; @@ -34,6 +39,7 @@ import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; public class CCSFieldCapabilitiesIT extends AbstractMultiClustersTestCase { @@ -268,6 +274,138 @@ public void testReturnAllLocal() { } } + public void testResolvedToMatchingEverywhere() { + String localIndex = "index-local"; + String remoteIndex = "index-remote"; + String remoteClusterAlias = "remote_cluster"; + populateIndices(localIndex, remoteIndex, remoteClusterAlias, false); + String remoteIndexWithCluster = String.join(":", remoteClusterAlias, remoteIndex); + FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndexWithCluster) + .setFields("*") + .setIncludeResolvedTo(true) + .get(); + + assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex, remoteIndexWithCluster)); + + ResolvedIndexExpressions local = response.getResolvedLocally(); + assertThat(local, notNullValue()); + assertThat(local.expressions(), hasSize(1)); + assertEquals( + local.expressions().get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + ); + + List localIndicesList = local.getLocalIndicesList(); + assertThat(localIndicesList, hasSize(1)); + assertThat(localIndicesList, containsInAnyOrder(localIndex)); + + Map remote = response.getResolvedRemotely(); + assertThat(remote, notNullValue()); + assertThat(remote, aMapWithSize(1)); + assertThat(remote.keySet(), contains(remoteClusterAlias)); + + ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias); + List remoteIndicesList = remoteResponse.getLocalIndicesList(); + assertThat(remoteIndicesList, hasSize(1)); + assertEquals( + remoteResponse.expressions().get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + ); + assertThat(remoteIndicesList, containsInAnyOrder(remoteIndex)); + } + + public void testResolvedToMatchingLocallyOnly() { + String localIndex = "index-local"; + String remoteIndex = "index-remote"; + String remoteClusterAlias = "remote_cluster"; + String nonExistentIndex = "non-existent-index"; + populateIndices(localIndex, remoteIndex, remoteClusterAlias, false); + String remoteIndexWithCluster = String.join(":", remoteClusterAlias, nonExistentIndex); + FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndexWithCluster) + .setFields("*") + .setIncludeResolvedTo(true) + .get(); + + assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex)); + + ResolvedIndexExpressions local = response.getResolvedLocally(); + assertThat(local, notNullValue()); + assertThat(local.expressions(), hasSize(1)); + assertEquals( + local.expressions().get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + ); + + List localIndicesList = local.getLocalIndicesList(); + assertThat(localIndicesList, hasSize(1)); + assertThat(localIndicesList, containsInAnyOrder(localIndex)); + + Map remote = response.getResolvedRemotely(); + assertThat(remote, notNullValue()); + assertThat(remote, aMapWithSize(1)); + assertThat(remote.keySet(), contains(remoteClusterAlias)); + + ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias); + List remoteIndicesList = remoteResponse.getLocalIndicesList(); + assertThat(remoteIndicesList, hasSize(0)); + List remoteResolvedExpressions = remoteResponse.expressions(); + assertEquals(1, remoteResolvedExpressions.size()); + assertEquals( + remoteResolvedExpressions.get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE + ); + assertEquals(0, remoteIndicesList.size()); + } + + public void testResolvedToMatchingRemotelyOnly() { + String localIndex = "index-local"; + String remoteIndex = "index-remote"; + String remoteClusterAlias = "remote_cluster"; + String nonExistentIndex = "non-existent-index"; + populateIndices(localIndex, remoteIndex, remoteClusterAlias, false); + String remoteIndexWithCluster = String.join(":", remoteClusterAlias, remoteIndex); + boolean ignoreUnavailable = true; + IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false); + + FieldCapabilitiesResponse response = client().prepareFieldCaps(nonExistentIndex, remoteIndexWithCluster) + .setFields("*") + .setIncludeResolvedTo(true) + .setIndicesOptions(options) // without ignore unavaliable would throw error + .get(); + + assertThat(response.getIndices(), arrayContainingInAnyOrder(remoteIndexWithCluster)); + + ResolvedIndexExpressions local = response.getResolvedLocally(); + assertThat(local, notNullValue()); + assertThat(local.expressions(), hasSize(1)); + assertEquals( + local.expressions().get(0).localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE + ); + + List localIndicesList = local.getLocalIndicesList(); + assertThat(localIndicesList, hasSize(0)); + + Map remote = response.getResolvedRemotely(); + assertThat(remote, notNullValue()); + assertThat(remote, aMapWithSize(1)); + assertThat(remote.keySet(), contains(remoteClusterAlias)); + + ResolvedIndexExpressions remoteResponse = remote.get(remoteClusterAlias); + List remoteIndicesList = remoteResponse.getLocalIndicesList(); + assertThat(remoteIndicesList, hasSize(1)); + assertThat(remoteIndicesList, containsInAnyOrder(remoteIndex)); + List remoteResolvedExpressions = remoteResponse.expressions(); + assertEquals(1, remoteResolvedExpressions.size()); + ResolvedIndexExpression remoteExpression = remoteResolvedExpressions.get(0); + assertEquals( + remoteExpression.localExpressions().localIndexResolutionResult(), + ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + ); + assertEquals(1, remoteExpression.localExpressions().indices().size()); + assertEquals(remoteIndex, remoteResolvedExpressions.get(0).original()); + } + public void testIncludesMinTransportVersion() { if (randomBoolean()) { assertAcked(client().admin().indices().prepareCreate("index")); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index b9fcec556213a..91a990d326e54 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -13,6 +13,8 @@ import org.apache.http.entity.StringEntity; import org.apache.logging.log4j.Level; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ResolvedIndexExpression; +import org.elasticsearch.action.ResolvedIndexExpressions; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; @@ -102,6 +104,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; @@ -913,6 +916,170 @@ public void testIndexMode() throws Exception { assertThat(actualIndexModes, equalTo(indexModes)); } + public void testResolvedExpressionWithIndexAlias() { + FieldCapabilitiesResponse response = client().prepareFieldCaps("current").setFields("*").setIncludeResolvedTo(true).get(); + assertIndices(response, "new_index"); + + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("current", expression.original()); + assertThat(expression.localExpressions().indices(), containsInAnyOrder("new_index")); + } + + public void testResolvedExpressionWithWildcard() { + FieldCapabilitiesResponse response = client().prepareFieldCaps("*index").setFields("*").setIncludeResolvedTo(true).get(); + assertIndices(response, "new_index", "old_index"); + + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("*index", expression.original()); + assertThat(expression.localExpressions().indices(), containsInAnyOrder("new_index", "old_index")); + } + + public void testResolvedExpressionWithClosedIndices() throws IOException { + // in addition to the existing "old_index" and "new_index", create two where the test query throws an error on rewrite + assertAcked(prepareCreate("index1-error"), prepareCreate("index2-error")); + ensureGreen("index1-error", "index2-error"); + + // Closed shards will result to index error because shards must be in readable state + closeShards(internalCluster(), "index1-error", "index2-error"); + + FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index", "new_index", "index1-error", "index2-error") + .setFields("*") + .setIncludeResolvedTo(true) + .get(); + Set openIndices = Set.of("old_index", "new_index"); + Set closedIndices = Set.of("index1-error", "index2-error"); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(4, resolvedLocally.expressions().size()); + for (ResolvedIndexExpression expression : expressions) { + ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions(); + if (openIndices.contains(expression.original())) { + assertThat(expression.localExpressions().indices(), containsInAnyOrder(expression.original())); + assertEquals(ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS, localExpressions.localIndexResolutionResult()); + } else if (closedIndices.contains(expression.original())) { + Set concreteIndices = localExpressions.indices(); + assertEquals(0, concreteIndices.size()); + assertEquals( + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + localExpressions.localIndexResolutionResult() + ); + } + } + } + + public void testResolvedExpressionWithAllIndices() { + FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("*").setIncludeResolvedTo(true).get(); + assertIndices(response, "new_index", "old_index"); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("_all", expression.original()); // not setting indices means _all + ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions(); + assertThat(expression.localExpressions().indices(), containsInAnyOrder("new_index", "old_index")); + assertEquals(ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS, localExpressions.localIndexResolutionResult()); + } + + public void testResolvedExpressionWithOnlyOneClosedIndexAndIgnoreUnavailable() { + boolean ignoreUnavailable = true; + IndicesOptions options = IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, true, true, false, false); + client().admin().indices().close(new CloseIndexRequest("old_index")).actionGet(); + FieldCapabilitiesResponse response = client().prepareFieldCaps("old_index") + .setFields("*") + .setIndicesOptions(options) + .setIncludeResolvedTo(true) + .get(); + + assertIndices(response); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, expressions.size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("old_index", expression.original()); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression.LocalExpressions localExpressions = expression.localExpressions(); + Set concreteIndices = localExpressions.indices(); + assertEquals(0, concreteIndices.size()); + assertEquals( + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + localExpressions.localIndexResolutionResult() + ); + } + + public void testResolvedExpressionWithIndexFilter() throws InterruptedException { + assertAcked( + prepareCreate("index-1").setMapping("timestamp", "type=date", "field1", "type=keyword"), + prepareCreate("index-2").setMapping("timestamp", "type=date", "field1", "type=long") + ); + + List reqs = new ArrayList<>(); + reqs.add(prepareIndex("index-1").setSource("timestamp", "2015-07-08")); + reqs.add(prepareIndex("index-1").setSource("timestamp", "2018-07-08")); + reqs.add(prepareIndex("index-2").setSource("timestamp", "2019-10-12")); + reqs.add(prepareIndex("index-2").setSource("timestamp", "2020-07-08")); + indexRandom(true, reqs); + + FieldCapabilitiesResponse response = client().prepareFieldCaps("index-*") + .setFields("*") + .setIndexFilter(QueryBuilders.rangeQuery("timestamp").gte("2019-11-01")) + .setIncludeResolvedTo(true) + .get(); + + assertIndices(response, "index-2"); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("index-*", expression.original()); + assertThat(expression.localExpressions().indices(), containsInAnyOrder("index-1", "index-2")); + } + + public void testNoneExpressionIndices() { + // The auth code injects the pattern ["*", "-*"] which effectively means a request that requests no indices + FieldCapabilitiesResponse response = client().prepareFieldCaps("*", "-*").setFields("*").get(); + + assertThat(response.getIndices().length, is(0)); + } + + public void testExclusion() { + assertAcked(prepareCreate("index-2024"), prepareCreate("index-2025")); + + prepareIndex("index-2024").setSource("timestamp", "2024", "f1", "1").get(); + prepareIndex("index-2025").setSource("timestamp", "2025", "f2", "2").get(); + + var response = client().prepareFieldCaps("index-*", "-*2025").setFields("*").get(); + assertIndices(response, "index-2024"); + } + + public void testExclusionWithResolvedTo() { + assertAcked(prepareCreate("index-2024"), prepareCreate("index-2025")); + + prepareIndex("index-2024").setSource("timestamp", "2024", "f1", "1").get(); + prepareIndex("index-2025").setSource("timestamp", "2025", "f2", "2").get(); + + var response = client().prepareFieldCaps("index-*", "-*2025").setFields("*").setIncludeResolvedTo(true).get(); + assertIndices(response, "index-2024"); + assertEquals(0, response.getResolvedRemotely().size()); + ResolvedIndexExpressions resolvedLocally = response.getResolvedLocally(); + List expressions = resolvedLocally.expressions(); + assertEquals(1, resolvedLocally.expressions().size()); + ResolvedIndexExpression expression = expressions.get(0); + assertEquals("index-*", expression.original()); + assertThat(expression.localExpressions().indices(), containsInAnyOrder("index-2024", "index-2025")); + } + private void assertIndices(FieldCapabilitiesResponse response, String... indices) { assertNotNull(response.getIndices()); Arrays.sort(indices); diff --git a/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpression.java b/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpression.java index e1db2128143e9..1204ab001e14e 100644 --- a/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpression.java +++ b/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpression.java @@ -71,6 +71,19 @@ public void writeTo(StreamOutput out) throws IOException { * or unauthorized concrete resources. * A wildcard expression resolving to nothing is still considered a successful resolution. * The NONE result indicates that no local resolution was attempted because the expression is known to be remote-only. + * + * This distinction is needed to return either 403 (forbidden) or 404 (not found) to the user, + * and must be propagated by the linked projects to the request coordinator. + * + * CONCRETE_RESOURCE_NOT_VISIBLE: Indicates that a non-wildcard expression was resolved to nothing, + * either because the index does not exist or is closed. + * + * CONCRETE_RESOURCE_UNAUTHORIZED: Indicates that the expression could be resolved to a concrete index, + * but the requesting user is not authorized to access it. + * + * NONE: No local resolution was attempted, typically because the expression is remote-only. + * + * SUCCESS: Local index resolution was successful. */ public enum LocalIndexResolutionResult { NONE, diff --git a/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java b/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java index 301db629d8d69..007db112183dd 100644 --- a/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java +++ b/server/src/main/java/org/elasticsearch/action/ResolvedIndexExpressions.java @@ -73,6 +73,14 @@ public void addExpressions( ); } + /** + * Add a new resolved expression. + * @param expression the expression you want to add. + */ + public void addExpression(ResolvedIndexExpression expression) { + expressions.add(expression); + } + public void addRemoteExpressions(String original, Set remoteExpressions) { Objects.requireNonNull(original); Objects.requireNonNull(remoteExpressions); diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java index c4c974ad6b668..c17cae1779056 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java @@ -40,6 +40,7 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implemen public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpenAndForbidClosed(); private static final TransportVersion FIELD_CAPS_ADD_CLUSTER_ALIAS = TransportVersion.fromName("field_caps_add_cluster_alias"); + static final TransportVersion RESOLVED_FIELDS_CAPS = TransportVersion.fromName("resolved_fields_caps"); private String clusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; @@ -58,6 +59,8 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implemen */ private transient boolean includeIndices = false; + private boolean includeResolvedTo = false; + /** * Controls whether all local indices should be returned if no remotes matched * See {@link org.elasticsearch.transport.RemoteClusterService#groupIndices} returnLocalAll argument. @@ -93,6 +96,11 @@ public FieldCapabilitiesRequest(StreamInput in) throws IOException { } else { clusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; } + if (in.getTransportVersion().supports(RESOLVED_FIELDS_CAPS)) { + includeResolvedTo = in.readBoolean(); + } else { + includeResolvedTo = false; + } } public FieldCapabilitiesRequest() {} @@ -145,6 +153,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().supports(FIELD_CAPS_ADD_CLUSTER_ALIAS)) { out.writeOptionalString(clusterAlias); } + if (out.getTransportVersion().supports(RESOLVED_FIELDS_CAPS)) { + out.writeBoolean(includeResolvedTo); + } } @Override @@ -223,6 +234,11 @@ public FieldCapabilitiesRequest includeIndices(boolean includeIndices) { return this; } + public FieldCapabilitiesRequest includeResolvedTo(boolean includeResolvedTo) { + this.includeResolvedTo = includeResolvedTo; + return this; + } + public FieldCapabilitiesRequest returnLocalAll(boolean returnLocalAll) { this.returnLocalAll = returnLocalAll; return this; @@ -256,6 +272,10 @@ public boolean includeIndices() { return includeIndices; } + public boolean includeResolvedTo() { + return includeResolvedTo; + } + public boolean returnLocalAll() { return returnLocalAll; } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java index 4437895c7e08d..9e79c52ce8ae1 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java @@ -63,4 +63,9 @@ public FieldCapabilitiesRequestBuilder setReturnLocalAll(boolean returnLocalAll) request().returnLocalAll(returnLocalAll); return this; } + + public FieldCapabilitiesRequestBuilder setIncludeResolvedTo(boolean resolvedTo) { + request().includeResolvedTo(resolvedTo); + return this; + } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java index daea9b1a5fc01..8372ad5909de1 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java @@ -11,6 +11,7 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ResolvedIndexExpressions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; @@ -27,6 +28,9 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; + +import static org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest.RESOLVED_FIELDS_CAPS; /** * Response for {@link FieldCapabilitiesRequest} requests. @@ -41,6 +45,19 @@ public class FieldCapabilitiesResponse extends ActionResponse implements Chunked public static final ParseField FAILURES_FIELD = new ParseField("failures"); private final String[] indices; + + // Index expressions resolved in the context of the node that creates this response. + // If created on a linked project, "local" refers to that linked project. + // If created on the coordinating node, "local" refers to the coordinator itself. + // This data is sent from linked projects to the coordinator to inform it of each remote's local resolution state. + private final ResolvedIndexExpressions resolvedLocally; + // Remotely resolved index expressions, keyed by project alias. + // This is only populated by the coordinating node with the `resolvedLocally` data structure it receives + // back from the remotes. Used in the coordinating node for error checking, it's never sent over the wire. + // Keeping this distinction (between resolvedLocally and resolvedRemotely) further prevents project chaining + // and simplifies resolution logic, because the remoteExpressions in the resolvedLocally data structure are + // used to access data in `resolvedRemotely`. + private final transient Map resolvedRemotely; private final Map> fields; private final List failures; private final List indexResponses; @@ -51,25 +68,45 @@ public FieldCapabilitiesResponse( Map> fields, List failures ) { - this(indices, fields, Collections.emptyList(), failures, null); + this(indices, null, Collections.emptyMap(), fields, Collections.emptyList(), failures, null); } public FieldCapabilitiesResponse(String[] indices, Map> fields) { - this(indices, fields, Collections.emptyList(), Collections.emptyList(), null); + this(indices, null, Collections.emptyMap(), fields, Collections.emptyList(), Collections.emptyList(), null); + } + + public static FieldCapabilitiesResponse empty() { + return new FieldCapabilitiesResponse( + Strings.EMPTY_ARRAY, + null, + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyList(), + Collections.emptyList(), + null + ); } public FieldCapabilitiesResponse(List indexResponses, List failures) { - this(Strings.EMPTY_ARRAY, Collections.emptyMap(), indexResponses, failures, null); + this(Strings.EMPTY_ARRAY, null, Collections.emptyMap(), Collections.emptyMap(), indexResponses, failures, null); + } + + public static FieldCapabilitiesResponse.Builder builder() { + return new FieldCapabilitiesResponse.Builder(); } private FieldCapabilitiesResponse( String[] indices, + ResolvedIndexExpressions resolvedLocally, + Map resolvedRemotely, Map> fields, List indexResponses, List failures, TransportVersion minTransportVersion ) { this.fields = Objects.requireNonNull(fields); + this.resolvedLocally = resolvedLocally; + this.resolvedRemotely = Objects.requireNonNull(resolvedRemotely); this.indexResponses = Objects.requireNonNull(indexResponses); this.indices = indices; this.failures = failures; @@ -84,6 +121,14 @@ public FieldCapabilitiesResponse(StreamInput in) throws IOException { this.minTransportVersion = in.getTransportVersion().supports(MIN_TRANSPORT_VERSION) ? in.readOptional(TransportVersion::readVersion) : null; + if (in.getTransportVersion().supports(RESOLVED_FIELDS_CAPS)) { + this.resolvedLocally = in.readOptionalWriteable(ResolvedIndexExpressions::new); + } else { + this.resolvedLocally = null; + } + // when receiving a response we expect the resolved remotely to be empty. + // It's only non-empty on the coordinating node if the FC requests targets remotes. + this.resolvedRemotely = Collections.emptyMap(); } /** @@ -126,6 +171,20 @@ public List getIndexResponses() { return indexResponses; } + /** + * Locally resolved index expressions + */ + public ResolvedIndexExpressions getResolvedLocally() { + return resolvedLocally; + } + + /** + * Remotely resolved index expressions, non-empty only in the FC coordinator + */ + public Map getResolvedRemotely() { + return resolvedRemotely; + } + /** * Get the field capabilities per type for the provided {@code field}. */ @@ -145,7 +204,7 @@ public TransportVersion minTransportVersion() { * Build a new response replacing the {@link #minTransportVersion()}. */ public FieldCapabilitiesResponse withMinTransportVersion(TransportVersion newMin) { - return new FieldCapabilitiesResponse(indices, fields, indexResponses, failures, newMin); + return new FieldCapabilitiesResponse(indices, resolvedLocally, resolvedRemotely, fields, indexResponses, failures, newMin); } /** @@ -172,6 +231,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().supports(MIN_TRANSPORT_VERSION)) { out.writeOptional((Writer) (o, v) -> TransportVersion.writeVersion(v, o), minTransportVersion); } + if (out.getTransportVersion().supports(RESOLVED_FIELDS_CAPS)) { + out.writeOptionalWriteable(resolvedLocally); + } } private static void writeField(StreamOutput out, Map map) throws IOException { @@ -210,6 +272,8 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; FieldCapabilitiesResponse that = (FieldCapabilitiesResponse) o; return Arrays.equals(indices, that.indices) + && Objects.equals(resolvedLocally, that.resolvedLocally) + && Objects.equals(resolvedRemotely, that.resolvedRemotely) && Objects.equals(fields, that.fields) && Objects.equals(indexResponses, that.indexResponses) && Objects.equals(failures, that.failures) @@ -218,7 +282,9 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(fields, indexResponses, failures, minTransportVersion) * 31 + Arrays.hashCode(indices); + int result = Objects.hash(resolvedLocally, resolvedRemotely, fields, indexResponses, failures, minTransportVersion); + result = 31 * result + Arrays.hashCode(indices); + return result; } @Override @@ -226,12 +292,10 @@ public String toString() { return indexResponses.isEmpty() ? Strings.toString(this) : "FieldCapabilitiesResponse{unmerged}"; } - public static Builder builder() { - return new Builder(); - } - public static class Builder { private String[] indices = Strings.EMPTY_ARRAY; + private ResolvedIndexExpressions resolvedLocally; + private Map resolvedRemotely = Collections.emptyMap(); private Map> fields = Collections.emptyMap(); private List indexResponses = Collections.emptyList(); private List failures = Collections.emptyList(); @@ -244,6 +308,24 @@ public Builder withIndices(String[] indices) { return this; } + public Builder withResolved(ResolvedIndexExpressions resolvedLocally, Map resolvedRemotely) { + this.resolvedLocally = resolvedLocally; + this.resolvedRemotely = resolvedRemotely; + return this; + } + + public Builder withResolvedRemotelyBuilder(Map resolvedRemotelyBuilder) { + this.resolvedRemotely = resolvedRemotelyBuilder.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().build())); + return this; + } + + public Builder withResolvedLocally(ResolvedIndexExpressions resolvedLocally) { + this.resolvedLocally = resolvedLocally; + return this; + } + public Builder withFields(Map> fields) { this.fields = fields; return this; @@ -265,7 +347,15 @@ public Builder withMinTransportVersion(TransportVersion minTransportVersion) { } public FieldCapabilitiesResponse build() { - return new FieldCapabilitiesResponse(indices, fields, indexResponses, failures, minTransportVersion); + return new FieldCapabilitiesResponse( + indices, + resolvedLocally, + resolvedRemotely, + fields, + indexResponses, + failures, + minTransportVersion + ); } } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index ffe14bcc6af50..87f74c0704c53 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -21,15 +21,20 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.RemoteClusterActionType; +import org.elasticsearch.action.ResolvedIndexExpression; +import org.elasticsearch.action.ResolvedIndexExpressions; import org.elasticsearch.action.support.AbstractThreadedActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ChannelActionListener; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.RefCountingRunnable; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.cluster.ProjectState; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -71,6 +76,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -80,9 +86,11 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import static org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest.RESOLVED_FIELDS_CAPS; import static org.elasticsearch.action.search.TransportSearchHelper.checkCCSVersionCompatibility; public class TransportFieldCapabilitiesAction extends HandledTransportAction { + public static final String EXCLUSION = "-"; public static final String NAME = "indices:data/read/field_caps"; public static final ActionType TYPE = new ActionType<>(NAME); public static final RemoteClusterActionType REMOTE_TYPE = new RemoteClusterActionType<>( @@ -191,21 +199,67 @@ private void doExecuteForked( final Map remoteClusterIndices = transportService.getRemoteClusterService() .groupIndices(request.indicesOptions(), request.indices(), request.returnLocalAll()); final OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); - // in the case we have one or more remote indices but no local we don't expand to all local indices and just do remote indices - final String[] concreteIndices = localIndices != null - ? indexNameExpressionResolver.concreteIndexNames(projectState.metadata(), localIndices) - : Strings.EMPTY_ARRAY; - - if (concreteIndices.length == 0 && remoteClusterIndices.isEmpty()) { - listener.onResponse( - linkedRequestExecutor.wrapPrimary( - FieldCapabilitiesResponse.builder().withMinTransportVersion(minTransportVersion.get()).build() - ) - ); + + final String[] concreteLocalIndices; + final List resolvedLocallyList; + if (request.getResolvedIndexExpressions() != null) { + // in CPS the Security Action Filter would populate resolvedExpressions for the local project + // thus we can get the concreteLocalIndices based on the resolvedLocallyList + resolvedLocallyList = request.getResolvedIndexExpressions().expressions(); + concreteLocalIndices = resolvedLocallyList.stream() + .map(r -> r.localExpressions().indices()) + .flatMap(Set::stream) + .distinct() + .toArray(String[]::new); + } else { + // In CCS/Local only search we have to populate resolvedLocallyList one by one for each localIndices.indices() + // only if the request is includeResolvedTo() + resolvedLocallyList = new ArrayList<>(); + if (localIndices == null) { + // in the case we have one or more remote indices but no local we don't expand to all local indices + // in this case resolvedLocallyList will remain empty + concreteLocalIndices = Strings.EMPTY_ARRAY; + } else { + concreteLocalIndices = indexNameExpressionResolver.concreteIndexNames(projectState.metadata(), localIndices); + if (request.includeResolvedTo()) { + ProjectMetadata projectMetadata = projectState.metadata(); + IndicesOptions indicesOptions = localIndices.indicesOptions(); + String[] localIndexNames = localIndices.indices(); + if (localIndexNames.length == 0) { + // Empty indices array means match all + String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames(projectMetadata, indicesOptions); + resolvedLocallyList.add(createResolvedIndexExpression(Metadata.ALL, concreteIndexNames)); + } else if (false == IndexNameExpressionResolver.isNoneExpression(localIndexNames)) { + // if it's neither match all nor match none, but we want to include resolutions we loop for all the indicesNames + for (String localIndexName : localIndexNames) { + if (false == localIndexName.startsWith(EXCLUSION)) { + // we populate resolvedLocally iff is not an exclusion + String[] concreteIndexNames = indexNameExpressionResolver.concreteIndexNames( + projectMetadata, + indicesOptions, + localIndices.includeDataStreams(), + localIndexName + ); + resolvedLocallyList.add(createResolvedIndexExpression(localIndexName, concreteIndexNames)); + } + } + } + } + + } + } + + if (concreteLocalIndices.length == 0 && remoteClusterIndices.isEmpty()) { + FieldCapabilitiesResponse.Builder responseBuilder = FieldCapabilitiesResponse.builder(); + responseBuilder.withMinTransportVersion(minTransportVersion.get()); + if (request.includeResolvedTo()) { + responseBuilder.withResolvedLocally(new ResolvedIndexExpressions(resolvedLocallyList)); + } + listener.onResponse(linkedRequestExecutor.wrapPrimary(responseBuilder.build())); return; } - checkIndexBlocks(projectState, concreteIndices); + checkIndexBlocks(projectState, concreteLocalIndices); final FailureCollector indexFailures = new FailureCollector(); final Map indexResponses = new HashMap<>(); // This map is used to share the index response for indices which have the same index mapping hash to reduce the memory usage. @@ -216,6 +270,10 @@ private void doExecuteForked( indexResponses.clear(); indexMappingHashToResponses.clear(); }; + Map resolvedRemotely = new ConcurrentHashMap<>(); + for (String clusterAlias : remoteClusterIndices.keySet()) { + resolvedRemotely.put(clusterAlias, ResolvedIndexExpressions.builder()); + } final Consumer handleIndexResponse = resp -> { if (fieldCapTask.isCancelled()) { releaseResourcesOnCancel.run(); @@ -282,6 +340,8 @@ private void doExecuteForked( fieldCapTask, indexResponses, indexFailures, + resolvedLocallyList, + resolvedRemotely, minTransportVersion, listener.map(linkedRequestExecutor::wrapPrimary) ); @@ -297,7 +357,7 @@ private void doExecuteForked( request, localIndices, nowInMillis, - concreteIndices, + concreteLocalIndices, singleThreadedExecutor, handleIndexResponse, handleIndexFailure, @@ -312,6 +372,21 @@ private void doExecuteForked( OriginalIndices originalIndices = remoteIndices.getValue(); FieldCapabilitiesRequest remoteRequest = prepareRemoteRequest(clusterAlias, request, originalIndices, nowInMillis); ActionListener remoteListener = ActionListener.wrap(response -> { + + if (request.includeResolvedTo() && response.getResolvedLocally() != null) { + ResolvedIndexExpressions resolvedOnRemoteProject = response.getResolvedLocally(); + // for bwc we need to check that resolvedOnRemoteProject Exists in the response + if (resolvedOnRemoteProject != null) { + for (ResolvedIndexExpression remoteResolvedExpression : resolvedOnRemoteProject.expressions()) { + resolvedRemotely.computeIfPresent(clusterAlias, (k, v) -> { + v.addExpression(remoteResolvedExpression); + return v; + }); + } + } + + } + for (FieldCapabilitiesIndexResponse resp : response.getIndexResponses()) { String indexName = RemoteClusterAware.buildRemoteIndexName(clusterAlias, resp.getIndexName()); handleIndexResponse.accept( @@ -328,6 +403,21 @@ private void doExecuteForked( Exception ex = failure.getException(); for (String index : failure.getIndices()) { handleIndexFailure.accept(RemoteClusterAware.buildRemoteIndexName(clusterAlias, index), ex); + if (request.includeResolvedTo()) { + ResolvedIndexExpression err = new ResolvedIndexExpression( + index, + new ResolvedIndexExpression.LocalExpressions( + Set.of(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + null + ), + Set.of() + ); + resolvedRemotely.computeIfPresent(clusterAlias, (k, v) -> { + v.addExpression(err); + return v; + }); + } } } minTransportVersion.accumulateAndGet(response.minTransportVersion(), (lhs, rhs) -> { @@ -339,6 +429,21 @@ private void doExecuteForked( }, ex -> { for (String index : originalIndices.indices()) { handleIndexFailure.accept(RemoteClusterAware.buildRemoteIndexName(clusterAlias, index), ex); + if (request.includeResolvedTo()) { + ResolvedIndexExpression err = new ResolvedIndexExpression( + index, + new ResolvedIndexExpression.LocalExpressions( + Set.of(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + null + ), + Set.of() + ); + resolvedRemotely.computeIfPresent(clusterAlias, (k, v) -> { + v.addExpression(err); + return v; + }); + } } }); @@ -377,6 +482,20 @@ private void doExecuteForked( } } + private static ResolvedIndexExpression createResolvedIndexExpression(String original, String[] concreteIndexNames) { + boolean isWildcard = Regex.isSimpleMatchPattern(original); + // if it is a wildcard we consider it successful even if it didn't resolve to any concrete index + ResolvedIndexExpression.LocalIndexResolutionResult resolutionResult = concreteIndexNames.length > 0 || isWildcard + ? ResolvedIndexExpression.LocalIndexResolutionResult.SUCCESS + : ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE; + + return new ResolvedIndexExpression( + original, + new ResolvedIndexExpression.LocalExpressions(Set.of(concreteIndexNames), resolutionResult, null), + Collections.emptySet() + ); + } + private Executor buildSingleThreadedExecutor() { final ThrottledTaskRunner throttledTaskRunner = new ThrottledTaskRunner("field_caps", 1, searchCoordinationExecutor); return r -> throttledTaskRunner.enqueueTask(new ActionListener<>() { @@ -433,19 +552,27 @@ private static void mergeIndexResponses( CancellableTask task, Map indexResponses, FailureCollector indexFailures, + List resolvedLocallyList, + Map resolvedRemotely, AtomicReference minTransportVersion, ActionListener listener ) { + ResolvedIndexExpressions resolvedLocally = new ResolvedIndexExpressions(resolvedLocallyList); List failures = indexFailures.build(indexResponses.keySet()); if (indexResponses.isEmpty() == false) { if (request.isMergeResults()) { - ActionListener.completeWith(listener, () -> merge(indexResponses, task, request, failures, minTransportVersion)); + ActionListener.completeWith( + listener, + () -> merge(indexResponses, resolvedLocally, resolvedRemotely, task, request, failures, minTransportVersion) + ); } else { listener.onResponse( FieldCapabilitiesResponse.builder() .withIndexResponses(new ArrayList<>(indexResponses.values())) - .withFailures(failures) + .withResolvedLocally(resolvedLocally) + .withResolvedRemotelyBuilder(resolvedRemotely) .withMinTransportVersion(minTransportVersion.get()) + .withFailures(failures) .build() ); } @@ -460,7 +587,12 @@ private static void mergeIndexResponses( && ise.getCause() instanceof ElasticsearchTimeoutException )) { listener.onResponse( - FieldCapabilitiesResponse.builder().withFailures(failures).withMinTransportVersion(minTransportVersion.get()).build() + FieldCapabilitiesResponse.builder() + .withFailures(failures) + .withResolvedLocally(resolvedLocally) + .withResolvedRemotelyBuilder(resolvedRemotely) + .withMinTransportVersion(minTransportVersion.get()) + .build() ); } else { // throw back the first exception @@ -489,6 +621,7 @@ private static FieldCapabilitiesRequest prepareRemoteRequest( remoteRequest.indexFilter(request.indexFilter()); remoteRequest.nowInMillis(nowInMillis); remoteRequest.includeEmptyFields(request.includeEmptyFields()); + remoteRequest.includeResolvedTo(request.includeResolvedTo()); return remoteRequest; } @@ -500,6 +633,8 @@ private static boolean hasSameMappingHash(FieldCapabilitiesIndexResponse r1, Fie private static FieldCapabilitiesResponse merge( Map indexResponsesMap, + ResolvedIndexExpressions resolvedLocally, + Map resolvedRemotely, CancellableTask task, FieldCapabilitiesRequest request, List failures, @@ -533,6 +668,22 @@ private static FieldCapabilitiesResponse merge( collectFields(fieldsBuilder, fields, request.includeIndices()); } + List failedIndices = failures.stream().flatMap(f -> Arrays.stream(f.getIndices())).toList(); + List collect = resolvedLocally.expressions().stream().map(expression -> { + if (failedIndices.contains(expression.original())) { + return new ResolvedIndexExpression( + expression.original(), + new ResolvedIndexExpression.LocalExpressions( + Set.of(), + ResolvedIndexExpression.LocalIndexResolutionResult.CONCRETE_RESOURCE_NOT_VISIBLE, + null + ), + expression.remoteExpressions() + ); + } + return expression; + }).toList(); + // The merge method is only called on the primary coordinator for cross-cluster field caps, so we // log relevant "5xx" errors that occurred in this 2xx response to ensure they are only logged once. // These failures have already been deduplicated, before this method was called. @@ -544,12 +695,17 @@ private static FieldCapabilitiesResponse merge( ); } } - return FieldCapabilitiesResponse.builder() + + FieldCapabilitiesResponse.Builder responseBuilder = FieldCapabilitiesResponse.builder() .withIndices(indices) .withFields(Collections.unmodifiableMap(fields)) .withFailures(failures) - .withMinTransportVersion(minTransportVersion.get()) - .build(); + .withMinTransportVersion(minTransportVersion.get()); + if (request.includeResolvedTo() && minTransportVersion.get().supports(RESOLVED_FIELDS_CAPS)) { + // add resolution to response iff includeResolvedTo and all the nodes in the cluster supports it + responseBuilder.withResolvedLocally(new ResolvedIndexExpressions(collect)).withResolvedRemotelyBuilder(resolvedRemotely); + } + return responseBuilder.build(); } private static boolean shouldLogException(Exception e) { diff --git a/server/src/main/resources/transport/definitions/referable/resolved_fields_caps.csv b/server/src/main/resources/transport/definitions/referable/resolved_fields_caps.csv new file mode 100644 index 0000000000000..4629e277c153b --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/resolved_fields_caps.csv @@ -0,0 +1 @@ +9212000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index a70b776735734..ce5e1c85f99fd 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -add_sample_method_downsample_dlm,9211000 +resolved_fields_caps,9212000 diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java index ceb84e4b2a0d9..55c8b842d3db8 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java @@ -53,7 +53,7 @@ protected FieldCapabilitiesResponse createTestInstance() { var indexMode = randomFrom(IndexMode.values()); responses.add(new FieldCapabilitiesIndexResponse("index_" + i, null, fieldCaps, randomBoolean(), indexMode)); } - randomResponse = new FieldCapabilitiesResponse(responses, Collections.emptyList()); + randomResponse = FieldCapabilitiesResponse.builder().withIndexResponses(responses).build(); return randomResponse; } @@ -88,7 +88,7 @@ protected FieldCapabilitiesResponse mutateInstance(FieldCapabilitiesResponse res ); } } - return new FieldCapabilitiesResponse(null, mutatedResponses, Collections.emptyList()); + return FieldCapabilitiesResponse.builder().withFields(mutatedResponses).build(); } public void testFailureSerialization() throws IOException { @@ -144,7 +144,7 @@ public static FieldCapabilitiesResponse createResponseWithFailures() { failures.get(failures.size() - 1).addIndex(index); } } - return new FieldCapabilitiesResponse(indices, Collections.emptyMap(), failures); + return FieldCapabilitiesResponse.builder().withIndices(indices).withFailures(failures).build(); } private static FieldCapabilitiesResponse randomCCSResponse(List indexResponses) { @@ -154,7 +154,7 @@ private static FieldCapabilitiesResponse randomCCSResponse(List failureMap = List.of( new FieldCapabilitiesFailure(new String[] { "errorindex", "errorindex2" }, new IllegalArgumentException("test")) ); - return new FieldCapabilitiesResponse(new String[] { "index1", "index2", "index3", "index4" }, responses, failureMap); + return FieldCapabilitiesResponse.builder() + .withIndices(new String[] { "index1", "index2", "index3", "index4" }) + .withFields(responses) + .withFailures(failureMap) + .build(); } public void testChunking() { diff --git a/test/framework/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapsUtils.java b/test/framework/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapsUtils.java index 84c057d3b6a81..4a181d816451b 100644 --- a/test/framework/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapsUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapsUtils.java @@ -61,7 +61,12 @@ public static FieldCapabilitiesFailure parseFailure(XContentParser parser) throw .collect(Collectors.toMap(Tuple::v1, Tuple::v2)); List indices = a[1] == null ? Collections.emptyList() : (List) a[1]; List failures = a[2] == null ? Collections.emptyList() : (List) a[2]; - return new FieldCapabilitiesResponse(indices.toArray(String[]::new), responseMap, failures); + + return FieldCapabilitiesResponse.builder() + .withIndices(indices.toArray(String[]::new)) + .withFields(responseMap) + .withFailures(failures) + .build(); } ); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java index d14bd8c8c8196..549a5f5714b8b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/ClassificationTests.java @@ -427,7 +427,7 @@ public void testFieldCardinalityLimitsIsNonEmpty() { } public void testGetResultMappings_DependentVariableMappingIsAbsent() { - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse(new String[0], Collections.emptyMap()); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.empty(); expectThrows( ElasticsearchStatusException.class, () -> new Classification("foo").getResultMappings("results", fieldCapabilitiesResponse) @@ -435,10 +435,9 @@ public void testGetResultMappings_DependentVariableMappingIsAbsent() { } public void testGetResultMappings_DependentVariableMappingHasNoTypes() { - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse( - new String[0], - Collections.singletonMap("foo", Collections.emptyMap()) - ); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.builder() + .withFields(Collections.singletonMap("foo", Collections.emptyMap())) + .build(); expectThrows( ElasticsearchStatusException.class, () -> new Classification("foo").getResultMappings("results", fieldCapabilitiesResponse) @@ -459,10 +458,9 @@ public void testGetResultMappings_DependentVariableMappingIsPresent() { Map.of("type", "double") ) ); - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse( - new String[0], - Collections.singletonMap("foo", Collections.singletonMap("dummy", createFieldCapabilities("foo", "dummy"))) - ); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.builder() + .withFields(Collections.singletonMap("foo", Collections.singletonMap("dummy", createFieldCapabilities("foo", "dummy")))) + .build(); Map resultMappings = new Classification("foo").getResultMappings("results", fieldCapabilitiesResponse); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index e31bfdd5364b2..75fc686cf5d0d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -3317,10 +3317,11 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithCa } public void testResolveDenseVector() { - FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( - List.of(fieldCapabilitiesIndexResponse("foo", Map.of("v", new IndexFieldCapabilitiesBuilder("v", "dense_vector").build()))), - List.of() - ); + FieldCapabilitiesResponse caps = FieldCapabilitiesResponse.builder() + .withIndexResponses( + List.of(fieldCapabilitiesIndexResponse("foo", Map.of("v", new IndexFieldCapabilitiesBuilder("v", "dense_vector").build()))) + ) + .build(); { IndexResolution resolution = IndexResolver.mergedMappings( "foo", @@ -3342,15 +3343,16 @@ public void testResolveDenseVector() { } public void testResolveAggregateMetricDouble() { - FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse( - "foo", - Map.of("v", new IndexFieldCapabilitiesBuilder("v", "aggregate_metric_double").build()) + FieldCapabilitiesResponse caps = FieldCapabilitiesResponse.builder() + .withIndexResponses( + List.of( + fieldCapabilitiesIndexResponse( + "foo", + Map.of("v", new IndexFieldCapabilitiesBuilder("v", "aggregate_metric_double").build()) + ) ) - ), - List.of() - ); + ) + .build(); { IndexResolution resolution = IndexResolver.mergedMappings( "foo", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java index 8ea5ecf231407..54e100e962c4b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java @@ -506,9 +506,9 @@ protected void fieldCaps.put(e.getKey(), f); } var indexResponse = new FieldCapabilitiesIndexResponse(alias, null, fieldCaps, true, IndexMode.STANDARD); - response = new FieldCapabilitiesResponse(List.of(indexResponse), List.of()); + response = FieldCapabilitiesResponse.builder().withIndexResponses(List.of(indexResponse)).build(); } else { - response = new FieldCapabilitiesResponse(List.of(), List.of()); + response = FieldCapabilitiesResponse.empty(); } threadPool().executor(ThreadPool.Names.SEARCH_COORDINATION) .execute(ActionRunnable.supply(listener, () -> (Response) new EsqlResolveFieldsResponse(response))); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java index c201f544372db..eda583457243c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java @@ -52,7 +52,7 @@ private void resolve(String esTypeName, TimeSeriesParams.MetricType metricType, ) ); - FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse(idxResponses, List.of()); + FieldCapabilitiesResponse caps = FieldCapabilitiesResponse.builder().withIndexResponses(idxResponses).build(); // IndexResolver uses EsqlDataTypeRegistry directly IndexResolution resolution = IndexResolver.mergedMappings( "idx-*", diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java index b7646f430726a..a3ebdadb20db8 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndexTests.java @@ -285,14 +285,20 @@ private Map testCreateDestinationIndex(DataFrameAnalysis analysi doAnswer(callListenerOnResponse(getMappingsResponse)).when(client) .execute(eq(GetMappingsAction.INSTANCE), getMappingsRequestCaptor.capture(), any()); - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse(new String[0], new HashMap<>() { - { - put(NUMERICAL_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(OUTER_FIELD + "." + INNER_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(ALIAS_TO_NUMERICAL_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(ALIAS_TO_NESTED_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - } - }); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.builder() + .withFields( + Map.of( + NUMERICAL_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + OUTER_FIELD + "." + INNER_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + ALIAS_TO_NUMERICAL_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + ALIAS_TO_NESTED_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")) + ) + ) + .build(); doAnswer(callListenerOnResponse(fieldCapabilitiesResponse)).when(client) .execute(eq(TransportFieldCapabilitiesAction.TYPE), fieldCapabilitiesRequestCaptor.capture(), any()); @@ -615,14 +621,20 @@ private Map testUpdateMappingsToDestIndex(DataFrameAnalysis anal doAnswer(callListenerOnResponse(AcknowledgedResponse.TRUE)).when(client) .execute(eq(TransportPutMappingAction.TYPE), putMappingRequestCaptor.capture(), any()); - FieldCapabilitiesResponse fieldCapabilitiesResponse = new FieldCapabilitiesResponse(new String[0], new HashMap<>() { - { - put(NUMERICAL_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(OUTER_FIELD + "." + INNER_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(ALIAS_TO_NUMERICAL_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - put(ALIAS_TO_NESTED_FIELD, singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer"))); - } - }); + FieldCapabilitiesResponse fieldCapabilitiesResponse = FieldCapabilitiesResponse.builder() + .withFields( + Map.of( + NUMERICAL_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + OUTER_FIELD + "." + INNER_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + ALIAS_TO_NUMERICAL_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")), + ALIAS_TO_NESTED_FIELD, + singletonMap("integer", createFieldCapabilities(NUMERICAL_FIELD, "integer")) + ) + ) + .build(); doAnswer(callListenerOnResponse(fieldCapabilitiesResponse)).when(client) .execute(eq(TransportFieldCapabilitiesAction.TYPE), fieldCapabilitiesRequestCaptor.capture(), any()); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java index f028f39c6069f..4b9c25d71f452 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/extractor/ExtractedFieldsDetectorTests.java @@ -1724,7 +1724,7 @@ private MockFieldCapsResponseBuilder addField(String field, boolean isMetadataFi } private FieldCapabilitiesResponse build() { - return new FieldCapabilitiesResponse(new String[] { "test" }, fieldCaps); + return FieldCapabilitiesResponse.builder().withIndices(new String[] { "test" }).withFields(fieldCaps).build(); } } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index 0610721c04537..27a4adb910bf0 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -600,7 +600,7 @@ private static IndexResolution mergedMappings( return IndexResolver.mergedMappings( SqlDataTypeRegistry.INSTANCE, indexPattern, - new FieldCapabilitiesResponse(indexNames, fieldCaps) + FieldCapabilitiesResponse.builder().withIndices(indexNames).withFields(fieldCaps).build() ); } @@ -612,7 +612,7 @@ private static List separateMappings( return IndexResolver.separateMappings( SqlDataTypeRegistry.INSTANCE, javaRegex, - new FieldCapabilitiesResponse(indexNames, fieldCaps), + FieldCapabilitiesResponse.builder().withIndices(indexNames).withFields(fieldCaps).build(), null ); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java index 18af78d704646..49b58f0dde2b4 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/common/DocumentConversionUtilsTests.java @@ -87,16 +87,18 @@ public void testRemoveInternalFields() { } public void testExtractFieldMappings() { - FieldCapabilitiesResponse response = new FieldCapabilitiesResponse( - new String[] { "some-index" }, - Map.ofEntries( - entry("field-1", Map.of("keyword", createFieldCapabilities("field-1", "keyword"))), - entry( - "field-2", - Map.of("long", createFieldCapabilities("field-2", "long"), "keyword", createFieldCapabilities("field-2", "keyword")) + FieldCapabilitiesResponse response = FieldCapabilitiesResponse.builder() + .withIndices(new String[] { "some-index" }) + .withFields( + Map.ofEntries( + entry("field-1", Map.of("keyword", createFieldCapabilities("field-1", "keyword"))), + entry( + "field-2", + Map.of("long", createFieldCapabilities("field-2", "long"), "keyword", createFieldCapabilities("field-2", "keyword")) + ) ) ) - ); + .build(); assertThat( DocumentConversionUtils.extractFieldMappings(response), diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java index d65428a3912de..580e3fb2bdbfa 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/SchemaUtilTests.java @@ -287,7 +287,10 @@ protected void responseMap.put(field, singletonMap(field, createFieldCapabilities(field, type))); } - final FieldCapabilitiesResponse response = new FieldCapabilitiesResponse(fieldCapsRequest.indices(), responseMap); + final FieldCapabilitiesResponse response = FieldCapabilitiesResponse.builder() + .withIndices(fieldCapsRequest.indices()) + .withFields(responseMap) + .build(); listener.onResponse((Response) response); return; } From 490a2d52f9f8e932545af4f8c0e45176694a8a2b Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Mon, 3 Nov 2025 09:55:28 -0600 Subject: [PATCH 29/32] Copy ES819TSDBDocValuesConsumer into tests for version 0 bwc tests (#137404) In #137139, ES819DocValuesFormat version will increment from 0 to 1. For bwc tests between version 0 and version 1, we need a copy of version 0. So copy the existing ES819TSDBDocValuesConsumer to ES819TSDBDocValuesConsumerVersion0 and add TestES819TSDBDocValuesFormatVersion0 which uses this consumer. Since ES819TSDBDocValuesConsumer and ES819TSDBDocValuesConsumerVersion0 are currently identical, this does not test anything yet, but will once #137139 increments the version. --- .../tsdb/es819/ES819TSDBDocValuesFormat.java | 2 +- .../codec/tsdb/TsdbDocValueBwcTests.java | 68 +- .../ES819TSDBDocValuesConsumerVersion0.java | 1004 +++++++++++++++++ .../es819/ES819TSDBDocValuesFormatTests.java | 23 + 4 files changed, 1073 insertions(+), 24 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesConsumerVersion0.java diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormat.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormat.java index fbdef488b8318..fb43ae176dedc 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormat.java @@ -118,7 +118,7 @@ private static boolean getOptimizedMergeEnabledDefault() { final int skipIndexIntervalSize; final int minDocsPerOrdinalForRangeEncoding; - private final boolean enableOptimizedMerge; + final boolean enableOptimizedMerge; /** Default constructor. */ public ES819TSDBDocValuesFormat() { diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/TsdbDocValueBwcTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/TsdbDocValueBwcTests.java index d2c8aae601977..0bee87b560a72 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/tsdb/TsdbDocValueBwcTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/TsdbDocValueBwcTests.java @@ -52,6 +52,7 @@ import java.util.Map; import java.util.function.IntSupplier; +import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormatTests.TestES819TSDBDocValuesFormatVersion0; import static org.hamcrest.Matchers.equalTo; public class TsdbDocValueBwcTests extends ESTestCase { @@ -62,6 +63,13 @@ public void testMixedIndex() throws Exception { testMixedIndex(oldCodec, newCodec); } + // TODO update Current to Version1 once version is incremented + public void testMixedIndexDocValueVersion0ToCurrent() throws Exception { + var oldCodec = TestUtil.alwaysDocValuesFormat(new TestES819TSDBDocValuesFormatVersion0()); + var newCodec = TestUtil.alwaysDocValuesFormat(new ES819TSDBDocValuesFormat()); + testMixedIndex(oldCodec, newCodec, this::assertVersion819, this::assertVersion819); + } + public void testMixedIndex816To900Lucene101() throws Exception { var oldCodec = new Elasticsearch816Codec() { @@ -84,8 +92,37 @@ public DocValuesFormat getDocValuesFormatForField(String field) { testMixedIndex(oldCodec, newCodec); } + void assertFieldInfoDocValuesFormat(DirectoryReader reader, String expectedSuffix, String expectedFormat) throws IOException, + NoSuchFieldException, IllegalAccessException { + // Assert per field format field info attributes: + // (XPerFieldDocValuesFormat must produce the same attributes as PerFieldDocValuesFormat for BWC. + // Otherwise, doc values fields may disappear) + for (var leaf : reader.leaves()) { + for (var fieldInfo : leaf.reader().getFieldInfos()) { + assertThat(fieldInfo.attributes(), Matchers.aMapWithSize(2)); + assertThat(fieldInfo.attributes(), Matchers.hasEntry("PerFieldDocValuesFormat.suffix", expectedSuffix)); + assertThat(fieldInfo.attributes(), Matchers.hasEntry("PerFieldDocValuesFormat.format", expectedFormat)); + } + } + } + + void assertVersion87(DirectoryReader reader) throws IOException, NoSuchFieldException, IllegalAccessException { + assert87DocValuesFormatVersion(reader); + assertFieldInfoDocValuesFormat(reader, "0", "ES87TSDB"); + } + + void assertVersion819(DirectoryReader reader) throws IOException, NoSuchFieldException, ClassNotFoundException, IllegalAccessException { + assert819DocValuesFormatVersion(reader); + assertFieldInfoDocValuesFormat(reader, "0", "ES819TSDB"); + } + void testMixedIndex(Codec oldCodec, Codec newCodec) throws IOException, NoSuchFieldException, IllegalAccessException, ClassNotFoundException { + testMixedIndex(oldCodec, newCodec, this::assertVersion87, this::assertVersion819); + } + + void testMixedIndex(Codec oldCodec, Codec newCodec, VersionAssert assertOldVersion, VersionAssert assertNewVersion) throws IOException, + NoSuchFieldException, IllegalAccessException, ClassNotFoundException { String timestampField = "@timestamp"; String hostnameField = "host.name"; long baseTimestamp = 1704067200000L; @@ -136,17 +173,7 @@ void testMixedIndex(Codec oldCodec, Codec newCodec) throws IOException, NoSuchFi } // Check documents before force merge: try (var reader = DirectoryReader.open(dir)) { - assertOldDocValuesFormatVersion(reader); - // Assert per field format field info attributes: - // (XPerFieldDocValuesFormat must produce the same attributes as PerFieldDocValuesFormat for BWC. - // Otherwise, doc values fields may disappear) - for (var leaf : reader.leaves()) { - for (var fieldInfo : leaf.reader().getFieldInfos()) { - assertThat(fieldInfo.attributes(), Matchers.aMapWithSize(2)); - assertThat(fieldInfo.attributes(), Matchers.hasEntry("PerFieldDocValuesFormat.suffix", "0")); - assertThat(fieldInfo.attributes(), Matchers.hasEntry("PerFieldDocValuesFormat.format", "ES87TSDB")); - } - } + assertOldVersion.run(reader); var hostNameDV = MultiDocValues.getSortedValues(reader, hostnameField); assertNotNull(hostNameDV); @@ -205,17 +232,9 @@ void testMixedIndex(Codec oldCodec, Codec newCodec) throws IOException, NoSuchFi try (var reader = DirectoryReader.open(iw)) { assertEquals(1, reader.leaves().size()); assertEquals(numDocs, reader.maxDoc()); - assertNewDocValuesFormatVersion(reader); - var leaf = reader.leaves().get(0).reader(); - // Assert per field format field info attributes: - // (XPerFieldDocValuesFormat must produce the same attributes as PerFieldDocValuesFormat for BWC. - // Otherwise, doc values fields may disappear) - for (var fieldInfo : leaf.getFieldInfos()) { - assertThat(fieldInfo.attributes(), Matchers.aMapWithSize(2)); - assertThat(fieldInfo.attributes(), Matchers.hasEntry("PerFieldDocValuesFormat.suffix", "0")); - assertThat(fieldInfo.attributes(), Matchers.hasEntry("PerFieldDocValuesFormat.format", "ES819TSDB")); - } + assertNewVersion.run(reader); + var leaf = reader.leaves().get(0).reader(); var hostNameDV = leaf.getSortedDocValues(hostnameField); assertNotNull(hostNameDV); var timestampDV = DocValues.unwrapSingleton(leaf.getSortedNumericDocValues(timestampField)); @@ -385,7 +404,7 @@ private IndexWriterConfig getTimeSeriesIndexWriterConfig(String hostnameField, S // A hacky way to figure out whether doc values format is written in what version. Need to use reflection, because // PerFieldDocValuesFormat hides the doc values formats it wraps. - private void assertOldDocValuesFormatVersion(DirectoryReader reader) throws NoSuchFieldException, IllegalAccessException, IOException { + private void assert87DocValuesFormatVersion(DirectoryReader reader) throws NoSuchFieldException, IllegalAccessException, IOException { if (System.getSecurityManager() != null) { // With jvm version 24 entitlements are used and security manager is nog longer used. // Making this assertion work with security manager requires granting the entire test codebase privileges to use @@ -406,7 +425,7 @@ private void assertOldDocValuesFormatVersion(DirectoryReader reader) throws NoSu } } - private void assertNewDocValuesFormatVersion(DirectoryReader reader) throws NoSuchFieldException, IllegalAccessException, IOException, + private void assert819DocValuesFormatVersion(DirectoryReader reader) throws NoSuchFieldException, IllegalAccessException, IOException, ClassNotFoundException { for (var leafReaderContext : reader.leaves()) { @@ -451,4 +470,7 @@ private static Field getFormatsFieldFromPerFieldFieldsReader(Class c) throws return field; } + interface VersionAssert { + void run(DirectoryReader reader) throws IOException, NoSuchFieldException, IllegalAccessException, ClassNotFoundException; + } } diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesConsumerVersion0.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesConsumerVersion0.java new file mode 100644 index 0000000000000..df669f007db2b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesConsumerVersion0.java @@ -0,0 +1,1004 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.codec.tsdb.es819; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.codecs.lucene90.IndexedDISI; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipIndexType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.MergeState; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.SortedSetSelector; +import org.apache.lucene.store.ByteArrayDataOutput; +import org.apache.lucene.store.ByteBuffersDataOutput; +import org.apache.lucene.store.ByteBuffersIndexOutput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.LongsRef; +import org.apache.lucene.util.StringHelper; +import org.apache.lucene.util.compress.LZ4; +import org.apache.lucene.util.packed.DirectMonotonicWriter; +import org.apache.lucene.util.packed.PackedInts; +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.index.codec.tsdb.TSDBDocValuesEncoder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.elasticsearch.index.codec.tsdb.es819.DocValuesConsumerUtil.compatibleWithOptimizedMerge; +import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; +import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.SKIP_INDEX_LEVEL_SHIFT; +import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.SKIP_INDEX_MAX_LEVEL; +import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.SORTED_SET; + +final class ES819TSDBDocValuesConsumerVersion0 extends XDocValuesConsumer { + + final Directory dir; + final IOContext context; + IndexOutput data, meta; + final int maxDoc; + private byte[] termsDictBuffer; + private final int skipIndexIntervalSize; + private final int minDocsPerOrdinalForOrdinalRangeEncoding; + final boolean enableOptimizedMerge; + private final int primarySortFieldNumber; + + ES819TSDBDocValuesConsumerVersion0( + SegmentWriteState state, + int skipIndexIntervalSize, + int minDocsPerOrdinalForOrdinalRangeEncoding, + boolean enableOptimizedMerge, + String dataCodec, + String dataExtension, + String metaCodec, + String metaExtension + ) throws IOException { + this.termsDictBuffer = new byte[1 << 14]; + this.dir = state.directory; + this.minDocsPerOrdinalForOrdinalRangeEncoding = minDocsPerOrdinalForOrdinalRangeEncoding; + this.primarySortFieldNumber = ES819TSDBDocValuesProducer.primarySortFieldNumber(state.segmentInfo, state.fieldInfos); + this.context = state.context; + boolean success = false; + try { + final String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension); + data = state.directory.createOutput(dataName, state.context); + CodecUtil.writeIndexHeader( + data, + dataCodec, + ES819TSDBDocValuesFormat.VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix + ); + String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension); + meta = state.directory.createOutput(metaName, state.context); + CodecUtil.writeIndexHeader( + meta, + metaCodec, + ES819TSDBDocValuesFormat.VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix + ); + maxDoc = state.segmentInfo.maxDoc(); + this.skipIndexIntervalSize = skipIndexIntervalSize; + this.enableOptimizedMerge = enableOptimizedMerge; + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } + } + + @Override + public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + meta.writeInt(field.number); + meta.writeByte(ES819TSDBDocValuesFormat.NUMERIC); + var producer = new TsdbDocValuesProducer(valuesProducer) { + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { + return DocValues.singleton(valuesProducer.getNumeric(field)); + } + }; + if (field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) { + writeSkipIndex(field, producer); + } + + writeField(field, producer, -1, null); + } + + private boolean shouldEncodeOrdinalRange(FieldInfo field, long maxOrd, int numDocsWithValue, long numValues) { + return maxDoc > 1 + && field.number == primarySortFieldNumber + && numDocsWithValue == numValues // Only single valued fields can be supported with range encoded ordinals format + && (numDocsWithValue / maxOrd) >= minDocsPerOrdinalForOrdinalRangeEncoding; + } + + private long[] writeField(FieldInfo field, TsdbDocValuesProducer valuesProducer, long maxOrd, OffsetsAccumulator offsetsAccumulator) + throws IOException { + int numDocsWithValue = 0; + long numValues = 0; + + SortedNumericDocValues values; + if (valuesProducer.mergeStats.supported()) { + numDocsWithValue = valuesProducer.mergeStats.sumNumDocsWithField(); + numValues = valuesProducer.mergeStats.sumNumValues(); + } else { + values = valuesProducer.getSortedNumeric(field); + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + numDocsWithValue++; + final int count = values.docValueCount(); + numValues += count; + } + } + + meta.writeLong(numValues); + meta.writeInt(numDocsWithValue); + + DISIAccumulator disiAccumulator = null; + try { + if (numValues > 0) { + assert numDocsWithValue > 0; + final ByteBuffersDataOutput indexOut = new ByteBuffersDataOutput(); + DirectMonotonicWriter indexWriter = null; + + final long valuesDataOffset = data.getFilePointer(); + if (maxOrd == 1) { + // Special case for maxOrd of 1, signal -1 that no blocks will be written + meta.writeInt(-1); + } else if (shouldEncodeOrdinalRange(field, maxOrd, numDocsWithValue, numValues)) { + assert offsetsAccumulator == null; + // When a field is sorted, use ordinal range encode for long runs of the same ordinal. + meta.writeInt(-2); + meta.writeVInt(Math.toIntExact(maxOrd)); + meta.writeByte((byte) ES819TSDBDocValuesFormat.ORDINAL_RANGE_ENCODING_BLOCK_SHIFT); + values = valuesProducer.getSortedNumeric(field); + if (valuesProducer.mergeStats.supported() && numDocsWithValue < maxDoc) { + disiAccumulator = new DISIAccumulator(dir, context, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER); + } + DirectMonotonicWriter startDocs = DirectMonotonicWriter.getInstance( + meta, + data, + maxOrd + 1, + ES819TSDBDocValuesFormat.ORDINAL_RANGE_ENCODING_BLOCK_SHIFT + ); + long lastOrd = 0; + startDocs.add(0); + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + if (disiAccumulator != null) { + disiAccumulator.addDocId(doc); + } + final long nextOrd = values.nextValue(); + if (nextOrd != lastOrd) { + lastOrd = nextOrd; + startDocs.add(doc); + } + } + startDocs.add(maxDoc); + startDocs.finish(); + } else { + indexWriter = DirectMonotonicWriter.getInstance( + meta, + new ByteBuffersIndexOutput(indexOut, "temp-dv-index", "temp-dv-index"), + 1L + ((numValues - 1) >>> ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SHIFT), + ES819TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT + ); + meta.writeInt(DIRECT_MONOTONIC_BLOCK_SHIFT); + final long[] buffer = new long[ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE]; + int bufferSize = 0; + final TSDBDocValuesEncoder encoder = new TSDBDocValuesEncoder(ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE); + values = valuesProducer.getSortedNumeric(field); + final int bitsPerOrd = maxOrd >= 0 ? PackedInts.bitsRequired(maxOrd - 1) : -1; + if (valuesProducer.mergeStats.supported() && numDocsWithValue < maxDoc) { + disiAccumulator = new DISIAccumulator(dir, context, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER); + } + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + if (disiAccumulator != null) { + disiAccumulator.addDocId(doc); + } + final int count = values.docValueCount(); + if (offsetsAccumulator != null) { + offsetsAccumulator.addDoc(count); + } + for (int i = 0; i < count; ++i) { + buffer[bufferSize++] = values.nextValue(); + if (bufferSize == ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE) { + indexWriter.add(data.getFilePointer() - valuesDataOffset); + if (maxOrd >= 0) { + encoder.encodeOrdinals(buffer, data, bitsPerOrd); + } else { + encoder.encode(buffer, data); + } + bufferSize = 0; + } + } + } + if (bufferSize > 0) { + indexWriter.add(data.getFilePointer() - valuesDataOffset); + // Fill unused slots in the block with zeroes rather than junk + Arrays.fill(buffer, bufferSize, ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE, 0L); + if (maxOrd >= 0) { + encoder.encodeOrdinals(buffer, data, bitsPerOrd); + } else { + encoder.encode(buffer, data); + } + } + } + + final long valuesDataLength = data.getFilePointer() - valuesDataOffset; + if (indexWriter != null) { + indexWriter.finish(); + } + final long indexDataOffset = data.getFilePointer(); + data.copyBytes(indexOut.toDataInput(), indexOut.size()); + meta.writeLong(indexDataOffset); + meta.writeLong(data.getFilePointer() - indexDataOffset); + + meta.writeLong(valuesDataOffset); + meta.writeLong(valuesDataLength); + } + + if (numDocsWithValue == 0) { // meta[-2, 0]: No documents with values + meta.writeLong(-2); // docsWithFieldOffset + meta.writeLong(0L); // docsWithFieldLength + meta.writeShort((short) -1); // jumpTableEntryCount + meta.writeByte((byte) -1); // denseRankPower + } else if (numDocsWithValue == maxDoc) { // meta[-1, 0]: All documents have values + meta.writeLong(-1); // docsWithFieldOffset + meta.writeLong(0L); // docsWithFieldLength + meta.writeShort((short) -1); // jumpTableEntryCount + meta.writeByte((byte) -1); // denseRankPower + } else { // meta[data.offset, data.length]: IndexedDISI structure for documents with values + long offset = data.getFilePointer(); + meta.writeLong(offset); // docsWithFieldOffset + final short jumpTableEntryCount; + if (maxOrd != 1 && disiAccumulator != null) { + jumpTableEntryCount = disiAccumulator.build(data); + } else { + values = valuesProducer.getSortedNumeric(field); + jumpTableEntryCount = IndexedDISI.writeBitSet(values, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER); + } + meta.writeLong(data.getFilePointer() - offset); // docsWithFieldLength + meta.writeShort(jumpTableEntryCount); + meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER); + } + } finally { + IOUtils.close(disiAccumulator); + } + + return new long[] { numDocsWithValue, numValues }; + } + + @Override + public void mergeNumericField(FieldInfo mergeFieldInfo, MergeState mergeState) throws IOException { + var result = compatibleWithOptimizedMerge(enableOptimizedMerge, mergeState, mergeFieldInfo); + if (result.supported()) { + mergeNumericField(result, mergeFieldInfo, mergeState); + } else { + super.mergeNumericField(mergeFieldInfo, mergeState); + } + } + + @Override + public void mergeBinaryField(FieldInfo mergeFieldInfo, MergeState mergeState) throws IOException { + var result = compatibleWithOptimizedMerge(enableOptimizedMerge, mergeState, mergeFieldInfo); + if (result.supported()) { + mergeBinaryField(result, mergeFieldInfo, mergeState); + } else { + super.mergeBinaryField(mergeFieldInfo, mergeState); + } + } + + @Override + public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + meta.writeInt(field.number); + meta.writeByte(ES819TSDBDocValuesFormat.BINARY); + + if (valuesProducer instanceof TsdbDocValuesProducer tsdbValuesProducer && tsdbValuesProducer.mergeStats.supported()) { + final int numDocsWithField = tsdbValuesProducer.mergeStats.sumNumDocsWithField(); + final int minLength = tsdbValuesProducer.mergeStats.minLength(); + final int maxLength = tsdbValuesProducer.mergeStats.maxLength(); + + assert numDocsWithField <= maxDoc; + + BinaryDocValues values = valuesProducer.getBinary(field); + long start = data.getFilePointer(); + meta.writeLong(start); // dataOffset + + OffsetsAccumulator offsetsAccumulator = null; + DISIAccumulator disiAccumulator = null; + try { + if (numDocsWithField > 0 && numDocsWithField < maxDoc) { + disiAccumulator = new DISIAccumulator(dir, context, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER); + } + + assert maxLength >= minLength; + if (maxLength > minLength) { + offsetsAccumulator = new OffsetsAccumulator(dir, context, data, numDocsWithField); + } + + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + BytesRef v = values.binaryValue(); + data.writeBytes(v.bytes, v.offset, v.length); + if (disiAccumulator != null) { + disiAccumulator.addDocId(doc); + } + if (offsetsAccumulator != null) { + offsetsAccumulator.addDoc(v.length); + } + } + meta.writeLong(data.getFilePointer() - start); // dataLength + + if (numDocsWithField == 0) { + meta.writeLong(-2); // docsWithFieldOffset + meta.writeLong(0L); // docsWithFieldLength + meta.writeShort((short) -1); // jumpTableEntryCount + meta.writeByte((byte) -1); // denseRankPower + } else if (numDocsWithField == maxDoc) { + meta.writeLong(-1); // docsWithFieldOffset + meta.writeLong(0L); // docsWithFieldLength + meta.writeShort((short) -1); // jumpTableEntryCount + meta.writeByte((byte) -1); // denseRankPower + } else { + long offset = data.getFilePointer(); + meta.writeLong(offset); // docsWithFieldOffset + final short jumpTableEntryCount = disiAccumulator.build(data); + meta.writeLong(data.getFilePointer() - offset); // docsWithFieldLength + meta.writeShort(jumpTableEntryCount); + meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER); + } + + meta.writeInt(numDocsWithField); + meta.writeInt(minLength); + meta.writeInt(maxLength); + if (offsetsAccumulator != null) { + offsetsAccumulator.build(meta, data); + } + } finally { + IOUtils.close(disiAccumulator, offsetsAccumulator); + } + } else { + BinaryDocValues values = valuesProducer.getBinary(field); + long start = data.getFilePointer(); + meta.writeLong(start); // dataOffset + int numDocsWithField = 0; + int minLength = Integer.MAX_VALUE; + int maxLength = 0; + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + numDocsWithField++; + BytesRef v = values.binaryValue(); + int length = v.length; + data.writeBytes(v.bytes, v.offset, v.length); + minLength = Math.min(length, minLength); + maxLength = Math.max(length, maxLength); + } + assert numDocsWithField <= maxDoc; + meta.writeLong(data.getFilePointer() - start); // dataLength + + if (numDocsWithField == 0) { + meta.writeLong(-2); // docsWithFieldOffset + meta.writeLong(0L); // docsWithFieldLength + meta.writeShort((short) -1); // jumpTableEntryCount + meta.writeByte((byte) -1); // denseRankPower + } else if (numDocsWithField == maxDoc) { + meta.writeLong(-1); // docsWithFieldOffset + meta.writeLong(0L); // docsWithFieldLength + meta.writeShort((short) -1); // jumpTableEntryCount + meta.writeByte((byte) -1); // denseRankPower + } else { + long offset = data.getFilePointer(); + meta.writeLong(offset); // docsWithFieldOffset + values = valuesProducer.getBinary(field); + final short jumpTableEntryCount = IndexedDISI.writeBitSet(values, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER); + meta.writeLong(data.getFilePointer() - offset); // docsWithFieldLength + meta.writeShort(jumpTableEntryCount); + meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER); + } + + meta.writeInt(numDocsWithField); + meta.writeInt(minLength); + meta.writeInt(maxLength); + if (maxLength > minLength) { + start = data.getFilePointer(); + meta.writeLong(start); + meta.writeVInt(ES819TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT); + + final DirectMonotonicWriter writer = DirectMonotonicWriter.getInstance( + meta, + data, + numDocsWithField + 1, + ES819TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT + ); + long addr = 0; + writer.add(addr); + values = valuesProducer.getBinary(field); + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + addr += values.binaryValue().length; + writer.add(addr); + } + writer.finish(); + meta.writeLong(data.getFilePointer() - start); + } + } + } + + @Override + public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + meta.writeInt(field.number); + meta.writeByte(ES819TSDBDocValuesFormat.SORTED); + doAddSortedField(field, valuesProducer, false); + } + + @Override + public void mergeSortedField(FieldInfo mergeFieldInfo, MergeState mergeState) throws IOException { + var result = compatibleWithOptimizedMerge(enableOptimizedMerge, mergeState, mergeFieldInfo); + if (result.supported()) { + mergeSortedField(result, mergeFieldInfo, mergeState); + } else { + super.mergeSortedField(mergeFieldInfo, mergeState); + } + } + + private void doAddSortedField(FieldInfo field, DocValuesProducer valuesProducer, boolean addTypeByte) throws IOException { + var producer = new TsdbDocValuesProducer(valuesProducer) { + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { + SortedDocValues sorted = valuesProducer.getSorted(field); + NumericDocValues sortedOrds = new NumericDocValues() { + @Override + public long longValue() throws IOException { + return sorted.ordValue(); + } + + @Override + public boolean advanceExact(int target) throws IOException { + return sorted.advanceExact(target); + } + + @Override + public int docID() { + return sorted.docID(); + } + + @Override + public int nextDoc() throws IOException { + return sorted.nextDoc(); + } + + @Override + public int advance(int target) throws IOException { + return sorted.advance(target); + } + + @Override + public long cost() { + return sorted.cost(); + } + }; + return DocValues.singleton(sortedOrds); + } + }; + if (field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) { + writeSkipIndex(field, producer); + } + if (addTypeByte) { + meta.writeByte((byte) 0); // multiValued (0 = singleValued) + } + SortedDocValues sorted = valuesProducer.getSorted(field); + int maxOrd = sorted.getValueCount(); + writeField(field, producer, maxOrd, null); + addTermsDict(DocValues.singleton(valuesProducer.getSorted(field))); + } + + private void addTermsDict(SortedSetDocValues values) throws IOException { + final long size = values.getValueCount(); + meta.writeVLong(size); + + int blockMask = ES819TSDBDocValuesFormat.TERMS_DICT_BLOCK_LZ4_MASK; + int shift = ES819TSDBDocValuesFormat.TERMS_DICT_BLOCK_LZ4_SHIFT; + + meta.writeInt(DIRECT_MONOTONIC_BLOCK_SHIFT); + ByteBuffersDataOutput addressBuffer = new ByteBuffersDataOutput(); + ByteBuffersIndexOutput addressOutput = new ByteBuffersIndexOutput(addressBuffer, "temp", "temp"); + long numBlocks = (size + blockMask) >>> shift; + DirectMonotonicWriter writer = DirectMonotonicWriter.getInstance(meta, addressOutput, numBlocks, DIRECT_MONOTONIC_BLOCK_SHIFT); + + BytesRefBuilder previous = new BytesRefBuilder(); + long ord = 0; + long start = data.getFilePointer(); + int maxLength = 0, maxBlockLength = 0; + TermsEnum iterator = values.termsEnum(); + + LZ4.FastCompressionHashTable ht = new LZ4.FastCompressionHashTable(); + ByteArrayDataOutput bufferedOutput = new ByteArrayDataOutput(termsDictBuffer); + int dictLength = 0; + + for (BytesRef term = iterator.next(); term != null; term = iterator.next()) { + if ((ord & blockMask) == 0) { + if (ord != 0) { + // flush the previous block + final int uncompressedLength = compressAndGetTermsDictBlockLength(bufferedOutput, dictLength, ht); + maxBlockLength = Math.max(maxBlockLength, uncompressedLength); + bufferedOutput.reset(termsDictBuffer); + } + + writer.add(data.getFilePointer() - start); + // Write the first term both to the index output, and to the buffer where we'll use it as a + // dictionary for compression + data.writeVInt(term.length); + data.writeBytes(term.bytes, term.offset, term.length); + bufferedOutput = maybeGrowBuffer(bufferedOutput, term.length); + bufferedOutput.writeBytes(term.bytes, term.offset, term.length); + dictLength = term.length; + } else { + final int prefixLength = StringHelper.bytesDifference(previous.get(), term); + final int suffixLength = term.length - prefixLength; + assert suffixLength > 0; // terms are unique + // Will write (suffixLength + 1 byte + 2 vint) bytes. Grow the buffer in need. + bufferedOutput = maybeGrowBuffer(bufferedOutput, suffixLength + 11); + bufferedOutput.writeByte((byte) (Math.min(prefixLength, 15) | (Math.min(15, suffixLength - 1) << 4))); + if (prefixLength >= 15) { + bufferedOutput.writeVInt(prefixLength - 15); + } + if (suffixLength >= 16) { + bufferedOutput.writeVInt(suffixLength - 16); + } + bufferedOutput.writeBytes(term.bytes, term.offset + prefixLength, suffixLength); + } + maxLength = Math.max(maxLength, term.length); + previous.copyBytes(term); + ++ord; + } + // Compress and write out the last block + if (bufferedOutput.getPosition() > dictLength) { + final int uncompressedLength = compressAndGetTermsDictBlockLength(bufferedOutput, dictLength, ht); + maxBlockLength = Math.max(maxBlockLength, uncompressedLength); + } + + writer.finish(); + meta.writeInt(maxLength); + // Write one more int for storing max block length. + meta.writeInt(maxBlockLength); + meta.writeLong(start); + meta.writeLong(data.getFilePointer() - start); + start = data.getFilePointer(); + addressBuffer.copyTo(data); + meta.writeLong(start); + meta.writeLong(data.getFilePointer() - start); + + // Now write the reverse terms index + writeTermsIndex(values); + } + + private int compressAndGetTermsDictBlockLength(ByteArrayDataOutput bufferedOutput, int dictLength, LZ4.FastCompressionHashTable ht) + throws IOException { + int uncompressedLength = bufferedOutput.getPosition() - dictLength; + data.writeVInt(uncompressedLength); + LZ4.compressWithDictionary(termsDictBuffer, 0, dictLength, uncompressedLength, data, ht); + return uncompressedLength; + } + + private ByteArrayDataOutput maybeGrowBuffer(ByteArrayDataOutput bufferedOutput, int termLength) { + int pos = bufferedOutput.getPosition(), originalLength = termsDictBuffer.length; + if (pos + termLength >= originalLength - 1) { + termsDictBuffer = ArrayUtil.grow(termsDictBuffer, originalLength + termLength); + bufferedOutput = new ByteArrayDataOutput(termsDictBuffer, pos, termsDictBuffer.length - pos); + } + return bufferedOutput; + } + + private void writeTermsIndex(SortedSetDocValues values) throws IOException { + final long size = values.getValueCount(); + meta.writeInt(ES819TSDBDocValuesFormat.TERMS_DICT_REVERSE_INDEX_SHIFT); + long start = data.getFilePointer(); + + long numBlocks = 1L + ((size + ES819TSDBDocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) + >>> ES819TSDBDocValuesFormat.TERMS_DICT_REVERSE_INDEX_SHIFT); + ByteBuffersDataOutput addressBuffer = new ByteBuffersDataOutput(); + DirectMonotonicWriter writer; + try (ByteBuffersIndexOutput addressOutput = new ByteBuffersIndexOutput(addressBuffer, "temp", "temp")) { + writer = DirectMonotonicWriter.getInstance(meta, addressOutput, numBlocks, DIRECT_MONOTONIC_BLOCK_SHIFT); + TermsEnum iterator = values.termsEnum(); + BytesRefBuilder previous = new BytesRefBuilder(); + long offset = 0; + long ord = 0; + for (BytesRef term = iterator.next(); term != null; term = iterator.next()) { + if ((ord & ES819TSDBDocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) == 0) { + writer.add(offset); + final int sortKeyLength; + if (ord == 0) { + // no previous term: no bytes to write + sortKeyLength = 0; + } else { + sortKeyLength = StringHelper.sortKeyLength(previous.get(), term); + } + offset += sortKeyLength; + data.writeBytes(term.bytes, term.offset, sortKeyLength); + } else if ((ord + & ES819TSDBDocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) == ES819TSDBDocValuesFormat.TERMS_DICT_REVERSE_INDEX_MASK) { + previous.copyBytes(term); + } + ++ord; + } + writer.add(offset); + writer.finish(); + meta.writeLong(start); + meta.writeLong(data.getFilePointer() - start); + start = data.getFilePointer(); + addressBuffer.copyTo(data); + meta.writeLong(start); + meta.writeLong(data.getFilePointer() - start); + } + } + + @Override + public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + meta.writeInt(field.number); + meta.writeByte(ES819TSDBDocValuesFormat.SORTED_NUMERIC); + writeSortedNumericField(field, new TsdbDocValuesProducer(valuesProducer), -1); + } + + private void writeSortedNumericField(FieldInfo field, TsdbDocValuesProducer valuesProducer, long maxOrd) throws IOException { + if (field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) { + writeSkipIndex(field, valuesProducer); + } + if (maxOrd > -1) { + meta.writeByte((byte) 1); // multiValued (1 = multiValued) + } + + if (valuesProducer.mergeStats.supported()) { + int numDocsWithField = valuesProducer.mergeStats.sumNumDocsWithField(); + long numValues = valuesProducer.mergeStats.sumNumValues(); + if (numDocsWithField == numValues) { + writeField(field, valuesProducer, maxOrd, null); + } else { + assert numValues > numDocsWithField; + try (var accumulator = new OffsetsAccumulator(dir, context, data, numDocsWithField)) { + writeField(field, valuesProducer, maxOrd, accumulator); + accumulator.build(meta, data); + } + } + } else { + long[] stats = writeField(field, valuesProducer, maxOrd, null); + int numDocsWithField = Math.toIntExact(stats[0]); + long numValues = stats[1]; + assert numValues >= numDocsWithField; + + if (numValues > numDocsWithField) { + long start = data.getFilePointer(); + meta.writeLong(start); + meta.writeVInt(ES819TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT); + + final DirectMonotonicWriter addressesWriter = DirectMonotonicWriter.getInstance( + meta, + data, + numDocsWithField + 1L, + ES819TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT + ); + long addr = 0; + addressesWriter.add(addr); + SortedNumericDocValues values = valuesProducer.getSortedNumeric(field); + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + addr += values.docValueCount(); + addressesWriter.add(addr); + } + addressesWriter.finish(); + meta.writeLong(data.getFilePointer() - start); + } + } + } + + @Override + public void mergeSortedNumericField(FieldInfo mergeFieldInfo, MergeState mergeState) throws IOException { + var result = compatibleWithOptimizedMerge(enableOptimizedMerge, mergeState, mergeFieldInfo); + if (result.supported()) { + mergeSortedNumericField(result, mergeFieldInfo, mergeState); + } else { + super.mergeSortedNumericField(mergeFieldInfo, mergeState); + } + } + + private static boolean isSingleValued(FieldInfo field, TsdbDocValuesProducer producer) throws IOException { + if (producer.mergeStats.supported()) { + return producer.mergeStats.sumNumValues() == producer.mergeStats.sumNumDocsWithField(); + } + + var values = producer.getSortedSet(field); + if (DocValues.unwrapSingleton(values) != null) { + return true; + } + + assert values.docID() == -1; + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + int docValueCount = values.docValueCount(); + assert docValueCount > 0; + if (docValueCount > 1) { + return false; + } + } + return true; + } + + @Override + public void mergeSortedSetField(FieldInfo mergeFieldInfo, MergeState mergeState) throws IOException { + var result = compatibleWithOptimizedMerge(enableOptimizedMerge, mergeState, mergeFieldInfo); + if (result.supported()) { + mergeSortedSetField(result, mergeFieldInfo, mergeState); + } else { + super.mergeSortedSetField(mergeFieldInfo, mergeState); + } + } + + @Override + public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + meta.writeInt(field.number); + meta.writeByte(SORTED_SET); + + if (isSingleValued(field, new TsdbDocValuesProducer(valuesProducer))) { + doAddSortedField(field, new TsdbDocValuesProducer(valuesProducer) { + @Override + public SortedDocValues getSorted(FieldInfo field) throws IOException { + return SortedSetSelector.wrap(valuesProducer.getSortedSet(field), SortedSetSelector.Type.MIN); + } + }, true); + return; + } + + SortedSetDocValues values = valuesProducer.getSortedSet(field); + long maxOrd = values.getValueCount(); + writeSortedNumericField(field, new TsdbDocValuesProducer(valuesProducer) { + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { + SortedSetDocValues values = valuesProducer.getSortedSet(field); + return new SortedNumericDocValues() { + + long[] ords = LongsRef.EMPTY_LONGS; + int i, docValueCount; + + @Override + public long nextValue() { + return ords[i++]; + } + + @Override + public int docValueCount() { + return docValueCount; + } + + @Override + public boolean advanceExact(int target) { + throw new UnsupportedOperationException(); + } + + @Override + public int docID() { + return values.docID(); + } + + @Override + public int nextDoc() throws IOException { + int doc = values.nextDoc(); + if (doc != NO_MORE_DOCS) { + docValueCount = values.docValueCount(); + ords = ArrayUtil.grow(ords, docValueCount); + for (int j = 0; j < docValueCount; j++) { + ords[j] = values.nextOrd(); + } + i = 0; + } + return doc; + } + + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + return values.cost(); + } + }; + } + }, maxOrd); + + addTermsDict(valuesProducer.getSortedSet(field)); + } + + @Override + public void close() throws IOException { + boolean success = false; + try { + if (meta != null) { + meta.writeInt(-1); // write EOF marker + CodecUtil.writeFooter(meta); // write checksum + } + if (data != null) { + CodecUtil.writeFooter(data); // write checksum + } + success = true; + } finally { + if (success) { + IOUtils.close(data, meta); + } else { + IOUtils.closeWhileHandlingException(data, meta); + } + meta = data = null; + } + } + + private static class SkipAccumulator { + int minDocID; + int maxDocID; + int docCount; + long minValue; + long maxValue; + + SkipAccumulator(int docID) { + minDocID = docID; + minValue = Long.MAX_VALUE; + maxValue = Long.MIN_VALUE; + docCount = 0; + } + + boolean isDone(int skipIndexIntervalSize, int valueCount, long nextValue, int nextDoc) { + if (docCount < skipIndexIntervalSize) { + return false; + } + // Once we reach the interval size, we will keep accepting documents if + // - next doc value is not a multi-value + // - current accumulator only contains a single value and next value is the same value + // - the accumulator is dense and the next doc keeps the density (no gaps) + return valueCount > 1 || minValue != maxValue || minValue != nextValue || docCount != nextDoc - minDocID; + } + + void accumulate(long value) { + minValue = Math.min(minValue, value); + maxValue = Math.max(maxValue, value); + } + + void accumulate(SkipAccumulator other) { + assert minDocID <= other.minDocID && maxDocID < other.maxDocID; + maxDocID = other.maxDocID; + minValue = Math.min(minValue, other.minValue); + maxValue = Math.max(maxValue, other.maxValue); + docCount += other.docCount; + } + + void nextDoc(int docID) { + maxDocID = docID; + ++docCount; + } + + public static SkipAccumulator merge(List list, int index, int length) { + SkipAccumulator acc = new SkipAccumulator(list.get(index).minDocID); + for (int i = 0; i < length; i++) { + acc.accumulate(list.get(index + i)); + } + return acc; + } + } + + private void writeSkipIndex(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { + assert field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE; + final long start = data.getFilePointer(); + final SortedNumericDocValues values = valuesProducer.getSortedNumeric(field); + long globalMaxValue = Long.MIN_VALUE; + long globalMinValue = Long.MAX_VALUE; + int globalDocCount = 0; + int maxDocId = -1; + final List accumulators = new ArrayList<>(); + SkipAccumulator accumulator = null; + final int maxAccumulators = 1 << (SKIP_INDEX_LEVEL_SHIFT * (SKIP_INDEX_MAX_LEVEL - 1)); + for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { + final long firstValue = values.nextValue(); + if (accumulator != null && accumulator.isDone(skipIndexIntervalSize, values.docValueCount(), firstValue, doc)) { + globalMaxValue = Math.max(globalMaxValue, accumulator.maxValue); + globalMinValue = Math.min(globalMinValue, accumulator.minValue); + globalDocCount += accumulator.docCount; + maxDocId = accumulator.maxDocID; + accumulator = null; + if (accumulators.size() == maxAccumulators) { + writeLevels(accumulators); + accumulators.clear(); + } + } + if (accumulator == null) { + accumulator = new SkipAccumulator(doc); + accumulators.add(accumulator); + } + accumulator.nextDoc(doc); + accumulator.accumulate(firstValue); + for (int i = 1, end = values.docValueCount(); i < end; ++i) { + accumulator.accumulate(values.nextValue()); + } + } + + if (accumulators.isEmpty() == false) { + globalMaxValue = Math.max(globalMaxValue, accumulator.maxValue); + globalMinValue = Math.min(globalMinValue, accumulator.minValue); + globalDocCount += accumulator.docCount; + maxDocId = accumulator.maxDocID; + writeLevels(accumulators); + } + meta.writeLong(start); // record the start in meta + meta.writeLong(data.getFilePointer() - start); // record the length + assert globalDocCount == 0 || globalMaxValue >= globalMinValue; + meta.writeLong(globalMaxValue); + meta.writeLong(globalMinValue); + assert globalDocCount <= maxDocId + 1; + meta.writeInt(globalDocCount); + meta.writeInt(maxDocId); + } + + private void writeLevels(List accumulators) throws IOException { + final List> accumulatorsLevels = new ArrayList<>(SKIP_INDEX_MAX_LEVEL); + accumulatorsLevels.add(accumulators); + for (int i = 0; i < SKIP_INDEX_MAX_LEVEL - 1; i++) { + accumulatorsLevels.add(buildLevel(accumulatorsLevels.get(i))); + } + int totalAccumulators = accumulators.size(); + for (int index = 0; index < totalAccumulators; index++) { + // compute how many levels we need to write for the current accumulator + final int levels = getLevels(index, totalAccumulators); + // write the number of levels + data.writeByte((byte) levels); + // write intervals in reverse order. This is done so we don't + // need to read all of them in case of slipping + for (int level = levels - 1; level >= 0; level--) { + final SkipAccumulator accumulator = accumulatorsLevels.get(level).get(index >> (SKIP_INDEX_LEVEL_SHIFT * level)); + data.writeInt(accumulator.maxDocID); + data.writeInt(accumulator.minDocID); + data.writeLong(accumulator.maxValue); + data.writeLong(accumulator.minValue); + data.writeInt(accumulator.docCount); + } + } + } + + private static List buildLevel(List accumulators) { + final int levelSize = 1 << SKIP_INDEX_LEVEL_SHIFT; + final List collector = new ArrayList<>(); + for (int i = 0; i < accumulators.size() - levelSize + 1; i += levelSize) { + collector.add(SkipAccumulator.merge(accumulators, i, levelSize)); + } + return collector; + } + + private static int getLevels(int index, int size) { + if (Integer.numberOfTrailingZeros(index) >= SKIP_INDEX_LEVEL_SHIFT) { + // TODO: can we do it in constant time rather than linearly with SKIP_INDEX_MAX_LEVEL? + final int left = size - index; + for (int level = SKIP_INDEX_MAX_LEVEL - 1; level > 0; level--) { + final int numberIntervals = 1 << (SKIP_INDEX_LEVEL_SHIFT * level); + if (left >= numberIntervals && index % numberIntervals == 0) { + return level + 1; + } + } + } + return 1; + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormatTests.java index 970b138462c90..208e5dff6b717 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormatTests.java @@ -10,6 +10,7 @@ package org.elasticsearch.index.codec.tsdb.es819; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; import org.apache.lucene.document.BinaryDocValuesField; @@ -27,6 +28,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.LogByteSizeMergePolicy; import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; @@ -82,6 +84,27 @@ public DocValuesFormat getDocValuesFormatForField(String field) { } }; + public static class TestES819TSDBDocValuesFormatVersion0 extends ES819TSDBDocValuesFormat { + + public TestES819TSDBDocValuesFormatVersion0() { + super(); + } + + @Override + public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + return new ES819TSDBDocValuesConsumerVersion0( + state, + skipIndexIntervalSize, + minDocsPerOrdinalForRangeEncoding, + enableOptimizedMerge, + DATA_CODEC, + DATA_EXTENSION, + META_CODEC, + META_EXTENSION + ); + } + } + @Override protected Codec getCodec() { return codec; From 2047c9abc9305cd0389a49b7eb286c6ed6bf0eee Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Mon, 3 Nov 2025 17:12:01 +0100 Subject: [PATCH 30/32] ESQL: Add support for exponential_histogram in code generation (#137459) --- .../compute/gen/AggregatorImplementer.java | 58 ++++++++++++------- .../compute/gen/EvaluatorImplementer.java | 7 ++- .../gen/GroupingAggregatorImplementer.java | 54 ++++++++--------- .../elasticsearch/compute/gen/Methods.java | 3 + .../org/elasticsearch/compute/gen/Types.java | 43 +++++++++++--- .../compute/gen/argument/Argument.java | 23 +++++--- .../compute/gen/argument/ArrayArgument.java | 18 ++++-- .../gen/argument/StandardArgument.java | 12 ++-- ...untDistinctBytesRefAggregatorFunction.java | 4 +- ...nctBytesRefGroupingAggregatorFunction.java | 12 ++-- ...CountDistinctDoubleAggregatorFunction.java | 4 +- ...tinctDoubleGroupingAggregatorFunction.java | 12 ++-- .../CountDistinctFloatAggregatorFunction.java | 4 +- ...stinctFloatGroupingAggregatorFunction.java | 12 ++-- .../CountDistinctIntAggregatorFunction.java | 4 +- ...DistinctIntGroupingAggregatorFunction.java | 12 ++-- .../CountDistinctLongAggregatorFunction.java | 4 +- ...istinctLongGroupingAggregatorFunction.java | 12 ++-- ...BytesRefByTimestampAggregatorFunction.java | 4 +- ...ByTimestampGroupingAggregatorFunction.java | 6 +- ...BytesRefByTimestampAggregatorFunction.java | 4 +- ...ByTimestampGroupingAggregatorFunction.java | 6 +- .../MaxBytesRefAggregatorFunction.java | 4 +- ...MaxBytesRefGroupingAggregatorFunction.java | 12 ++-- .../aggregation/MaxIpAggregatorFunction.java | 4 +- .../MaxIpGroupingAggregatorFunction.java | 12 ++-- ...luteDeviationDoubleAggregatorFunction.java | 4 +- ...ationDoubleGroupingAggregatorFunction.java | 12 ++-- ...oluteDeviationFloatAggregatorFunction.java | 4 +- ...iationFloatGroupingAggregatorFunction.java | 12 ++-- ...bsoluteDeviationIntAggregatorFunction.java | 4 +- ...eviationIntGroupingAggregatorFunction.java | 12 ++-- ...soluteDeviationLongAggregatorFunction.java | 4 +- ...viationLongGroupingAggregatorFunction.java | 12 ++-- .../MinBytesRefAggregatorFunction.java | 4 +- ...MinBytesRefGroupingAggregatorFunction.java | 12 ++-- .../aggregation/MinIpAggregatorFunction.java | 4 +- .../MinIpGroupingAggregatorFunction.java | 12 ++-- .../PercentileDoubleAggregatorFunction.java | 4 +- ...ntileDoubleGroupingAggregatorFunction.java | 12 ++-- .../PercentileFloatAggregatorFunction.java | 4 +- ...entileFloatGroupingAggregatorFunction.java | 12 ++-- .../PercentileIntAggregatorFunction.java | 4 +- ...rcentileIntGroupingAggregatorFunction.java | 12 ++-- .../PercentileLongAggregatorFunction.java | 4 +- ...centileLongGroupingAggregatorFunction.java | 12 ++-- .../SampleBooleanAggregatorFunction.java | 2 +- ...mpleBooleanGroupingAggregatorFunction.java | 6 +- .../SampleBytesRefAggregatorFunction.java | 2 +- ...pleBytesRefGroupingAggregatorFunction.java | 6 +- .../SampleDoubleAggregatorFunction.java | 2 +- ...ampleDoubleGroupingAggregatorFunction.java | 6 +- .../SampleIntAggregatorFunction.java | 2 +- .../SampleIntGroupingAggregatorFunction.java | 6 +- .../SampleLongAggregatorFunction.java | 2 +- .../SampleLongGroupingAggregatorFunction.java | 6 +- .../TopBytesRefAggregatorFunction.java | 2 +- ...TopBytesRefGroupingAggregatorFunction.java | 6 +- .../aggregation/TopIpAggregatorFunction.java | 2 +- .../TopIpGroupingAggregatorFunction.java | 6 +- .../ValuesBytesRefAggregatorFunction.java | 2 +- 61 files changed, 317 insertions(+), 251 deletions(-) diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java index 1327db4bc0bea..80ae710a3c16c 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java @@ -37,6 +37,7 @@ import javax.lang.model.util.Elements; import static java.util.stream.Collectors.joining; +import static org.elasticsearch.compute.gen.Methods.getMethod; import static org.elasticsearch.compute.gen.Methods.optionalStaticMethod; import static org.elasticsearch.compute.gen.Methods.requireAnyArgs; import static org.elasticsearch.compute.gen.Methods.requireAnyType; @@ -53,7 +54,6 @@ import static org.elasticsearch.compute.gen.Types.BLOCK; import static org.elasticsearch.compute.gen.Types.BLOCK_ARRAY; import static org.elasticsearch.compute.gen.Types.BOOLEAN_VECTOR; -import static org.elasticsearch.compute.gen.Types.BYTES_REF; import static org.elasticsearch.compute.gen.Types.DRIVER_CONTEXT; import static org.elasticsearch.compute.gen.Types.ELEMENT_TYPE; import static org.elasticsearch.compute.gen.Types.INTERMEDIATE_STATE_DESC; @@ -62,6 +62,8 @@ import static org.elasticsearch.compute.gen.Types.PAGE; import static org.elasticsearch.compute.gen.Types.WARNINGS; import static org.elasticsearch.compute.gen.Types.blockType; +import static org.elasticsearch.compute.gen.Types.fromString; +import static org.elasticsearch.compute.gen.Types.scratchType; import static org.elasticsearch.compute.gen.Types.vectorType; /** @@ -85,7 +87,7 @@ public class AggregatorImplementer { private final AggregationState aggState; private final List aggParams; - private final boolean hasOnlyBlockArguments; + private final boolean tryToUseVectors; public AggregatorImplementer( Elements elements, @@ -119,7 +121,8 @@ public AggregatorImplementer( return a; }).filter(a -> a instanceof PositionArgument == false).toList(); - this.hasOnlyBlockArguments = this.aggParams.stream().allMatch(a -> a instanceof BlockArgument); + this.tryToUseVectors = aggParams.stream().anyMatch(a -> (a instanceof BlockArgument) == false) + && aggParams.stream().noneMatch(a -> a.supportsVectorReadAccess() == false); this.createParameters = init.getParameters() .stream() @@ -199,7 +202,7 @@ private TypeSpec type() { builder.addMethod(addRawInput()); builder.addMethod(addRawInputExploded(true)); builder.addMethod(addRawInputExploded(false)); - if (hasOnlyBlockArguments == false) { + if (tryToUseVectors) { builder.addMethod(addRawVector(false)); builder.addMethod(addRawVector(true)); } @@ -340,16 +343,18 @@ private MethodSpec addRawInputExploded(boolean hasMask) { builder.addStatement("$T $L = page.getBlock(channels.get($L))", a.dataType(true), a.blockName(), i); } - for (Argument a : aggParams) { - String rawBlock = "addRawBlock(" - + aggParams.stream().map(arg -> arg.blockName()).collect(joining(", ")) - + (hasMask ? ", mask" : "") - + ")"; + if (tryToUseVectors) { + for (Argument a : aggParams) { + String rawBlock = "addRawBlock(" + + aggParams.stream().map(arg -> arg.blockName()).collect(joining(", ")) + + (hasMask ? ", mask" : "") + + ")"; - a.resolveVectors(builder, rawBlock, "return"); + a.resolveVectors(builder, rawBlock, "return"); + } } - builder.addStatement(invokeAddRaw(hasOnlyBlockArguments, hasMask)); + builder.addStatement(invokeAddRaw(tryToUseVectors == false, hasMask)); return builder.build(); } @@ -499,9 +504,9 @@ private MethodSpec.Builder initAddRaw(boolean blockStyle, boolean masked) { builder.addParameter(BOOLEAN_VECTOR, "mask"); } for (Argument a : aggParams) { - if (a.isBytesRef()) { - // Add bytes_ref scratch var that will be used for bytes_ref blocks/vectors - builder.addStatement("$T $L = new $T()", BYTES_REF, a.scratchName(), BYTES_REF); + if (a.scratchType() != null) { + // Add scratch var that will be used for some blocks/vectors, e.g. for bytes_ref + builder.addStatement("$T $L = new $T()", a.scratchType(), a.scratchName(), a.scratchType()); } } return builder; @@ -610,8 +615,8 @@ private MethodSpec addIntermediateInput() { ).map(Methods::requireType).toArray(TypeMatcher[]::new) ) ); - if (intermediateState.stream().map(IntermediateStateDesc::elementType).anyMatch(n -> n.equals("BYTES_REF"))) { - builder.addStatement("$T scratch = new $T()", BYTES_REF, BYTES_REF); + for (IntermediateStateDesc interState : intermediateState) { + interState.addScratchDeclaration(builder); } builder.addStatement("$T.combineIntermediate(state, " + intermediateStateRowAccess() + ")", declarationType); } @@ -706,13 +711,25 @@ public String access(String position) { if (block) { return name(); } - String s = name() + "." + vectorAccessorName(elementType()) + "(" + position; - if (elementType().equals("BYTES_REF")) { - s += ", scratch"; + String s = name() + "."; + if (vectorType(elementType) != null) { + s += vectorAccessorName(elementType()) + "(" + position; + } else { + s += getMethod(fromString(elementType())) + "(" + name() + ".getFirstValueIndex(" + position + ")"; + } + if (scratchType(elementType()) != null) { + s += ", " + name() + "Scratch"; } return s + ")"; } + public void addScratchDeclaration(MethodSpec.Builder builder) { + ClassName scratchType = scratchType(elementType()); + if (scratchType != null) { + builder.addStatement("$T $L = new $T()", scratchType, name() + "Scratch", scratchType); + } + } + public void assignToVariable(MethodSpec.Builder builder, int offset) { builder.addStatement("Block $L = page.getBlock(channels.get($L))", name + "Uncast", offset); ClassName blockType = blockType(elementType()); @@ -721,7 +738,7 @@ public void assignToVariable(MethodSpec.Builder builder, int offset) { builder.addStatement("return"); builder.endControlFlow(); } - if (block) { + if (block || vectorType(elementType) == null) { builder.addStatement("$T $L = ($T) $L", blockType, name, blockType, name + "Uncast"); } else { builder.addStatement("$T $L = (($T) $L).asVector()", vectorType(elementType), name, blockType, name + "Uncast"); @@ -732,6 +749,7 @@ public TypeName combineArgType() { var type = Types.fromString(elementType); return block ? blockType(type) : type; } + } /** diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java index bba578f45f5b0..3b1b184cb02d0 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java @@ -51,6 +51,7 @@ public class EvaluatorImplementer { private final ProcessFunction processFunction; private final ClassName implementation; private final boolean processOutputsMultivalued; + private final boolean vectorsUnsupported; private final boolean allNullsIsNull; public EvaluatorImplementer( @@ -69,6 +70,8 @@ public EvaluatorImplementer( declarationType.getSimpleName() + extraName + "Evaluator" ); this.processOutputsMultivalued = this.processFunction.hasBlockType; + boolean anyParameterNotSupportingVectors = this.processFunction.args.stream().anyMatch(a -> a.supportsVectorReadAccess() == false); + vectorsUnsupported = processOutputsMultivalued || anyParameterNotSupportingVectors; this.allNullsIsNull = allNullsIsNull; } @@ -101,7 +104,7 @@ private TypeSpec type() { builder.addMethod(eval()); builder.addMethod(processFunction.baseRamBytesUsed()); - if (processOutputsMultivalued) { + if (vectorsUnsupported) { if (processFunction.args.stream().anyMatch(x -> x instanceof FixedArgument == false)) { builder.addMethod(realEval(true)); } @@ -145,7 +148,7 @@ private MethodSpec eval() { builder.addModifiers(Modifier.PUBLIC).returns(BLOCK).addParameter(PAGE, "page"); processFunction.args.forEach(a -> a.evalToBlock(builder)); String invokeBlockEval = invokeRealEval(true); - if (processOutputsMultivalued) { + if (vectorsUnsupported) { builder.addStatement(invokeBlockEval); } else { processFunction.args.forEach(a -> a.resolveVectors(builder, invokeBlockEval)); diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java index 399f45b5df667..76edfbb1f70e3 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java @@ -51,7 +51,6 @@ import static org.elasticsearch.compute.gen.Types.BIG_ARRAYS; import static org.elasticsearch.compute.gen.Types.BLOCK; import static org.elasticsearch.compute.gen.Types.BLOCK_ARRAY; -import static org.elasticsearch.compute.gen.Types.BYTES_REF; import static org.elasticsearch.compute.gen.Types.DRIVER_CONTEXT; import static org.elasticsearch.compute.gen.Types.ELEMENT_TYPE; import static org.elasticsearch.compute.gen.Types.GROUPING_AGGREGATOR_EVALUATOR_CONTEXT; @@ -93,6 +92,7 @@ public class GroupingAggregatorImplementer { private final AggregationState aggState; private final List aggParams; private final boolean hasOnlyBlockArguments; + private final boolean allArgumentsSupportVectors; public GroupingAggregatorImplementer( Elements elements, @@ -128,6 +128,7 @@ public GroupingAggregatorImplementer( }).filter(a -> a instanceof PositionArgument == false).toList(); this.hasOnlyBlockArguments = this.aggParams.stream().allMatch(a -> a instanceof BlockArgument); + this.allArgumentsSupportVectors = aggParams.stream().noneMatch(a -> a.supportsVectorReadAccess() == false); this.createParameters = init.getParameters() .stream() @@ -204,7 +205,7 @@ private TypeSpec type() { builder.addMethod(prepareProcessRawInputPage()); for (ClassName groupIdClass : GROUP_IDS_CLASSES) { builder.addMethod(addRawInputLoop(groupIdClass, false)); - if (hasOnlyBlockArguments == false) { + if (hasOnlyBlockArguments == false && allArgumentsSupportVectors) { builder.addMethod(addRawInputLoop(groupIdClass, true)); } builder.addMethod(addIntermediateInput(groupIdClass)); @@ -330,26 +331,31 @@ private MethodSpec prepareProcessRawInputPage() { builder.addStatement("$T $L = page.getBlock(channels.get($L))", a.dataType(true), a.blockName(), i); } - for (Argument a : aggParams) { - builder.addStatement( - "$T $L = $L.asVector()", - vectorType(a.elementType()), - (a instanceof BlockArgument) ? (a.name() + "Vector") : a.vectorName(), - a.blockName() - ); - builder.beginControlFlow("if ($L == null)", (a instanceof BlockArgument) ? (a.name() + "Vector") : a.vectorName()); - { + String groupIdTrackingStatement = "maybeEnableGroupIdTracking(seenGroupIds, " + + aggParams.stream().map(arg -> arg.blockName()).collect(joining(", ")) + + ")"; + + if (allArgumentsSupportVectors) { + + for (Argument a : aggParams) { builder.addStatement( - "maybeEnableGroupIdTracking(seenGroupIds, " - + aggParams.stream().map(arg -> arg.blockName()).collect(joining(", ")) - + ")" + "$T $L = $L.asVector()", + vectorType(a.elementType()), + (a instanceof BlockArgument) ? (a.name() + "Vector") : a.vectorName(), + a.blockName() ); - returnAddInput(builder, false); + builder.beginControlFlow("if ($L == null)", (a instanceof BlockArgument) ? (a.name() + "Vector") : a.vectorName()); + { + builder.addStatement(groupIdTrackingStatement); + returnAddInput(builder, false); + } + builder.endControlFlow(); } - builder.endControlFlow(); + returnAddInput(builder, true); + } else { + builder.addStatement(groupIdTrackingStatement); + returnAddInput(builder, false); } - - returnAddInput(builder, true); return builder.build(); } @@ -443,9 +449,9 @@ private MethodSpec addRawInputLoop(TypeName groupsType, boolean valuesAreVector) ); } for (Argument a : aggParams) { - if (a.isBytesRef()) { - // Add bytes_ref scratch var that will be used for bytes_ref blocks/vectors - builder.addStatement("$T $L = new $T()", BYTES_REF, a.scratchName(), BYTES_REF); + if (a.scratchType() != null) { + // Add scratch var that will be used for some blocks/vectors, e.g. for bytes_ref + builder.addStatement("$T $L = new $T()", a.scratchType(), a.scratchName(), a.scratchType()); } } @@ -645,11 +651,7 @@ private MethodSpec addIntermediateInput(TypeName groupsType) { .collect(Collectors.joining(", ")); builder.addStatement("$T.combineIntermediate(state, positionOffset, groups, " + states + ")", declarationType); } else { - if (intermediateState.stream() - .map(AggregatorImplementer.IntermediateStateDesc::elementType) - .anyMatch(n -> n.equals("BYTES_REF"))) { - builder.addStatement("$T scratch = new $T()", BYTES_REF, BYTES_REF); - } + intermediateState.forEach(state -> state.addScratchDeclaration(builder)); builder.beginControlFlow("for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++)"); { if (groupsIsBlock) { diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Methods.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Methods.java index 15520c6ed93ab..2a06777c62d6e 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Methods.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Methods.java @@ -314,6 +314,9 @@ public static String getMethod(TypeName elementType) { if (elementType.equals(TypeName.FLOAT)) { return "getFloat"; } + if (elementType.equals(Types.EXPONENTIAL_HISTOGRAM)) { + return "getExponentialHistogram"; + } throw new IllegalArgumentException("unknown get method for [" + elementType + "]"); } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java index 82b3450343ced..d31679665efb5 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java @@ -52,6 +52,8 @@ public class Types { public static final ClassName LONG_BLOCK = ClassName.get(DATA_PACKAGE, "LongBlock"); public static final ClassName DOUBLE_BLOCK = ClassName.get(DATA_PACKAGE, "DoubleBlock"); public static final ClassName FLOAT_BLOCK = ClassName.get(DATA_PACKAGE, "FloatBlock"); + public static final ClassName EXPONENTIAL_HISTOGRAM_BLOCK = ClassName.get(DATA_PACKAGE, "ExponentialHistogramBlock"); + public static final ClassName EXPONENTIAL_HISTOGRAM_SCRATCH = ClassName.get(DATA_PACKAGE, "ExponentialHistogramScratch"); static final ClassName BOOLEAN_BLOCK_BUILDER = BOOLEAN_BLOCK.nestedClass("Builder"); static final ClassName BYTES_REF_BLOCK_BUILDER = BYTES_REF_BLOCK.nestedClass("Builder"); @@ -59,6 +61,7 @@ public class Types { static final ClassName LONG_BLOCK_BUILDER = LONG_BLOCK.nestedClass("Builder"); static final ClassName DOUBLE_BLOCK_BUILDER = DOUBLE_BLOCK.nestedClass("Builder"); static final ClassName FLOAT_BLOCK_BUILDER = FLOAT_BLOCK.nestedClass("Builder"); + static final ClassName EXPONENTIAL_HISTOGRAM_BLOCK_BUILDER = ClassName.get(DATA_PACKAGE, "ExponentialHistogramBlockBuilder"); static final ClassName ELEMENT_TYPE = ClassName.get(DATA_PACKAGE, "ElementType"); @@ -133,24 +136,32 @@ public class Types { static final ClassName SOURCE = ClassName.get("org.elasticsearch.xpack.esql.core.tree", "Source"); public static final ClassName BYTES_REF = ClassName.get("org.apache.lucene.util", "BytesRef"); + public static final ClassName EXPONENTIAL_HISTOGRAM = ClassName.get("org.elasticsearch.exponentialhistogram", "ExponentialHistogram"); public static final ClassName RELEASABLE = ClassName.get("org.elasticsearch.core", "Releasable"); public static final ClassName RELEASABLES = ClassName.get("org.elasticsearch.core", "Releasables"); - private record TypeDef(TypeName type, String alias, ClassName block, ClassName vector) { + private record TypeDef(TypeName type, String alias, ClassName block, ClassName vector, ClassName scratch) { - public static TypeDef of(TypeName type, String alias, String block, String vector) { - return new TypeDef(type, alias, ClassName.get(DATA_PACKAGE, block), ClassName.get(DATA_PACKAGE, vector)); + public static TypeDef of(TypeName type, String alias, String block, String vector, ClassName scratch) { + return new TypeDef( + type, + alias, + ClassName.get(DATA_PACKAGE, block), + vector == null ? null : ClassName.get(DATA_PACKAGE, vector), + scratch + ); } } private static final Map TYPES = Stream.of( - TypeDef.of(TypeName.BOOLEAN, "BOOLEAN", "BooleanBlock", "BooleanVector"), - TypeDef.of(TypeName.INT, "INT", "IntBlock", "IntVector"), - TypeDef.of(TypeName.LONG, "LONG", "LongBlock", "LongVector"), - TypeDef.of(TypeName.FLOAT, "FLOAT", "FloatBlock", "FloatVector"), - TypeDef.of(TypeName.DOUBLE, "DOUBLE", "DoubleBlock", "DoubleVector"), - TypeDef.of(BYTES_REF, "BYTES_REF", "BytesRefBlock", "BytesRefVector") + TypeDef.of(TypeName.BOOLEAN, "BOOLEAN", "BooleanBlock", "BooleanVector", null), + TypeDef.of(TypeName.INT, "INT", "IntBlock", "IntVector", null), + TypeDef.of(TypeName.LONG, "LONG", "LongBlock", "LongVector", null), + TypeDef.of(TypeName.FLOAT, "FLOAT", "FloatBlock", "FloatVector", null), + TypeDef.of(TypeName.DOUBLE, "DOUBLE", "DoubleBlock", "DoubleVector", null), + TypeDef.of(BYTES_REF, "BYTES_REF", "BytesRefBlock", "BytesRefVector", BYTES_REF), + TypeDef.of(EXPONENTIAL_HISTOGRAM, "EXPONENTIAL_HISTOGRAM", "ExponentialHistogramBlock", null, EXPONENTIAL_HISTOGRAM_SCRATCH) ) .flatMap(def -> Stream.of(def.type.toString(), def.type + "[]", def.alias).map(alias -> Map.entry(alias, def))) .collect(toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -183,6 +194,14 @@ static ClassName vectorType(String elementType) { return findRequired(elementType, "vector").vector; } + public static ClassName scratchType(String elementType) { + TypeDef typeDef = TYPES.get(elementType); + if (typeDef != null) { + return typeDef.scratch; + } + return null; + } + static ClassName builderType(TypeName resultType) { if (resultType.equals(BOOLEAN_BLOCK)) { return BOOLEAN_BLOCK_BUILDER; @@ -220,6 +239,9 @@ static ClassName builderType(TypeName resultType) { if (resultType.equals(FLOAT_VECTOR)) { return FLOAT_VECTOR_BUILDER; } + if (resultType.equals(EXPONENTIAL_HISTOGRAM_BLOCK)) { + return EXPONENTIAL_HISTOGRAM_BLOCK_BUILDER; + } throw new IllegalArgumentException("unknown builder type for [" + resultType + "]"); } @@ -261,6 +283,9 @@ public static TypeName elementType(TypeName t) { if (t.equals(FLOAT_BLOCK) || t.equals(FLOAT_VECTOR) || t.equals(FLOAT_BLOCK_BUILDER)) { return TypeName.FLOAT; } + if (t.equals(EXPONENTIAL_HISTOGRAM_BLOCK) || t.equals(EXPONENTIAL_HISTOGRAM_BLOCK_BUILDER)) { + return EXPONENTIAL_HISTOGRAM; + } throw new IllegalArgumentException("unknown element type for [" + t + "]"); } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/Argument.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/Argument.java index 4a2100a177e92..420989cfd0ddf 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/Argument.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/Argument.java @@ -17,7 +17,6 @@ import org.elasticsearch.compute.gen.Types; import java.util.List; -import java.util.Objects; import javax.lang.model.element.VariableElement; import javax.lang.model.type.ArrayType; @@ -25,7 +24,6 @@ import javax.lang.model.type.TypeMirror; import static org.elasticsearch.compute.gen.Methods.getMethod; -import static org.elasticsearch.compute.gen.Types.BYTES_REF; import static org.elasticsearch.compute.gen.Types.blockType; import static org.elasticsearch.compute.gen.Types.vectorType; import static org.elasticsearch.compute.gen.argument.StandardArgument.isBlockType; @@ -93,18 +91,18 @@ default String offsetName() { return name() + "Offset"; } + default ClassName scratchType() { + return Types.scratchType(type().toString()); + } + default String scratchName() { - if (isBytesRef() == false) { - throw new IllegalStateException("can't build scratch for non-BytesRef"); + if (scratchType() == null) { + throw new IllegalStateException("can't build scratch for " + type()); } return name() + "Scratch"; } - default boolean isBytesRef() { - return Objects.equals(type(), BYTES_REF); - } - String name(); TypeName type(); @@ -119,6 +117,13 @@ default TypeName elementType() { */ TypeName dataType(boolean blockStyle); + /** + * False if and only if there is a block backing this parameter and that block does not support access as a vector. Otherwise true. + */ + default boolean supportsVectorReadAccess() { + return true; + } + /** * The parameter passed to the real evaluation function */ @@ -198,7 +203,7 @@ default TypeName elementType() { */ default void read(MethodSpec.Builder builder, String accessor, String firstParam) { String params = firstParam; - if (isBytesRef()) { + if (scratchType() != null) { params += ", " + scratchName(); } builder.addStatement("$T $L = $L.$L($L)", type(), valueName(), accessor, getMethod(type()), params); diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/ArrayArgument.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/ArrayArgument.java index 0ee2bf1998a74..08dd5fa478f2a 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/ArrayArgument.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/ArrayArgument.java @@ -8,6 +8,7 @@ package org.elasticsearch.compute.gen.argument; import com.squareup.javapoet.ArrayTypeName; +import com.squareup.javapoet.ClassName; import com.squareup.javapoet.MethodSpec; import com.squareup.javapoet.TypeName; import com.squareup.javapoet.TypeSpec; @@ -18,7 +19,6 @@ import javax.lang.model.element.Modifier; import static org.elasticsearch.compute.gen.Methods.getMethod; -import static org.elasticsearch.compute.gen.Types.BYTES_REF; import static org.elasticsearch.compute.gen.Types.EXPRESSION_EVALUATOR; import static org.elasticsearch.compute.gen.Types.EXPRESSION_EVALUATOR_FACTORY; import static org.elasticsearch.compute.gen.Types.RELEASABLE; @@ -35,6 +35,11 @@ public TypeName dataType(boolean blockStyle) { return ArrayTypeName.of(vectorType(type)); } + @Override + public boolean supportsVectorReadAccess() { + return vectorType(type) != null; + } + @Override public String paramName(boolean blockStyle) { return name + (blockStyle ? "Block" : "Vector") + "s"; @@ -105,10 +110,11 @@ public void resolveVectors(MethodSpec.Builder builder, String... invokeBlockEval @Override public void createScratch(MethodSpec.Builder builder) { builder.addStatement("$T[] $LValues = new $T[$L.length]", type, name, type, name); - if (isBytesRef()) { - builder.addStatement("$T[] $LScratch = new $T[$L.length]", type, name, type, name); + ClassName scratchType = scratchType(); + if (scratchType != null) { + builder.addStatement("$T[] $LScratch = new $T[$L.length]", scratchType, name, scratchType, name); builder.beginControlFlow("for (int i = 0; i < $L.length; i++)", name); - builder.addStatement("$LScratch[i] = new $T()", name, BYTES_REF); + builder.addStatement("$LScratch[i] = new $T()", name, scratchType); builder.endControlFlow(); } } @@ -136,8 +142,8 @@ public void read(MethodSpec.Builder builder, boolean blockStyle) { } else { lookupVar = "p"; } - if (isBytesRef()) { - builder.addStatement("$LValues[i] = $L[i].getBytesRef($L, $LScratch[i])", name, paramName(blockStyle), lookupVar, name); + if (scratchType() != null) { + builder.addStatement("$LValues[i] = $L[i].$L($L, $LScratch[i])", name, paramName(blockStyle), getMethod(type), lookupVar, name); } else { builder.addStatement("$LValues[i] = $L[i].$L($L)", name, paramName(blockStyle), getMethod(type), lookupVar); } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/StandardArgument.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/StandardArgument.java index 981e3de3f22f4..575c40a766ec1 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/StandardArgument.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/argument/StandardArgument.java @@ -17,7 +17,6 @@ import static org.elasticsearch.compute.gen.Methods.getMethod; import static org.elasticsearch.compute.gen.Types.BOOLEAN_BLOCK; -import static org.elasticsearch.compute.gen.Types.BYTES_REF; import static org.elasticsearch.compute.gen.Types.BYTES_REF_BLOCK; import static org.elasticsearch.compute.gen.Types.DOUBLE_BLOCK; import static org.elasticsearch.compute.gen.Types.EXPRESSION_EVALUATOR; @@ -36,6 +35,11 @@ public TypeName dataType(boolean blockStyle) { return vectorType(type); } + @Override + public boolean supportsVectorReadAccess() { + return dataType(false) != null; + } + @Override public String paramName(boolean blockStyle) { return name + (blockStyle ? "Block" : "Vector"); @@ -93,8 +97,8 @@ public void resolveVectors(MethodSpec.Builder builder, String... invokeBlockEval @Override public void createScratch(MethodSpec.Builder builder) { - if (isBytesRef()) { - builder.addStatement("$T $LScratch = new $T()", BYTES_REF, name, BYTES_REF); + if (scratchType() != null) { + builder.addStatement("$T $LScratch = new $T()", scratchType(), name, scratchType()); } } @@ -123,7 +127,7 @@ static boolean isBlockType(TypeName type) { @Override public void read(MethodSpec.Builder builder, boolean blockStyle) { String params = blockStyle ? paramName(true) + ".getFirstValueIndex(p)" : "p"; - if (isBytesRef()) { + if (scratchType() != null) { params += ", " + scratchName(); } builder.addStatement("$T $L = $L.$L($L)", type, name, paramName(blockStyle), getMethod(type), params); diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefAggregatorFunction.java index 71a3ec03278eb..b9737cc15976a 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefAggregatorFunction.java @@ -151,8 +151,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); assert hll.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - CountDistinctBytesRefAggregator.combineIntermediate(state, hll.getBytesRef(0, scratch)); + BytesRef hllScratch = new BytesRef(); + CountDistinctBytesRefAggregator.combineIntermediate(state, hll.getBytesRef(0, hllScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefGroupingAggregatorFunction.java index 1abea90e89ee3..8c11792fa4f4e 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctBytesRefGroupingAggregatorFunction.java @@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -168,7 +168,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctBytesRefAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctBytesRefAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -223,7 +223,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -233,7 +233,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctBytesRefAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctBytesRefAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -274,11 +274,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - CountDistinctBytesRefAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctBytesRefAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleAggregatorFunction.java index 68d7270a49d0b..c791c3fd11b7a 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleAggregatorFunction.java @@ -149,8 +149,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); assert hll.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - CountDistinctDoubleAggregator.combineIntermediate(state, hll.getBytesRef(0, scratch)); + BytesRef hllScratch = new BytesRef(); + CountDistinctDoubleAggregator.combineIntermediate(state, hll.getBytesRef(0, hllScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleGroupingAggregatorFunction.java index c839f787c991f..3520b9f53f4c9 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctDoubleGroupingAggregatorFunction.java @@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -168,7 +168,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctDoubleAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctDoubleAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -221,7 +221,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -231,7 +231,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctDoubleAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctDoubleAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -270,11 +270,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - CountDistinctDoubleAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctDoubleAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java index c01c34887cf4e..06068bf343978 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java @@ -149,8 +149,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); assert hll.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - CountDistinctFloatAggregator.combineIntermediate(state, hll.getBytesRef(0, scratch)); + BytesRef hllScratch = new BytesRef(); + CountDistinctFloatAggregator.combineIntermediate(state, hll.getBytesRef(0, hllScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java index a8f284f7e8fdc..4a000e848c73b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java @@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -168,7 +168,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctFloatAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctFloatAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -221,7 +221,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -231,7 +231,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctFloatAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctFloatAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -270,11 +270,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - CountDistinctFloatAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctFloatAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntAggregatorFunction.java index 25a82171c54e4..1d378d7d8bb06 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntAggregatorFunction.java @@ -149,8 +149,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); assert hll.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - CountDistinctIntAggregator.combineIntermediate(state, hll.getBytesRef(0, scratch)); + BytesRef hllScratch = new BytesRef(); + CountDistinctIntAggregator.combineIntermediate(state, hll.getBytesRef(0, hllScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntGroupingAggregatorFunction.java index ad7e329d8ea7d..5320bff130fa5 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctIntGroupingAggregatorFunction.java @@ -157,7 +157,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -167,7 +167,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctIntAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctIntAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -220,7 +220,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -230,7 +230,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctIntAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctIntAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -269,11 +269,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - CountDistinctIntAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctIntAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongAggregatorFunction.java index 34c5b2ee2bda5..88ddb7261f378 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongAggregatorFunction.java @@ -149,8 +149,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); assert hll.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - CountDistinctLongAggregator.combineIntermediate(state, hll.getBytesRef(0, scratch)); + BytesRef hllScratch = new BytesRef(); + CountDistinctLongAggregator.combineIntermediate(state, hll.getBytesRef(0, hllScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongGroupingAggregatorFunction.java index 8bdff2e73ebb9..5a76c5440615c 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctLongGroupingAggregatorFunction.java @@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -168,7 +168,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctLongAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctLongAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -221,7 +221,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -231,7 +231,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - CountDistinctLongAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctLongAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } } @@ -270,11 +270,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef hllScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - CountDistinctLongAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, scratch)); + CountDistinctLongAggregator.combineIntermediate(state, groupId, hll.getBytesRef(valuesPosition, hllScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstBytesRefByTimestampAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstBytesRefByTimestampAggregatorFunction.java index eb88ed39323d2..91822de9973aa 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstBytesRefByTimestampAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstBytesRefByTimestampAggregatorFunction.java @@ -237,8 +237,8 @@ public void addIntermediateInput(Page page) { } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert seen.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - FirstBytesRefByTimestampAggregator.combineIntermediate(state, timestamps.getLong(0), values.getBytesRef(0, scratch), seen.getBoolean(0)); + BytesRef valuesScratch = new BytesRef(); + FirstBytesRefByTimestampAggregator.combineIntermediate(state, timestamps.getLong(0), values.getBytesRef(0, valuesScratch), seen.getBoolean(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstBytesRefByTimestampGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstBytesRefByTimestampGroupingAggregatorFunction.java index aee8d6296dc02..42b016f4580d8 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstBytesRefByTimestampGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstBytesRefByTimestampGroupingAggregatorFunction.java @@ -200,7 +200,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page } BytesRefBlock values = (BytesRefBlock) valuesUncast; assert timestamps.getPositionCount() == values.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef valuesScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -282,7 +282,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa } BytesRefBlock values = (BytesRefBlock) valuesUncast; assert timestamps.getPositionCount() == values.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef valuesScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -350,7 +350,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page } BytesRefBlock values = (BytesRefBlock) valuesUncast; assert timestamps.getPositionCount() == values.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef valuesScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/LastBytesRefByTimestampAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/LastBytesRefByTimestampAggregatorFunction.java index 0233a00ccd45f..70b05e5ae2910 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/LastBytesRefByTimestampAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/LastBytesRefByTimestampAggregatorFunction.java @@ -237,8 +237,8 @@ public void addIntermediateInput(Page page) { } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert seen.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - LastBytesRefByTimestampAggregator.combineIntermediate(state, timestamps.getLong(0), values.getBytesRef(0, scratch), seen.getBoolean(0)); + BytesRef valuesScratch = new BytesRef(); + LastBytesRefByTimestampAggregator.combineIntermediate(state, timestamps.getLong(0), values.getBytesRef(0, valuesScratch), seen.getBoolean(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/LastBytesRefByTimestampGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/LastBytesRefByTimestampGroupingAggregatorFunction.java index c5ee2afaf04c7..4ee67fdbdb818 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/LastBytesRefByTimestampGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/LastBytesRefByTimestampGroupingAggregatorFunction.java @@ -200,7 +200,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page } BytesRefBlock values = (BytesRefBlock) valuesUncast; assert timestamps.getPositionCount() == values.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef valuesScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -282,7 +282,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa } BytesRefBlock values = (BytesRefBlock) valuesUncast; assert timestamps.getPositionCount() == values.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef valuesScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -350,7 +350,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page } BytesRefBlock values = (BytesRefBlock) valuesUncast; assert timestamps.getPositionCount() == values.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef valuesScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java index 80810713866ce..e606864ba63f4 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefAggregatorFunction.java @@ -156,8 +156,8 @@ public void addIntermediateInput(Page page) { } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert seen.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - MaxBytesRefAggregator.combineIntermediate(state, max.getBytesRef(0, scratch), seen.getBoolean(0)); + BytesRef maxScratch = new BytesRef(); + MaxBytesRefAggregator.combineIntermediate(state, max.getBytesRef(0, maxScratch), seen.getBoolean(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunction.java index 793570db726a9..ab162190fa43d 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxBytesRefGroupingAggregatorFunction.java @@ -164,7 +164,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert max.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef maxScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -174,7 +174,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MaxBytesRefAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MaxBytesRefAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, maxScratch), seen.getBoolean(valuesPosition)); } } } @@ -236,7 +236,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert max.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef maxScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -246,7 +246,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MaxBytesRefAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MaxBytesRefAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, maxScratch), seen.getBoolean(valuesPosition)); } } } @@ -293,11 +293,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert max.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef maxScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - MaxBytesRefAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MaxBytesRefAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, maxScratch), seen.getBoolean(valuesPosition)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunction.java index d192718d14e10..119cdb5917c06 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpAggregatorFunction.java @@ -156,8 +156,8 @@ public void addIntermediateInput(Page page) { } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert seen.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - MaxIpAggregator.combineIntermediate(state, max.getBytesRef(0, scratch), seen.getBoolean(0)); + BytesRef maxScratch = new BytesRef(); + MaxIpAggregator.combineIntermediate(state, max.getBytesRef(0, maxScratch), seen.getBoolean(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpGroupingAggregatorFunction.java index bfee9d26f3996..5de02faac858a 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxIpGroupingAggregatorFunction.java @@ -164,7 +164,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert max.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef maxScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -174,7 +174,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MaxIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MaxIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, maxScratch), seen.getBoolean(valuesPosition)); } } } @@ -236,7 +236,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert max.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef maxScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -246,7 +246,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MaxIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MaxIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, maxScratch), seen.getBoolean(valuesPosition)); } } } @@ -293,11 +293,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert max.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef maxScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - MaxIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MaxIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, maxScratch), seen.getBoolean(valuesPosition)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleAggregatorFunction.java index 6c50628613ae1..43b4170f68d53 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleAggregatorFunction.java @@ -146,8 +146,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); assert quart.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - MedianAbsoluteDeviationDoubleAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + BytesRef quartScratch = new BytesRef(); + MedianAbsoluteDeviationDoubleAggregator.combineIntermediate(state, quart.getBytesRef(0, quartScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunction.java index 3fd6a54970645..27531cb01ea14 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunction.java @@ -155,7 +155,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -165,7 +165,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -218,7 +218,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -228,7 +228,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -267,11 +267,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java index 7efedf93a8910..27a580e1da6f7 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java @@ -146,8 +146,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); assert quart.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + BytesRef quartScratch = new BytesRef(); + MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, quart.getBytesRef(0, quartScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java index c706d6a68b9ad..2f57442e14ea6 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java @@ -155,7 +155,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -165,7 +165,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -218,7 +218,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -228,7 +228,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -267,11 +267,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntAggregatorFunction.java index 73756681f5c18..68c9d8ce75174 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntAggregatorFunction.java @@ -146,8 +146,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); assert quart.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - MedianAbsoluteDeviationIntAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + BytesRef quartScratch = new BytesRef(); + MedianAbsoluteDeviationIntAggregator.combineIntermediate(state, quart.getBytesRef(0, quartScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntGroupingAggregatorFunction.java index 171e4238f68c7..6f6e711157a80 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationIntGroupingAggregatorFunction.java @@ -154,7 +154,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -164,7 +164,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -217,7 +217,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -227,7 +227,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -266,11 +266,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongAggregatorFunction.java index 4c6afaba77234..8f71e4cc249b9 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongAggregatorFunction.java @@ -146,8 +146,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); assert quart.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - MedianAbsoluteDeviationLongAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + BytesRef quartScratch = new BytesRef(); + MedianAbsoluteDeviationLongAggregator.combineIntermediate(state, quart.getBytesRef(0, quartScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongGroupingAggregatorFunction.java index 2136c8d0fd1fd..3a75aa3ba2e73 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationLongGroupingAggregatorFunction.java @@ -155,7 +155,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -165,7 +165,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -218,7 +218,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -228,7 +228,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -267,11 +267,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - MedianAbsoluteDeviationLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + MedianAbsoluteDeviationLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java index 92660be1185a0..a7ab1814d670d 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefAggregatorFunction.java @@ -156,8 +156,8 @@ public void addIntermediateInput(Page page) { } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert seen.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - MinBytesRefAggregator.combineIntermediate(state, min.getBytesRef(0, scratch), seen.getBoolean(0)); + BytesRef minScratch = new BytesRef(); + MinBytesRefAggregator.combineIntermediate(state, min.getBytesRef(0, minScratch), seen.getBoolean(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunction.java index e3acfe1ab1943..03e07ad54cb1f 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinBytesRefGroupingAggregatorFunction.java @@ -164,7 +164,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert min.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef minScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -174,7 +174,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MinBytesRefAggregator.combineIntermediate(state, groupId, min.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MinBytesRefAggregator.combineIntermediate(state, groupId, min.getBytesRef(valuesPosition, minScratch), seen.getBoolean(valuesPosition)); } } } @@ -236,7 +236,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert min.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef minScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -246,7 +246,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MinBytesRefAggregator.combineIntermediate(state, groupId, min.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MinBytesRefAggregator.combineIntermediate(state, groupId, min.getBytesRef(valuesPosition, minScratch), seen.getBoolean(valuesPosition)); } } } @@ -293,11 +293,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert min.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef minScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - MinBytesRefAggregator.combineIntermediate(state, groupId, min.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MinBytesRefAggregator.combineIntermediate(state, groupId, min.getBytesRef(valuesPosition, minScratch), seen.getBoolean(valuesPosition)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpAggregatorFunction.java index db61c405d440e..61898ca139071 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpAggregatorFunction.java @@ -156,8 +156,8 @@ public void addIntermediateInput(Page page) { } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert seen.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - MinIpAggregator.combineIntermediate(state, max.getBytesRef(0, scratch), seen.getBoolean(0)); + BytesRef maxScratch = new BytesRef(); + MinIpAggregator.combineIntermediate(state, max.getBytesRef(0, maxScratch), seen.getBoolean(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpGroupingAggregatorFunction.java index 69b3c227a2dab..ea678434b7dbb 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinIpGroupingAggregatorFunction.java @@ -164,7 +164,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert max.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef maxScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -174,7 +174,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MinIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MinIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, maxScratch), seen.getBoolean(valuesPosition)); } } } @@ -236,7 +236,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert max.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef maxScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -246,7 +246,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - MinIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MinIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, maxScratch), seen.getBoolean(valuesPosition)); } } } @@ -293,11 +293,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert max.getPositionCount() == seen.getPositionCount(); - BytesRef scratch = new BytesRef(); + BytesRef maxScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - MinIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, scratch), seen.getBoolean(valuesPosition)); + MinIpAggregator.combineIntermediate(state, groupId, max.getBytesRef(valuesPosition, maxScratch), seen.getBoolean(valuesPosition)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleAggregatorFunction.java index 7ae73ed7e1eb9..48a54247588d3 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleAggregatorFunction.java @@ -149,8 +149,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); assert quart.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - PercentileDoubleAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + BytesRef quartScratch = new BytesRef(); + PercentileDoubleAggregator.combineIntermediate(state, quart.getBytesRef(0, quartScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleGroupingAggregatorFunction.java index f726423df4d4d..81e2d2189e133 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileDoubleGroupingAggregatorFunction.java @@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -168,7 +168,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - PercentileDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -221,7 +221,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -231,7 +231,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - PercentileDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -270,11 +270,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - PercentileDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileDoubleAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java index 29acca005738d..5b172e43a7cfe 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java @@ -149,8 +149,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); assert quart.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - PercentileFloatAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + BytesRef quartScratch = new BytesRef(); + PercentileFloatAggregator.combineIntermediate(state, quart.getBytesRef(0, quartScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java index b5be9cc7655bd..c5cf407d86dc0 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java @@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -168,7 +168,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - PercentileFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -221,7 +221,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -231,7 +231,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - PercentileFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -270,11 +270,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - PercentileFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntAggregatorFunction.java index 21784c215f3f6..081120592aebd 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntAggregatorFunction.java @@ -149,8 +149,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); assert quart.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - PercentileIntAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + BytesRef quartScratch = new BytesRef(); + PercentileIntAggregator.combineIntermediate(state, quart.getBytesRef(0, quartScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntGroupingAggregatorFunction.java index c94cc37109e4e..a278b28d1d317 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileIntGroupingAggregatorFunction.java @@ -157,7 +157,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -167,7 +167,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - PercentileIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -220,7 +220,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -230,7 +230,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - PercentileIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -269,11 +269,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - PercentileIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileIntAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongAggregatorFunction.java index df49223605ec2..6d81e228ca662 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongAggregatorFunction.java @@ -149,8 +149,8 @@ public void addIntermediateInput(Page page) { } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); assert quart.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - PercentileLongAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + BytesRef quartScratch = new BytesRef(); + PercentileLongAggregator.combineIntermediate(state, quart.getBytesRef(0, quartScratch)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongGroupingAggregatorFunction.java index d48770a77abf2..2f22604be77c4 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileLongGroupingAggregatorFunction.java @@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -168,7 +168,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - PercentileLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -221,7 +221,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -231,7 +231,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); int valuesPosition = groupPosition + positionOffset; - PercentileLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } } @@ -270,11 +270,11 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); - BytesRef scratch = new BytesRef(); + BytesRef quartScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; - PercentileLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, scratch)); + PercentileLongAggregator.combineIntermediate(state, groupId, quart.getBytesRef(valuesPosition, quartScratch)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBooleanAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBooleanAggregatorFunction.java index a5cfcbb1e03e2..9ce0bffb4e642 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBooleanAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBooleanAggregatorFunction.java @@ -147,7 +147,7 @@ public void addIntermediateInput(Page page) { } BytesRefBlock sample = (BytesRefBlock) sampleUncast; assert sample.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); SampleBooleanAggregator.combineIntermediate(state, sample); } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBooleanGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBooleanGroupingAggregatorFunction.java index c675030a16bf2..c2d86ef1a9e54 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBooleanGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBooleanGroupingAggregatorFunction.java @@ -157,7 +157,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -220,7 +220,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -269,7 +269,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBytesRefAggregatorFunction.java index 45e55a2f9a5a3..dc3af45e9bec2 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBytesRefAggregatorFunction.java @@ -151,7 +151,7 @@ public void addIntermediateInput(Page page) { } BytesRefBlock sample = (BytesRefBlock) sampleUncast; assert sample.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); SampleBytesRefAggregator.combineIntermediate(state, sample); } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBytesRefGroupingAggregatorFunction.java index db0d675499c44..ae911a74c4ca4 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBytesRefGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleBytesRefGroupingAggregatorFunction.java @@ -158,7 +158,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -224,7 +224,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -275,7 +275,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleDoubleAggregatorFunction.java index fc5681e427572..e63c9fe414ce7 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleDoubleAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleDoubleAggregatorFunction.java @@ -148,7 +148,7 @@ public void addIntermediateInput(Page page) { } BytesRefBlock sample = (BytesRefBlock) sampleUncast; assert sample.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); SampleDoubleAggregator.combineIntermediate(state, sample); } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleDoubleGroupingAggregatorFunction.java index b81c56db60297..f62100ee2e0ec 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleDoubleGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleDoubleGroupingAggregatorFunction.java @@ -157,7 +157,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -220,7 +220,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -269,7 +269,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleIntAggregatorFunction.java index 232a664854401..b4035708a567b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleIntAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleIntAggregatorFunction.java @@ -148,7 +148,7 @@ public void addIntermediateInput(Page page) { } BytesRefBlock sample = (BytesRefBlock) sampleUncast; assert sample.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); SampleIntAggregator.combineIntermediate(state, sample); } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleIntGroupingAggregatorFunction.java index 41f2f7a608e8e..199676fa2fd2d 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleIntGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleIntGroupingAggregatorFunction.java @@ -156,7 +156,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -219,7 +219,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -268,7 +268,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleLongAggregatorFunction.java index df885bd2292db..b78d48381ef8e 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleLongAggregatorFunction.java @@ -148,7 +148,7 @@ public void addIntermediateInput(Page page) { } BytesRefBlock sample = (BytesRefBlock) sampleUncast; assert sample.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); SampleLongAggregator.combineIntermediate(state, sample); } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleLongGroupingAggregatorFunction.java index 675827ad0caa3..4310849a9ebb7 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleLongGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SampleLongGroupingAggregatorFunction.java @@ -157,7 +157,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -220,7 +220,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -269,7 +269,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefBlock sample = (BytesRefBlock) sampleUncast; - BytesRef scratch = new BytesRef(); + BytesRef sampleScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefAggregatorFunction.java index c9aad2f067c4c..5042ba27a83a6 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefAggregatorFunction.java @@ -154,7 +154,7 @@ public void addIntermediateInput(Page page) { } BytesRefBlock top = (BytesRefBlock) topUncast; assert top.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); + BytesRef topScratch = new BytesRef(); TopBytesRefAggregator.combineIntermediate(state, top); } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefGroupingAggregatorFunction.java index dc6db83fc0f0d..676c70b480abe 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopBytesRefGroupingAggregatorFunction.java @@ -162,7 +162,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefBlock top = (BytesRefBlock) topUncast; - BytesRef scratch = new BytesRef(); + BytesRef topScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -227,7 +227,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefBlock top = (BytesRefBlock) topUncast; - BytesRef scratch = new BytesRef(); + BytesRef topScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -278,7 +278,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefBlock top = (BytesRefBlock) topUncast; - BytesRef scratch = new BytesRef(); + BytesRef topScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunction.java index f5781c44f5e65..38c6e2c88c9f5 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpAggregatorFunction.java @@ -154,7 +154,7 @@ public void addIntermediateInput(Page page) { } BytesRefBlock top = (BytesRefBlock) topUncast; assert top.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); + BytesRef topScratch = new BytesRef(); TopIpAggregator.combineIntermediate(state, top); } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunction.java index 18a6925ebbb34..a813402811302 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopIpGroupingAggregatorFunction.java @@ -162,7 +162,7 @@ public void addIntermediateInput(int positionOffset, IntArrayBlock groups, Page return; } BytesRefBlock top = (BytesRefBlock) topUncast; - BytesRef scratch = new BytesRef(); + BytesRef topScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -227,7 +227,7 @@ public void addIntermediateInput(int positionOffset, IntBigArrayBlock groups, Pa return; } BytesRefBlock top = (BytesRefBlock) topUncast; - BytesRef scratch = new BytesRef(); + BytesRef topScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { continue; @@ -278,7 +278,7 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefBlock top = (BytesRefBlock) topUncast; - BytesRef scratch = new BytesRef(); + BytesRef topScratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); int valuesPosition = groupPosition + positionOffset; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregatorFunction.java index 048467dddad48..d5da83e215f0c 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesBytesRefAggregatorFunction.java @@ -148,7 +148,7 @@ public void addIntermediateInput(Page page) { } BytesRefBlock values = (BytesRefBlock) valuesUncast; assert values.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); + BytesRef valuesScratch = new BytesRef(); ValuesBytesRefAggregator.combineIntermediate(state, values); } From 20ce72d7b9a4a1c6848c11ca4a372cf4b95689fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Mon, 3 Nov 2025 17:26:07 +0100 Subject: [PATCH 31/32] Unmute FullClusterRestartIT tests (#137254) --- muted-tests.yml | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 3b420b5e51563..d43272b21542a 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -417,33 +417,6 @@ tests: - class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT method: test {p0=field_caps/10_basic/Field caps for number field with only doc values} issue: https://github.com/elastic/elasticsearch/issues/136244 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testDataStreams {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/136353 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testServiceAccountApiKey {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/136390 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testWatcher {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/136391 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testSingleDoc {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/136392 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testTransformLegacyTemplateCleanup {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/136393 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testApiKeySuperuser {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/136394 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testSecurityNativeRealm {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/136395 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testSlmPolicyAndStats {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/136399 -- class: org.elasticsearch.xpack.restart.FullClusterRestartIT - method: testRollupAfterRestart {cluster=UPGRADED} - issue: https://github.com/elastic/elasticsearch/issues/136437 - class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT method: test {p0=search.vectors/200_dense_vector_docvalue_fields/Enable docvalue_fields parameter for dense_vector fields} issue: https://github.com/elastic/elasticsearch/issues/136443 From c925aa0a332a15d1bf743dcd6c802c5f1a657123 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 3 Nov 2025 16:55:10 +0000 Subject: [PATCH 32/32] Clean up `TransportResizeAction` (#137471) Mainly this is about removing the trappy implicit default master node timeout in resize (shrink/split/clone) requests as per #107984, but also we drop the unnecessary `ActionType` subclass and the test-only `ResizeRequestBuilder`, migrating the affected tests over to a new `ResizeIndexTestUtils` instead. We also fix the order of index-name arguments and make the `ResizeType` a mandatory parameter rather than (trappily) defaulting to `SHRINK`. --- .../TSDBPassthroughIndexingIT.java | 12 +- .../admin/indices/create/CloneIndexIT.java | 103 +++++++++------- .../admin/indices/create/ShrinkIndexIT.java | 115 ++++++++++-------- .../admin/indices/create/SplitIndexIT.java | 50 ++++---- .../decider/DiskThresholdDeciderIT.java | 20 ++- ...sAvailabilityHealthIndicatorServiceIT.java | 3 +- .../index/LookupIndexModeIT.java | 44 ++++--- .../routing/PartitionedRoutingIT.java | 11 +- .../DedicatedClusterSnapshotRestoreIT.java | 4 +- .../elasticsearch/action/ActionModule.java | 3 +- .../admin/indices/shrink/ResizeAction.java | 24 ---- .../admin/indices/shrink/ResizeRequest.java | 62 +++------- .../indices/shrink/ResizeRequestBuilder.java | 79 ------------ .../indices/shrink/TransportResizeAction.java | 5 +- .../client/internal/IndicesAdminClient.java | 11 -- .../admin/indices/RestResizeHandler.java | 14 ++- .../indices/shrink/ResizeRequestTests.java | 82 +++++++++++-- .../shrink/TransportResizeActionTests.java | 97 +++++++-------- .../admin/indices/ResizeIndexTestUtils.java | 39 ++++++ .../storage/ReactiveStorageIT.java | 29 ++--- .../DataTierAllocationDeciderIT.java | 8 +- .../xpack/core/ilm/ResizeIndexStep.java | 14 ++- .../xpack/core/ilm/ResizeIndexStepTests.java | 39 +++--- .../xpack/logsdb/LogsIndexingIT.java | 11 +- .../FrozenSearchableSnapshotsIntegTests.java | 19 +-- .../SearchableSnapshotsIntegTests.java | 17 +-- .../SearchableSnapshotsResizeIntegTests.java | 45 ++++--- .../ShrinkIndexWithSecurityTests.java | 5 +- .../ResizeRequestInterceptorTests.java | 24 +++- 29 files changed, 482 insertions(+), 507 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java delete mode 100644 server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java create mode 100644 test/framework/src/main/java/org/elasticsearch/action/admin/indices/ResizeIndexTestUtils.java diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java index 27cfc62822441..fe99d78fa6823 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.delete.DeleteRequest; @@ -44,6 +45,7 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.resizeRequest; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; @@ -267,12 +269,10 @@ public void testIndexingGettingAndSearchingShrunkIndex() throws Exception { assertThat(updateSettingsResponse.isAcknowledged(), is(true)); String shrunkenTarget = "k8s-shrunken"; - var shrinkIndexResponse = client().admin() - .indices() - .prepareResizeIndex(sourceIndex, shrunkenTarget) - .setResizeType(ResizeType.SHRINK) - .setSettings(indexSettings(2, 0).build()) - .get(); + final var shrinkIndexResponse = client().execute( + TransportResizeAction.TYPE, + resizeRequest(ResizeType.SHRINK, sourceIndex, shrunkenTarget, indexSettings(2, 0)) + ).actionGet(); assertThat(shrinkIndexResponse.isAcknowledged(), is(true)); assertThat(shrinkIndexResponse.index(), equalTo(shrunkenTarget)); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java index 654a1393aa146..0e25b1071a271 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CloneIndexIT.java @@ -25,6 +25,7 @@ import java.util.List; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.executeResize; import static org.elasticsearch.action.admin.indices.create.ShrinkIndexIT.assertNoResizeSourceIndexSettings; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; @@ -67,11 +68,12 @@ public void testCreateCloneIndex() { final boolean createWithReplicas = randomBoolean(); assertAcked( - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.CLONE) - .setSettings( - Settings.builder().put("index.number_of_replicas", createWithReplicas ? 1 : 0).putNull("index.blocks.write").build() - ) + executeResize( + ResizeType.CLONE, + "source", + "target", + Settings.builder().put("index.number_of_replicas", createWithReplicas ? 1 : 0).putNull("index.blocks.write") + ) ); ensureGreen(); assertNoResizeSourceIndexSettings("target"); @@ -125,9 +127,10 @@ public void testResizeChangeIndexMode() { Settings.builder().put("index.mode", "lookup").build() ); for (Settings settings : indexSettings) { - IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> { - indicesAdmin().prepareResizeIndex("source", "target").setResizeType(ResizeType.CLONE).setSettings(settings).get(); - }); + IllegalArgumentException error = expectThrows( + IllegalArgumentException.class, + () -> executeResize(ResizeType.CLONE, "source", "target", Settings.builder().put(settings)).actionGet() + ); assertThat(error.getMessage(), equalTo("can't change setting [index.mode] during resize")); } } @@ -137,12 +140,15 @@ public void testResizeChangeSyntheticSource() { .setMapping("@timestamp", "type=date", "host.name", "type=keyword") .get(); updateIndexSettings(Settings.builder().put("index.blocks.write", true), "source"); - IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> { - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.CLONE) - .setSettings(Settings.builder().put("index.mapping.source.mode", "synthetic").putNull("index.blocks.write").build()) - .get(); - }); + IllegalArgumentException error = expectThrows( + IllegalArgumentException.class, + () -> executeResize( + ResizeType.CLONE, + "source", + "target", + Settings.builder().put("index.mapping.source.mode", "synthetic").putNull("index.blocks.write") + ).actionGet() + ); assertThat(error.getMessage(), containsString("can't change setting [index.mapping.source.mode] during resize")); } @@ -159,26 +165,26 @@ public void testResizeChangeRecoveryUseSyntheticSource() { ) ).setMapping("@timestamp", "type=date", "host.name", "type=keyword").get(); updateIndexSettings(Settings.builder().put("index.blocks.write", true), "source"); - IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> { - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.CLONE) - .setSettings( - Settings.builder() - .put( - "index.version.created", - IndexVersionUtils.randomVersionBetween( - random(), - IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY, - IndexVersion.current() - ) + IllegalArgumentException error = expectThrows( + IllegalArgumentException.class, + () -> executeResize( + ResizeType.CLONE, + "source", + "target", + Settings.builder() + .put( + "index.version.created", + IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.USE_SYNTHETIC_SOURCE_FOR_RECOVERY, + IndexVersion.current() ) - .put("index.recovery.use_synthetic_source", true) - .put("index.mode", "logsdb") - .putNull("index.blocks.write") - .build() - ) - .get(); - }); + ) + .put("index.recovery.use_synthetic_source", true) + .put("index.mode", "logsdb") + .putNull("index.blocks.write") + ).actionGet() + ); // The index.recovery.use_synthetic_source setting requires either index.mode or index.mapping.source.mode // to be present in the settings. Since these are all unmodifiable settings with a non-deterministic evaluation // order, any of them may trigger a failure first. @@ -196,12 +202,11 @@ public void testResizeChangeIndexSorts() { .setMapping("@timestamp", "type=date", "host.name", "type=keyword") .get(); updateIndexSettings(Settings.builder().put("index.blocks.write", true), "source"); - ValidationException error = expectThrows(ValidationException.class, () -> { - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.CLONE) - .setSettings(Settings.builder().putList("index.sort.field", List.of("@timestamp")).build()) - .get(); - }); + ValidationException error = expectThrows( + ValidationException.class, + () -> executeResize(ResizeType.CLONE, "source", "target", Settings.builder().putList("index.sort.field", List.of("@timestamp"))) + .actionGet() + ); assertThat(error.getMessage(), containsString("can't override index sort when resizing an index")); } @@ -216,11 +221,13 @@ public void testCloneLogsdbIndexWithNonDefaultTimestamp() { ensureGreen(); // Clone the index - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.CLONE) + executeResize( + ResizeType.CLONE, + "source", + "target", // We need to explicitly set the number of replicas in case the source has 0 replicas and the cluster has only 1 data node - .setSettings(Settings.builder().put("index.number_of_replicas", numberOfReplicas).build()) - .get(); + Settings.builder().put("index.number_of_replicas", numberOfReplicas) + ).actionGet(); // Verify that the target index has the correct @timestamp mapping final var targetMappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "target").get(); @@ -246,11 +253,13 @@ public void testCloneTimeSeriesIndexWithNonDefaultTimestamp() { ensureGreen(); // Clone the index - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.CLONE) + executeResize( + ResizeType.CLONE, + "source", + "target", // We need to explicitly set the number of replicas in case the source has 0 replicas and the cluster has only 1 data node - .setSettings(Settings.builder().put("index.number_of_replicas", numberOfReplicas).build()) - .get(); + Settings.builder().put("index.number_of_replicas", numberOfReplicas) + ).actionGet(); // Verify that the target index has the correct @timestamp mapping final var targetMappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "target").get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java index 31831f7eb6fe5..75f6a949c976d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java @@ -18,11 +18,13 @@ import org.elasticsearch.action.admin.cluster.reroute.TransportClusterRerouteAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.indices.ResizeIndexTestUtils; import org.elasticsearch.action.admin.indices.segments.IndexShardSegments; import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse; import org.elasticsearch.action.admin.indices.segments.ShardSegments; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; @@ -58,6 +60,7 @@ import java.util.Map; import java.util.stream.IntStream; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.executeResize; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.containsString; @@ -102,8 +105,7 @@ public void testCreateShrinkIndexToN() { ensureGreen(); // now merge source into a 4 shard index assertAcked( - indicesAdmin().prepareResizeIndex("source", "first_shrink") - .setSettings(indexSettings(shardSplits[1], 0).putNull("index.blocks.write").build()) + executeResize(ResizeType.SHRINK, "source", "first_shrink", indexSettings(shardSplits[1], 0).putNull("index.blocks.write")) ); ensureGreen(); assertHitCount(prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), 20); @@ -128,10 +130,12 @@ public void testCreateShrinkIndexToN() { ensureGreen(); // now merge source into a 2 shard index assertAcked( - indicesAdmin().prepareResizeIndex("first_shrink", "second_shrink") - .setSettings( - indexSettings(shardSplits[2], 0).putNull("index.blocks.write").putNull("index.routing.allocation.require._name").build() - ) + executeResize( + ResizeType.SHRINK, + "first_shrink", + "second_shrink", + indexSettings(shardSplits[2], 0).putNull("index.blocks.write").putNull("index.routing.allocation.require._name") + ) ); ensureGreen(); assertHitCount(prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), 20); @@ -221,8 +225,7 @@ public void testShrinkIndexPrimaryTerm() throws Exception { final long beforeShrinkPrimaryTerm = IntStream.range(0, numberOfShards).mapToLong(indexMetadata::primaryTerm).max().getAsLong(); // now merge source into target - final Settings shrinkSettings = indexSettings(numberOfTargetShards, 0).build(); - assertAcked(indicesAdmin().prepareResizeIndex("source", "target").setSettings(shrinkSettings).get()); + assertAcked(executeResize(ResizeType.SHRINK, "source", "target", indexSettings(numberOfTargetShards, 0))); ensureGreen(TimeValue.timeValueSeconds(120)); @@ -273,15 +276,17 @@ public void testCreateShrinkIndex() { // now merge source into a single shard index final boolean createWithReplicas = randomBoolean(); assertAcked( - indicesAdmin().prepareResizeIndex("source", "target") - .setSettings( - Settings.builder() - .put("index.number_of_replicas", createWithReplicas ? 1 : 0) - .putNull("index.blocks.write") - .putNull("index.routing.allocation.require._name") - .build() - ) + executeResize( + ResizeType.SHRINK, + "source", + "target", + Settings.builder() + .put("index.number_of_replicas", createWithReplicas ? 1 : 0) + .putNull("index.blocks.write") + .putNull("index.routing.allocation.require._name") + ) ); + ensureGreen(); assertNoResizeSourceIndexSettings("target"); @@ -374,16 +379,17 @@ public void testCreateShrinkIndexFails() throws Exception { ensureGreen(); // now merge source into a single shard index - indicesAdmin().prepareResizeIndex("source", "target") - .setWaitForActiveShards(ActiveShardCount.NONE) - .setSettings( - Settings.builder() - .put("index.routing.allocation.exclude._name", mergeNode) // we manually exclude the merge node to forcefully fuck it up - .put("index.number_of_replicas", 0) - .put("index.allocation.max_retries", 1) - .build() - ) - .get(); + final var resizeRequest = ResizeIndexTestUtils.resizeRequest( + ResizeType.SHRINK, + "source", + "target", + Settings.builder() + .put("index.routing.allocation.exclude._name", mergeNode) // we manually exclude the merge node to forcefully mess it up + .put("index.number_of_replicas", 0) + .put("index.allocation.max_retries", 1) + ); + resizeRequest.setWaitForActiveShards(ActiveShardCount.NONE); + client().execute(TransportResizeAction.TYPE, resizeRequest).actionGet(); clusterAdmin().prepareHealth(TEST_REQUEST_TIMEOUT, "target").setWaitForEvents(Priority.LANGUID).get(); // now we move all shards away from the merge node @@ -464,14 +470,12 @@ public void testCreateShrinkWithIndexSort() throws Exception { // check that index sort cannot be set on the target index IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, - indicesAdmin().prepareResizeIndex("source", "target").setSettings(indexSettings(2, 0).put("index.sort.field", "foo").build()) + executeResize(ResizeType.SHRINK, "source", "target", indexSettings(2, 0).put("index.sort.field", "foo")) ); assertThat(exc.getMessage(), containsString("can't override index sort when resizing an index")); // check that the index sort order of `source` is correctly applied to the `target` - assertAcked( - indicesAdmin().prepareResizeIndex("source", "target").setSettings(indexSettings(2, 0).putNull("index.blocks.write").build()) - ); + assertAcked(executeResize(ResizeType.SHRINK, "source", "target", indexSettings(2, 0).putNull("index.blocks.write"))); ensureGreen(); assertNoResizeSourceIndexSettings("target"); @@ -516,10 +520,7 @@ public void testShrinkCommitsMergeOnIdle() throws Exception { updateClusterSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none")); try { // now merge source into a single shard index - assertAcked( - indicesAdmin().prepareResizeIndex("source", "target") - .setSettings(Settings.builder().put("index.number_of_replicas", 0).build()) - ); + assertAcked(executeResize(ResizeType.SHRINK, "source", "target", Settings.builder().put("index.number_of_replicas", 0))); ensureGreen(); assertNoResizeSourceIndexSettings("target"); @@ -584,13 +585,14 @@ public void testShrinkThenSplitWithFailedNode() throws Exception { ensureGreen(); assertAcked( - indicesAdmin().prepareResizeIndex("original", "shrunk") - .setSettings( - indexSettings(1, 1).putNull( - IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey() - ).build() + executeResize( + ResizeType.SHRINK, + "original", + "shrunk", + indexSettings(1, 1).putNull( + IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey() ) - .setResizeType(ResizeType.SHRINK) + ) ); ensureGreen(); @@ -603,13 +605,14 @@ public void testShrinkThenSplitWithFailedNode() throws Exception { logger.info("--> executing split"); assertAcked( - indicesAdmin().prepareResizeIndex("shrunk", "splitagain") - .setSettings( - indexSettings(shardCount, 0).putNull( - IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey() - ).build() + executeResize( + ResizeType.SPLIT, + "shrunk", + "splitagain", + indexSettings(shardCount, 0).putNull( + IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey() ) - .setResizeType(ResizeType.SPLIT) + ) ); ensureGreen("splitagain"); assertNoResizeSourceIndexSettings("splitagain"); @@ -627,11 +630,13 @@ public void testShrinkLogsdbIndexWithNonDefaultTimestamp() { ensureGreen(); // Shrink the index - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.SHRINK) + executeResize( + ResizeType.SHRINK, + "source", + "target", // We need to explicitly set the number of replicas in case the source has 0 replicas and the cluster has only 1 data node - .setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()) - .get(); + indexSettings(1, 0) + ).actionGet(); // Verify that the target index has the correct @timestamp mapping final var targetMappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "target").get(); @@ -657,11 +662,13 @@ public void testShrinkTimeSeriesIndexWithNonDefaultTimestamp() { ensureGreen(); // Shrink the index - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.SHRINK) + executeResize( + ResizeType.SHRINK, + "source", + "target", // We need to explicitly set the number of replicas in case the source has 0 replicas and the cluster has only 1 data node - .setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build()) - .get(); + indexSettings(1, 0) + ).actionGet(); // Verify that the target index has the correct @timestamp mapping final var targetMappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "target").get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java index 5055430310703..ffa479bfe1ff2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java @@ -57,6 +57,7 @@ import java.util.function.BiFunction; import java.util.stream.IntStream; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.executeResize; import static org.elasticsearch.action.admin.indices.create.ShrinkIndexIT.assertNoResizeSourceIndexSettings; import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; @@ -184,11 +185,7 @@ private void splitToN(int sourceShards, int firstSplitShards, int secondSplitSha if (sourceShards == 1 && useRoutingPartition == false && randomBoolean()) { // try to set it if we have a source index with 1 shard firstSplitSettingsBuilder.put("index.number_of_routing_shards", secondSplitShards); } - assertAcked( - indicesAdmin().prepareResizeIndex("source", "first_split") - .setResizeType(ResizeType.SPLIT) - .setSettings(firstSplitSettingsBuilder.build()) - ); + assertAcked(executeResize(ResizeType.SPLIT, "source", "first_split", firstSplitSettingsBuilder)); ensureGreen(); assertHitCount(prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), numDocs); assertNoResizeSourceIndexSettings("first_split"); @@ -212,9 +209,12 @@ private void splitToN(int sourceShards, int firstSplitShards, int secondSplitSha ensureGreen(); // now split source into a new index assertAcked( - indicesAdmin().prepareResizeIndex("first_split", "second_split") - .setResizeType(ResizeType.SPLIT) - .setSettings(indexSettings(secondSplitShards, 0).putNull("index.blocks.write").build()) + executeResize( + ResizeType.SPLIT, + "first_split", + "second_split", + indexSettings(secondSplitShards, 0).putNull("index.blocks.write") + ) ); ensureGreen(); assertHitCount(prepareSearch("second_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")), numDocs); @@ -325,8 +325,9 @@ public void testSplitIndexPrimaryTerm() throws Exception { final long beforeSplitPrimaryTerm = IntStream.range(0, numberOfShards).mapToLong(indexMetadata::primaryTerm).max().getAsLong(); // now split source into target - final Settings splitSettings = indexSettings(numberOfTargetShards, 0).putNull("index.blocks.write").build(); - assertAcked(indicesAdmin().prepareResizeIndex("source", "target").setResizeType(ResizeType.SPLIT).setSettings(splitSettings).get()); + assertAcked( + executeResize(ResizeType.SPLIT, "source", "target", indexSettings(numberOfTargetShards, 0).putNull("index.blocks.write")) + ); ensureGreen(TimeValue.timeValueSeconds(120)); // needs more than the default to relocate many shards @@ -372,9 +373,12 @@ public void testCreateSplitIndex() { final boolean createWithReplicas = randomBoolean(); assertAcked( - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.SPLIT) - .setSettings(indexSettings(2, createWithReplicas ? 1 : 0).putNull("index.blocks.write").build()) + executeResize( + ResizeType.SPLIT, + "source", + "target", + indexSettings(2, createWithReplicas ? 1 : 0).putNull("index.blocks.write") + ) ); ensureGreen(); assertNoResizeSourceIndexSettings("target"); @@ -470,18 +474,12 @@ public void testCreateSplitWithIndexSort() throws Exception { // check that index sort cannot be set on the target index IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.SPLIT) - .setSettings(indexSettings(4, 0).put("index.sort.field", "foo").build()) + executeResize(ResizeType.SPLIT, "source", "target", indexSettings(4, 0).put("index.sort.field", "foo")) ); assertThat(exc.getMessage(), containsString("can't override index sort when resizing an index")); // check that the index sort order of `source` is correctly applied to the `target` - assertAcked( - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.SPLIT) - .setSettings(indexSettings(4, 0).putNull("index.blocks.write").build()) - ); + assertAcked(executeResize(ResizeType.SPLIT, "source", "target", indexSettings(4, 0).putNull("index.blocks.write"))); ensureGreen(); flushAndRefresh(); GetSettingsResponse settingsResponse = indicesAdmin().prepareGetSettings(TEST_REQUEST_TIMEOUT, "target").get(); @@ -510,11 +508,13 @@ public void testSplitLogsdbIndexWithNonDefaultTimestamp() { ensureGreen(); // Split the index - indicesAdmin().prepareResizeIndex("source", "target") - .setResizeType(ResizeType.SPLIT) + executeResize( + ResizeType.SPLIT, + "source", + "target", // We need to explicitly set the number of replicas in case the source has 0 replicas and the cluster has only 1 data node - .setSettings(Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", numberOfReplicas).build()) - .get(); + indexSettings(2, numberOfReplicas) + ).actionGet(); // Verify that the target index has the correct @timestamp mapping final var targetMappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "target").get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java index 8918d995646b8..560e01187febc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java @@ -12,9 +12,10 @@ import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; +import org.elasticsearch.action.admin.indices.ResizeIndexTestUtils; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.ActiveShardCount; @@ -23,7 +24,6 @@ import org.elasticsearch.cluster.ClusterInfoServiceUtils; import org.elasticsearch.cluster.DiskUsageIntegTestCase; import org.elasticsearch.cluster.InternalClusterInfoService; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; @@ -143,18 +143,16 @@ public void testAllocateCloneIgnoresLowWatermark() throws Exception { refreshDiskUsage(); final var targetIndexName = "target-" + randomIdentifier(); - final var resizeRequest = new ResizeRequest(targetIndexName, sourceIndexName); - resizeRequest.setResizeType(ResizeType.CLONE); - resizeRequest.masterNodeTimeout(TEST_REQUEST_TIMEOUT); - resizeRequest.ackTimeout(TEST_REQUEST_TIMEOUT); + final var resizeRequest = ResizeIndexTestUtils.resizeRequest( + ResizeType.CLONE, + sourceIndexName, + targetIndexName, + indexSettings(1, 0) + ); resizeRequest.setWaitForActiveShards(ActiveShardCount.ALL); - resizeRequest.getTargetIndexRequest() - .settings( - Settings.builder().put(resizeRequest.getTargetIndexRequest().settings()).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - ); safeAwait( - SubscribableListener.newForked(l -> indicesAdmin().resizeIndex(resizeRequest, l)) + SubscribableListener.newForked(l -> client().execute(TransportResizeAction.TYPE, resizeRequest, l)) .andThenAccept( createIndexResponse -> assertThat( true, diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceIT.java index e3586f330a32e..dda581d15a9e0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.cluster.routing.allocation.shards; +import org.elasticsearch.action.admin.indices.ResizeIndexTestUtils; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterStateListener; @@ -109,7 +110,7 @@ public void testIsGreenDuringIndexClone() { updateIndexSettings(Settings.builder().put("index.blocks.write", true), sourceIndex); assertHealthDuring(equalTo(GREEN), () -> { - indicesAdmin().prepareResizeIndex(sourceIndex, targetIndex).setResizeType(ResizeType.CLONE).get(); + ResizeIndexTestUtils.executeResize(ResizeType.CLONE, sourceIndex, targetIndex, Settings.builder()).actionGet(); ensureGreen(targetIndex); }); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java index 583ab18d4f8b3..aec61b877766e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java @@ -9,11 +9,12 @@ package org.elasticsearch.index; +import org.elasticsearch.action.admin.indices.ResizeIndexTestUtils; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; -import org.elasticsearch.action.admin.indices.shrink.ResizeAction; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponse; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; import org.elasticsearch.action.search.SearchResponse; @@ -140,9 +141,7 @@ public void testResizeLookupIndex() { assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createIndexRequest)); client().admin().indices().prepareAddBlock(IndexMetadata.APIBlock.WRITE, "lookup-1").get(); - ResizeRequest clone = new ResizeRequest("lookup-2", "lookup-1"); - clone.setResizeType(ResizeType.CLONE); - assertAcked(client().admin().indices().execute(ResizeAction.INSTANCE, clone).actionGet()); + assertAcked(ResizeIndexTestUtils.executeResize(ResizeType.CLONE, "lookup-1", "lookup-2", Settings.builder())); Settings settings = client().admin() .indices() .prepareGetSettings(TEST_REQUEST_TIMEOUT, "lookup-2") @@ -152,12 +151,15 @@ public void testResizeLookupIndex() { assertThat(settings.get("index.mode"), equalTo("lookup")); assertThat(settings.get("index.number_of_shards"), equalTo("1")); - ResizeRequest split = new ResizeRequest("lookup-3", "lookup-1"); - split.setResizeType(ResizeType.SPLIT); - split.getTargetIndexRequest().settings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3)); + ResizeRequest split = ResizeIndexTestUtils.resizeRequest( + ResizeType.SPLIT, + "lookup-1", + "lookup-3", + Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3) + ); IllegalArgumentException error = expectThrows( IllegalArgumentException.class, - () -> client().admin().indices().execute(ResizeAction.INSTANCE, split).actionGet() + () -> client().execute(TransportResizeAction.TYPE, split).actionGet() ); assertThat( error.getMessage(), @@ -186,27 +188,29 @@ public void testResizeRegularIndexToLookup() { .setSettings(Settings.builder().put("index.number_of_replicas", 0)) .get(); - ResizeRequest clone = new ResizeRequest("lookup-3", "regular-1"); - clone.setResizeType(ResizeType.CLONE); - clone.getTargetIndexRequest().settings(Settings.builder().put("index.mode", "lookup")); + ResizeRequest clone = ResizeIndexTestUtils.resizeRequest( + ResizeType.CLONE, + "regular-1", + "lookup-3", + Settings.builder().put("index.mode", "lookup") + ); IllegalArgumentException error = expectThrows( IllegalArgumentException.class, - () -> client().admin().indices().execute(ResizeAction.INSTANCE, clone).actionGet() + () -> client().execute(TransportResizeAction.TYPE, clone).actionGet() ); assertThat( error.getMessage(), equalTo("index with [lookup] mode must have [index.number_of_shards] set to 1 or unset; provided 2") ); - ResizeRequest shrink = new ResizeRequest("lookup-4", "regular-1"); - shrink.setResizeType(ResizeType.SHRINK); - shrink.getTargetIndexRequest() - .settings(Settings.builder().put("index.mode", "lookup").put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)); - - error = expectThrows( - IllegalArgumentException.class, - () -> client().admin().indices().execute(ResizeAction.INSTANCE, shrink).actionGet() + ResizeRequest shrink = ResizeIndexTestUtils.resizeRequest( + ResizeType.SHRINK, + "regular-1", + "lookup-4", + Settings.builder().put("index.mode", "lookup").put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) ); + + error = expectThrows(IllegalArgumentException.class, () -> client().execute(TransportResizeAction.TYPE, shrink).actionGet()); assertThat(error.getMessage(), equalTo("can't change setting [index.mode] during resize")); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java index 68bc6656cec7f..adda57d8d290d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/routing/PartitionedRoutingIT.java @@ -10,6 +10,7 @@ package org.elasticsearch.routing; import org.apache.lucene.util.Constants; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.IndexScopedSettings; @@ -23,6 +24,7 @@ import java.util.Map; import java.util.Set; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.executeResize; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.hamcrest.CoreMatchers.containsString; @@ -117,9 +119,12 @@ public void testShrinking() throws Exception { index = "index_" + currentShards; logger.info("--> shrinking index [" + previousIndex + "] to [" + index + "]"); - indicesAdmin().prepareResizeIndex(previousIndex, index) - .setSettings(indexSettings(currentShards, numberOfReplicas()).putNull("index.routing.allocation.require._name").build()) - .get(); + executeResize( + ResizeType.SHRINK, + previousIndex, + index, + indexSettings(currentShards, numberOfReplicas()).putNull("index.routing.allocation.require._name") + ).actionGet(); ensureGreen(); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 46d665fde38e8..42ab675008d94 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -18,6 +18,8 @@ import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStats; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; +import org.elasticsearch.action.admin.indices.ResizeIndexTestUtils; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActionTestUtils; @@ -653,7 +655,7 @@ public void testRestoreShrinkIndex() throws Exception { logger.info("--> shrink the index"); updateIndexSettings(Settings.builder().put("index.blocks.write", true), sourceIdx); - assertAcked(indicesAdmin().prepareResizeIndex(sourceIdx, shrunkIdx).get()); + assertAcked(ResizeIndexTestUtils.executeResize(ResizeType.SHRINK, sourceIdx, shrunkIdx, Settings.builder())); logger.info("--> snapshot the shrunk index"); createSnapshot(repo, snapshot, Collections.singletonList(shrunkIdx)); diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 8074fee2826f7..7ac39ab974e3a 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -151,7 +151,6 @@ import org.elasticsearch.action.admin.indices.settings.get.TransportGetSettingsAction; import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction; import org.elasticsearch.action.admin.indices.shards.TransportIndicesShardStoresAction; -import org.elasticsearch.action.admin.indices.shrink.ResizeAction; import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.admin.indices.stats.FieldUsageStatsAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; @@ -705,7 +704,7 @@ public void reg actions.register(IndicesSegmentsAction.INSTANCE, TransportIndicesSegmentsAction.class); actions.register(TransportIndicesShardStoresAction.TYPE, TransportIndicesShardStoresAction.class); actions.register(TransportCreateIndexAction.TYPE, TransportCreateIndexAction.class); - actions.register(ResizeAction.INSTANCE, TransportResizeAction.class); + actions.register(TransportResizeAction.TYPE, TransportResizeAction.class); actions.register(RolloverAction.INSTANCE, TransportRolloverAction.class); actions.register(LazyRolloverAction.INSTANCE, LazyRolloverAction.TransportLazyRolloverAction.class); actions.register(TransportDeleteIndexAction.TYPE, TransportDeleteIndexAction.class); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java deleted file mode 100644 index 3c8ee569670dc..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.action.admin.indices.shrink; - -import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; - -public class ResizeAction extends ActionType { - - public static final ResizeAction INSTANCE = new ResizeAction(); - public static final String NAME = "indices:admin/resize"; - - private ResizeAction() { - super(NAME); - } - -} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java index 21013525fcc8c..8430ad210eb3e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; -import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.support.ActiveShardCount; @@ -19,11 +18,11 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; -import org.elasticsearch.xcontent.ToXContentObject; -import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; @@ -34,10 +33,11 @@ /** * Request class to shrink an index into a single shard */ -public class ResizeRequest extends AcknowledgedRequest implements IndicesRequest, ToXContentObject { +public class ResizeRequest extends AcknowledgedRequest implements IndicesRequest { public static final ObjectParser PARSER = new ObjectParser<>("resize_request"); - private static final ParseField MAX_PRIMARY_SHARD_SIZE = new ParseField("max_primary_shard_size"); + public static final ParseField MAX_PRIMARY_SHARD_SIZE = new ParseField("max_primary_shard_size"); + static { PARSER.declareField( (parser, request, context) -> request.getTargetIndexRequest().settings(parser.map()), @@ -58,8 +58,8 @@ public class ResizeRequest extends AcknowledgedRequest implements } private CreateIndexRequest targetIndexRequest; - private String sourceIndex; - private ResizeType type = ResizeType.SHRINK; + private final String sourceIndex; + private final ResizeType type; private Boolean copySettings = true; private ByteSizeValue maxPrimaryShardSize; @@ -74,14 +74,11 @@ public ResizeRequest(StreamInput in) throws IOException { } } - ResizeRequest() { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT); - } - - public ResizeRequest(String targetIndex, String sourceIndex) { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT, DEFAULT_ACK_TIMEOUT); + public ResizeRequest(TimeValue masterNodeTimeout, TimeValue ackTimeout, ResizeType resizeType, String sourceIndex, String targetIndex) { + super(masterNodeTimeout, ackTimeout); this.targetIndexRequest = new CreateIndexRequest(targetIndex); this.sourceIndex = sourceIndex; + this.type = resizeType; } @Override @@ -106,10 +103,6 @@ public ActionRequestValidationException validate() { return validationException; } - public void setSourceIndex(String index) { - this.sourceIndex = index; - } - @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -175,13 +168,6 @@ public void setWaitForActiveShards(final int waitForActiveShards) { setWaitForActiveShards(ActiveShardCount.from(waitForActiveShards)); } - /** - * The type of the resize operation - */ - public void setResizeType(ResizeType type) { - this.type = Objects.requireNonNull(type); - } - /** * Returns the type of the resize operation */ @@ -219,30 +205,6 @@ public ByteSizeValue getMaxPrimaryShardSize() { return maxPrimaryShardSize; } - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - { - builder.startObject(CreateIndexRequest.SETTINGS.getPreferredName()); - { - targetIndexRequest.settings().toXContent(builder, params); - } - builder.endObject(); - builder.startObject(CreateIndexRequest.ALIASES.getPreferredName()); - { - for (Alias alias : targetIndexRequest.aliases()) { - alias.toXContent(builder, params); - } - } - builder.endObject(); - if (maxPrimaryShardSize != null) { - builder.field(MAX_PRIMARY_SHARD_SIZE.getPreferredName(), maxPrimaryShardSize); - } - } - builder.endObject(); - return builder; - } - public void fromXContent(XContentParser parser) throws IOException { PARSER.parse(parser, this, null); } @@ -263,4 +225,8 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(targetIndexRequest, sourceIndex, type, copySettings, maxPrimaryShardSize); } + + public void setTargetIndexSettings(Settings.Builder settings) { + targetIndexRequest.settings(settings); + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java deleted file mode 100644 index 8d14ca18c7c20..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ -package org.elasticsearch.action.admin.indices.shrink; - -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.support.ActiveShardCount; -import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; -import org.elasticsearch.client.internal.ElasticsearchClient; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; - -public class ResizeRequestBuilder extends AcknowledgedRequestBuilder { - public ResizeRequestBuilder(ElasticsearchClient client) { - super(client, ResizeAction.INSTANCE, new ResizeRequest()); - } - - public ResizeRequestBuilder setTargetIndex(CreateIndexRequest request) { - this.request.setTargetIndex(request); - return this; - } - - public ResizeRequestBuilder setSourceIndex(String index) { - this.request.setSourceIndex(index); - return this; - } - - public ResizeRequestBuilder setSettings(Settings settings) { - this.request.getTargetIndexRequest().settings(settings); - return this; - } - - /** - * Sets the number of shard copies that should be active for creation of the - * new shrunken index to return. Defaults to {@link ActiveShardCount#DEFAULT}, which will - * wait for one shard copy (the primary) to become active. Set this value to - * {@link ActiveShardCount#ALL} to wait for all shards (primary and all replicas) to be active - * before returning. Otherwise, use {@link ActiveShardCount#from(int)} to set this value to any - * non-negative integer, up to the number of copies per shard (number of replicas + 1), - * to wait for the desired amount of shard copies to become active before returning. - * Index creation will only wait up until the timeout value for the number of shard copies - * to be active before returning. Check {@link CreateIndexResponse#isShardsAcknowledged()} to - * determine if the requisite shard copies were all started before returning or timing out. - * - * @param waitForActiveShards number of active shard copies to wait on - */ - public ResizeRequestBuilder setWaitForActiveShards(ActiveShardCount waitForActiveShards) { - this.request.setWaitForActiveShards(waitForActiveShards); - return this; - } - - /** - * A shortcut for {@link #setWaitForActiveShards(ActiveShardCount)} where the numerical - * shard count is passed in, instead of having to first call {@link ActiveShardCount#from(int)} - * to get the ActiveShardCount. - */ - public ResizeRequestBuilder setWaitForActiveShards(final int waitForActiveShards) { - return setWaitForActiveShards(ActiveShardCount.from(waitForActiveShards)); - } - - public ResizeRequestBuilder setResizeType(ResizeType type) { - this.request.setResizeType(type); - return this; - } - - /** - * Sets the max primary shard size of the target index. - */ - public ResizeRequestBuilder setMaxPrimaryShardSize(ByteSizeValue maxPrimaryShardSize) { - this.request.setMaxPrimaryShardSize(maxPrimaryShardSize); - return this; - } -} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index 2f8765fc20b8c..4a781a5a761fe 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -10,6 +10,7 @@ package org.elasticsearch.action.admin.indices.shrink; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; @@ -48,6 +49,8 @@ */ public class TransportResizeAction extends TransportMasterNodeAction { + public static final ActionType TYPE = new ActionType<>("indices:admin/resize"); + private final MetadataCreateIndexService createIndexService; private final ProjectResolver projectResolver; private final Client client; @@ -62,7 +65,7 @@ public TransportResizeAction( ProjectResolver projectResolver, Client client ) { - this(ResizeAction.NAME, transportService, clusterService, threadPool, createIndexService, actionFilters, projectResolver, client); + this(TYPE.name(), transportService, clusterService, threadPool, createIndexService, actionFilters, projectResolver, client); } protected TransportResizeAction( diff --git a/server/src/main/java/org/elasticsearch/client/internal/IndicesAdminClient.java b/server/src/main/java/org/elasticsearch/client/internal/IndicesAdminClient.java index 98d621955926a..b7f265b81f442 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/IndicesAdminClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/IndicesAdminClient.java @@ -94,9 +94,6 @@ import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder; -import org.elasticsearch.action.admin.indices.shrink.ResizeAction; -import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; -import org.elasticsearch.action.admin.indices.shrink.ResizeRequestBuilder; import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; @@ -466,14 +463,6 @@ public GetSettingsRequestBuilder prepareGetSettings(TimeValue masterTimeout, Str return new GetSettingsRequestBuilder(this, masterTimeout, indices); } - public ResizeRequestBuilder prepareResizeIndex(String sourceIndex, String targetIndex) { - return new ResizeRequestBuilder(this).setSourceIndex(sourceIndex).setTargetIndex(new CreateIndexRequest(targetIndex)); - } - - public void resizeIndex(ResizeRequest request, ActionListener listener) { - execute(ResizeAction.INSTANCE, request, listener); - } - public RolloverRequestBuilder prepareRolloverIndex(String alias) { return new RolloverRequestBuilder(this).setRolloverTarget(alias); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java index 5fd4ec83c1a18..c6b8a01f78bf9 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; @@ -36,13 +37,16 @@ public abstract class RestResizeHandler extends BaseRestHandler { @Override public final RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - final ResizeRequest resizeRequest = new ResizeRequest(request.param("target"), request.param("index")); - resizeRequest.setResizeType(getResizeType()); + final ResizeRequest resizeRequest = new ResizeRequest( + getMasterNodeTimeout(request), + getAckTimeout(request), + getResizeType(), + request.param("index"), + request.param("target") + ); request.applyContentParser(resizeRequest::fromXContent); - resizeRequest.ackTimeout(getAckTimeout(request)); - resizeRequest.masterNodeTimeout(getMasterNodeTimeout(request)); resizeRequest.setWaitForActiveShards(ActiveShardCount.parseString(request.param("wait_for_active_shards"))); - return channel -> client.admin().indices().resizeIndex(resizeRequest, new RestToXContentListener<>(channel)); + return channel -> client.execute(TransportResizeAction.TYPE, resizeRequest, new RestToXContentListener<>(channel)); } // no @ServerlessScope on purpose, not available diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java index e1316b40e7ce3..efd797c769f04 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.RandomCreateIndexGenerator; import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; @@ -34,6 +35,7 @@ import java.util.function.Supplier; import static org.elasticsearch.action.admin.indices.create.CreateIndexRequestTests.assertAliasesEqual; +import static org.elasticsearch.action.admin.indices.shrink.ResizeRequest.MAX_PRIMARY_SHARD_SIZE; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; import static org.hamcrest.Matchers.containsString; @@ -53,7 +55,13 @@ public void testCopySettingsValidation() { private void runTestCopySettingsValidation(final Boolean copySettings, final Consumer> consumer) { consumer.accept(() -> { - final ResizeRequest request = new ResizeRequest(); + final ResizeRequest request = new ResizeRequest( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + randomFrom(ResizeType.values()), + randomIdentifier(), + randomIdentifier() + ); request.setCopySettings(copySettings); return request; }); @@ -61,19 +69,25 @@ private void runTestCopySettingsValidation(final Boolean copySettings, final Con public void testToXContent() throws IOException { { - ResizeRequest request = new ResizeRequest("target", "source"); - String actualRequestBody = Strings.toString(request); + ResizeRequest request = new ResizeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, ResizeType.SPLIT, "source", "target"); + String actualRequestBody = Strings.toString(resizeRequestToXContent(request)); assertEquals("{\"settings\":{},\"aliases\":{}}", actualRequestBody); } { - ResizeRequest request = new ResizeRequest("target", "source"); + ResizeRequest request = new ResizeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, ResizeType.SPLIT, "source", "target"); request.setMaxPrimaryShardSize(ByteSizeValue.of(100, ByteSizeUnit.MB)); - String actualRequestBody = Strings.toString(request); + String actualRequestBody = Strings.toString(resizeRequestToXContent(request)); assertEquals(""" {"settings":{},"aliases":{},"max_primary_shard_size":"100mb"}""", actualRequestBody); } { - ResizeRequest request = new ResizeRequest(); + ResizeRequest request = new ResizeRequest( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + randomFrom(ResizeType.values()), + randomIdentifier(), + randomIdentifier() + ); CreateIndexRequest target = new CreateIndexRequest("target"); Alias alias = new Alias("test_alias"); alias.routing("1"); @@ -85,7 +99,7 @@ public void testToXContent() throws IOException { settings.put(SETTING_NUMBER_OF_SHARDS, 10); target.settings(settings); request.setTargetIndex(target); - String actualRequestBody = Strings.toString(request); + String actualRequestBody = Strings.toString(resizeRequestToXContent(request)); String expectedRequestBody = """ { "settings": { @@ -114,11 +128,19 @@ public void testToAndFromXContent() throws IOException { boolean humanReadable = randomBoolean(); final XContentType xContentType = randomFrom(XContentType.values()); - BytesReference originalBytes = toShuffledXContent(resizeRequest, xContentType, EMPTY_PARAMS, humanReadable); + BytesReference originalBytes = toShuffledXContent( + resizeRequestToXContent(resizeRequest), + xContentType, + EMPTY_PARAMS, + humanReadable + ); ResizeRequest parsedResizeRequest = new ResizeRequest( - randomValueOtherThan(resizeRequest.getTargetIndexRequest().index(), () -> randomAlphaOfLength(5)), - randomValueOtherThan(resizeRequest.getSourceIndex(), () -> randomAlphaOfLength(5)) + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + randomFrom(ResizeType.values()), + randomValueOtherThan(resizeRequest.getSourceIndex(), () -> randomAlphaOfLength(5)), + randomValueOtherThan(resizeRequest.getTargetIndexRequest().index(), () -> randomAlphaOfLength(5)) ); try (XContentParser xParser = createParser(xContentType.xContent(), originalBytes)) { parsedResizeRequest.fromXContent(xParser); @@ -135,7 +157,12 @@ public void testToAndFromXContent() throws IOException { assertEquals(resizeRequest.getTargetIndexRequest().settings(), parsedResizeRequest.getTargetIndexRequest().settings()); assertEquals(resizeRequest.getMaxPrimaryShardSize(), parsedResizeRequest.getMaxPrimaryShardSize()); - BytesReference finalBytes = toShuffledXContent(parsedResizeRequest, xContentType, EMPTY_PARAMS, humanReadable); + BytesReference finalBytes = toShuffledXContent( + resizeRequestToXContent(parsedResizeRequest), + xContentType, + EMPTY_PARAMS, + humanReadable + ); ElasticsearchAssertions.assertToXContentEquivalent(originalBytes, finalBytes, xContentType); } @@ -163,7 +190,13 @@ protected Writeable.Reader instanceReader() { @Override protected ResizeRequest createTestInstance() { - ResizeRequest resizeRequest = new ResizeRequest(randomAlphaOfLengthBetween(3, 10), randomAlphaOfLengthBetween(3, 10)); + ResizeRequest resizeRequest = new ResizeRequest( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + randomFrom(ResizeType.values()), + randomAlphaOfLengthBetween(3, 10), + randomAlphaOfLengthBetween(3, 10) + ); if (randomBoolean()) { resizeRequest.setTargetIndex(RandomCreateIndexGenerator.randomCreateIndexRequest()); } @@ -177,4 +210,29 @@ protected ResizeRequest createTestInstance() { protected ResizeRequest mutateInstance(ResizeRequest instance) { return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929 } + + private static ToXContentObject resizeRequestToXContent(ResizeRequest resizeRequest) { + return (builder, params) -> { + builder.startObject(); + { + builder.startObject(CreateIndexRequest.SETTINGS.getPreferredName()); + { + resizeRequest.getTargetIndexRequest().settings().toXContent(builder, params); + } + builder.endObject(); + builder.startObject(CreateIndexRequest.ALIASES.getPreferredName()); + { + for (Alias alias : resizeRequest.getTargetIndexRequest().aliases()) { + alias.toXContent(builder, params); + } + } + builder.endObject(); + if (resizeRequest.getMaxPrimaryShardSize() != null) { + builder.field(MAX_PRIMARY_SHARD_SIZE.getPreferredName(), resizeRequest.getMaxPrimaryShardSize()); + } + } + builder.endObject(); + return builder; + }; + } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java index 66fb8a58bba1c..dac353919d0f9 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java @@ -46,6 +46,7 @@ import java.util.Collections; import java.util.Set; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.resizeRequest; import static org.hamcrest.Matchers.equalTo; public class TransportResizeActionTests extends ESTestCase { @@ -94,7 +95,7 @@ public void testErrorCondition() { expectThrows( IllegalStateException.class, () -> TransportResizeAction.prepareCreateIndexRequest( - new ResizeRequest("target", "source"), + resizeRequest(ResizeType.SHRINK, "source", "target", Settings.builder()), projectId, state, "target", @@ -106,28 +107,28 @@ public void testErrorCondition() { ).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ") ); - assertTrue(expectThrows(IllegalStateException.class, () -> { - ResizeRequest req = new ResizeRequest("target", "source"); - req.getTargetIndexRequest().settings(Settings.builder().put("index.number_of_shards", 4)); - TransportResizeAction.prepareCreateIndexRequest( - req, - projectId, - createClusterState("source", 8, 1, Settings.builder().put("index.blocks.write", true).build()).metadata() - .getProject(projectId) - .index("source"), - "target", - new ResizeNumberOfShardsCalculator.ShrinkShardsCalculator( - new StoreStats(between(1, 100), between(0, 100), between(1, 100)), - (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE / 2, between(1, 1000), between(1, 10000)) : null + assertTrue( + expectThrows( + IllegalStateException.class, + () -> TransportResizeAction.prepareCreateIndexRequest( + resizeRequest(ResizeType.SHRINK, "source", "target", Settings.builder().put("index.number_of_shards", 4)), + projectId, + createClusterState("source", 8, 1, Settings.builder().put("index.blocks.write", true).build()).metadata() + .getProject(projectId) + .index("source"), + "target", + new ResizeNumberOfShardsCalculator.ShrinkShardsCalculator( + new StoreStats(between(1, 100), between(0, 100), between(1, 100)), + (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE / 2, between(1, 1000), between(1, 10000)) : null + ) ) - ); - }).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ")); - - IllegalArgumentException softDeletesError = expectThrows(IllegalArgumentException.class, () -> { - ResizeRequest req = new ResizeRequest("target", "source"); - req.getTargetIndexRequest().settings(Settings.builder().put("index.soft_deletes.enabled", false)); - TransportResizeAction.prepareCreateIndexRequest( - req, + ).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ") + ); + + IllegalArgumentException softDeletesError = expectThrows( + IllegalArgumentException.class, + () -> TransportResizeAction.prepareCreateIndexRequest( + resizeRequest(ResizeType.SHRINK, "source", "target", Settings.builder().put("index.soft_deletes.enabled", false)), projectId, createClusterState( "source", @@ -140,8 +141,8 @@ public void testErrorCondition() { new StoreStats(between(1, 100), between(0, 100), between(1, 100)), (i) -> new DocsStats(between(10, 1000), between(1, 10), between(1, 10000)) ) - ); - }); + ) + ); assertThat(softDeletesError.getMessage(), equalTo("Can't disable [index.soft_deletes.enabled] setting on resize")); // create one that won't fail @@ -164,7 +165,7 @@ public void testErrorCondition() { clusterState = ClusterState.builder(clusterState).putRoutingTable(projectId, routingTable).build(); TransportResizeAction.prepareCreateIndexRequest( - new ResizeRequest("target", "source"), + resizeRequest(ResizeType.SHRINK, "source", "target", Settings.builder()), projectId, clusterState.metadata().getProject(projectId).index("source"), "target", @@ -194,24 +195,22 @@ public void testPassNumRoutingShards() { routingTable = ESAllocationTestCase.startInitializingShardsAndReroute(service, clusterState, "source").routingTable(projectId); clusterState = ClusterState.builder(clusterState).putRoutingTable(projectId, routingTable).build(); - ResizeRequest resizeRequest = new ResizeRequest("target", "source"); - resizeRequest.setResizeType(ResizeType.SPLIT); - resizeRequest.getTargetIndexRequest().settings(Settings.builder().put("index.number_of_shards", 2).build()); IndexMetadata indexMetadata = clusterState.metadata().getProject(projectId).index("source"); TransportResizeAction.prepareCreateIndexRequest( - resizeRequest, + resizeRequest(ResizeType.SPLIT, "source", "target", Settings.builder().put("index.number_of_shards", 2)), projectId, indexMetadata, "target", new ResizeNumberOfShardsCalculator.SplitShardsCalculator() ); - resizeRequest.getTargetIndexRequest() - .settings( - Settings.builder().put("index.number_of_routing_shards", randomIntBetween(2, 10)).put("index.number_of_shards", 2).build() - ); TransportResizeAction.prepareCreateIndexRequest( - resizeRequest, + resizeRequest( + ResizeType.SPLIT, + "source", + "target", + Settings.builder().put("index.number_of_routing_shards", randomIntBetween(2, 10)).put("index.number_of_shards", 2) + ), projectId, indexMetadata, "target", @@ -239,26 +238,24 @@ public void testPassNumRoutingShardsAndFail() { routingTable = ESAllocationTestCase.startInitializingShardsAndReroute(service, clusterState, "source").routingTable(projectId); clusterState = ClusterState.builder(clusterState).putRoutingTable(projectId, routingTable).build(); - ResizeRequest resizeRequest = new ResizeRequest("target", "source"); - resizeRequest.setResizeType(ResizeType.SPLIT); - resizeRequest.getTargetIndexRequest().settings(Settings.builder().put("index.number_of_shards", numShards * 2).build()); TransportResizeAction.prepareCreateIndexRequest( - resizeRequest, + resizeRequest(ResizeType.SPLIT, "source", "target", Settings.builder().put("index.number_of_shards", numShards * 2)), projectId, clusterState.metadata().getProject(projectId).index("source"), "target", new ResizeNumberOfShardsCalculator.SplitShardsCalculator() ); - resizeRequest.getTargetIndexRequest() - .settings( - Settings.builder().put("index.number_of_shards", numShards * 2).put("index.number_of_routing_shards", numShards * 2).build() - ); ClusterState finalState = clusterState; IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, () -> TransportResizeAction.prepareCreateIndexRequest( - resizeRequest, + resizeRequest( + ResizeType.SPLIT, + "source", + "target", + Settings.builder().put("index.number_of_shards", numShards * 2).put("index.number_of_routing_shards", numShards * 2) + ), projectId, finalState.metadata().getProject(projectId).index("source"), "target", @@ -290,7 +287,7 @@ public void testShrinkIndexSettings() { clusterState = ClusterState.builder(clusterState).putRoutingTable(projectId, routingTable).build(); int numSourceShards = clusterState.metadata().getProject(projectId).index(indexName).getNumberOfShards(); DocsStats stats = new DocsStats(between(0, (IndexWriter.MAX_DOCS) / numSourceShards), between(1, 1000), between(1, 10000)); - ResizeRequest target = new ResizeRequest("target", indexName); + ResizeRequest target = resizeRequest(ResizeType.SHRINK, indexName, "target", Settings.builder()); final ActiveShardCount activeShardCount = randomBoolean() ? ActiveShardCount.ALL : ActiveShardCount.ONE; target.setWaitForActiveShards(activeShardCount); CreateIndexClusterStateUpdateRequest request = TransportResizeAction.prepareCreateIndexRequest( @@ -318,9 +315,13 @@ public void testShrinkWithMaxPrimaryShardSize() { randomIntBetween(0, 10), Settings.builder().put("index.blocks.write", true).build() ).metadata().getProject(projectId).index("source"); - ResizeRequest resizeRequest = new ResizeRequest("target", "source"); + ResizeRequest resizeRequest = resizeRequest( + ResizeType.SHRINK, + "source", + "target", + Settings.builder().put("index.number_of_shards", 2) + ); resizeRequest.setMaxPrimaryShardSize(ByteSizeValue.ofBytes(10)); - resizeRequest.getTargetIndexRequest().settings(Settings.builder().put("index.number_of_shards", 2).build()); assertTrue( expectThrows( IllegalArgumentException.class, @@ -359,7 +360,7 @@ public void testShrinkWithMaxPrimaryShardSize() { DocsStats stats = new DocsStats(between(0, (IndexWriter.MAX_DOCS) / numSourceShards), between(1, 1000), between(1, 10000)); // each shard's storage will not be greater than the `max_primary_shard_size` - ResizeRequest target1 = new ResizeRequest("target", "source"); + ResizeRequest target1 = resizeRequest(ResizeType.SHRINK, "source", "target", Settings.builder()); target1.setMaxPrimaryShardSize(ByteSizeValue.ofBytes(2)); StoreStats storeStats = new StoreStats(10, between(0, 100), between(1, 100)); final int targetIndexShardsNum1 = 5; @@ -381,7 +382,7 @@ public void testShrinkWithMaxPrimaryShardSize() { // if `max_primary_shard_size` is less than the single shard size of the source index, // the shards number of the target index will be equal to the source index's shards number - ResizeRequest target2 = new ResizeRequest("target2", "source"); + ResizeRequest target2 = resizeRequest(ResizeType.SHRINK, "source", "target2", Settings.builder()); target2.setMaxPrimaryShardSize(ByteSizeValue.ofBytes(1)); StoreStats storeStats2 = new StoreStats(100, between(0, 100), between(1, 100)); final int targetIndexShardsNum2 = 10; diff --git a/test/framework/src/main/java/org/elasticsearch/action/admin/indices/ResizeIndexTestUtils.java b/test/framework/src/main/java/org/elasticsearch/action/admin/indices/ResizeIndexTestUtils.java new file mode 100644 index 0000000000000..7073c124e23d9 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/action/admin/indices/ResizeIndexTestUtils.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action.admin.indices; + +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; +import org.elasticsearch.common.settings.Settings; + +import static org.elasticsearch.test.ESIntegTestCase.client; +import static org.elasticsearch.test.ESTestCase.TEST_REQUEST_TIMEOUT; + +public enum ResizeIndexTestUtils { + ; + + public static ResizeRequest resizeRequest(ResizeType resizeType, String sourceIndex, String targetIndex, Settings.Builder settings) { + final var resizeRequest = new ResizeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, resizeType, sourceIndex, targetIndex); + resizeRequest.setTargetIndexSettings(settings); + return resizeRequest; + } + + public static ActionFuture executeResize( + ResizeType resizeType, + String sourceIndex, + String targetIndex, + Settings.Builder settings + ) { + return client().execute(TransportResizeAction.TYPE, resizeRequest(resizeType, sourceIndex, targetIndex, settings)); + } +} diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java index bca057b4d3416..0b18d4b152888 100644 --- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java +++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils; import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.ActiveShardCount; @@ -41,6 +42,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.resizeRequest; import static org.elasticsearch.index.store.Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.xpack.autoscaling.storage.ReactiveStorageDeciderService.AllocationState.MAX_AMOUNT_OF_SHARD_DECISIONS; @@ -347,16 +349,9 @@ public void testScaleWhileShrinking() throws Exception { }); String shrinkName = "shrink-" + indexName; - assertAcked( - indicesAdmin().prepareResizeIndex(indexName, shrinkName) - .setSettings( - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .build() - ) - .setWaitForActiveShards(ActiveShardCount.NONE) - ); + final var shrinkRequest = resizeRequest(ResizeType.SHRINK, indexName, shrinkName, indexSettings(1, 0)); + shrinkRequest.setWaitForActiveShards(ActiveShardCount.NONE); + assertAcked(client().execute(TransportResizeAction.TYPE, shrinkRequest)); // * 2 since worst case is no hard links, see DiskThresholdDecider.getExpectedShardSize. long requiredSpaceForShrink = used * 2 + LOW_WATERMARK_BYTES; @@ -455,17 +450,9 @@ public void testScaleDuringSplit() throws Exception { updateIndexSettings(Settings.builder().put("index.blocks.write", true), indexName); String cloneName = "clone-" + indexName; int resizedShardCount = between(2, 10); - assertAcked( - indicesAdmin().prepareResizeIndex(indexName, cloneName) - .setSettings( - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, resizedShardCount) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .build() - ) - .setWaitForActiveShards(ActiveShardCount.NONE) - .setResizeType(ResizeType.SPLIT) - ); + final var splitRequest = resizeRequest(ResizeType.SPLIT, indexName, cloneName, indexSettings(resizedShardCount, 0)); + splitRequest.setWaitForActiveShards(ActiveShardCount.NONE); + assertAcked(client().execute(TransportResizeAction.TYPE, splitRequest)); // * 2 since worst case is no hard links, see DiskThresholdDecider.getExpectedShardSize. long requiredSpaceForClone = used * 2 + LOW_WATERMARK_BYTES; diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java index 041be4ea40e6d..421a010daec50 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDeciderIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanationUtils; import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesAction; import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesRequest; +import org.elasticsearch.action.admin.indices.ResizeIndexTestUtils; import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.cluster.health.ClusterHealthStatus; @@ -324,12 +325,7 @@ public void testShrinkStaysOnTier() { .get(); indicesAdmin().prepareAddBlock(IndexMetadata.APIBlock.READ_ONLY, index).get(); - indicesAdmin().prepareResizeIndex(index, index + "-shrunk") - .setResizeType(ResizeType.SHRINK) - .setSettings( - Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build() - ) - .get(); + ResizeIndexTestUtils.executeResize(ResizeType.SHRINK, index, index + "-shrunk", indexSettings(1, 0)).actionGet(); ensureGreen(index + "-shrunk"); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ResizeIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ResizeIndexStep.java index 847647b0cd5d3..81315944498cf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ResizeIndexStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ResizeIndexStep.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.ProjectState; @@ -95,10 +96,13 @@ public void performAction( .put(LifecycleSettings.LIFECYCLE_SKIP, true) .build(); - ResizeRequest resizeRequest = new ResizeRequest(targetIndexName, indexMetadata.getIndex().getName()).masterNodeTimeout( - TimeValue.MAX_VALUE + ResizeRequest resizeRequest = new ResizeRequest( + TimeValue.MAX_VALUE, + TimeValue.MAX_VALUE, + resizeType, + indexMetadata.getIndex().getName(), + targetIndexName ); - resizeRequest.setResizeType(resizeType); resizeRequest.getTargetIndexRequest().settings(relevantTargetSettings); if (resizeType == ResizeType.SHRINK) { resizeRequest.setMaxPrimaryShardSize(maxPrimaryShardSize); @@ -106,9 +110,7 @@ public void performAction( // This request does not wait for (successful) completion of the resize operation - it fires-and-forgets. // It's up to a subsequent step to check for the existence of the target index and wait for it to be green. - getClient(currentState.projectId()).admin() - .indices() - .resizeIndex(resizeRequest, listener.delegateFailureAndWrap((l, response) -> l.onResponse(null))); + getClient(currentState.projectId()).execute(TransportResizeAction.TYPE, resizeRequest, listener.map(response -> null)); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ResizeIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ResizeIndexStepTests.java index 9fb230885a4b9..8316c587384af 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ResizeIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ResizeIndexStepTests.java @@ -8,9 +8,9 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.cluster.ProjectState; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -119,9 +119,8 @@ public void testPerformAction() throws Exception { .build(); Mockito.doAnswer(invocation -> { - ResizeRequest request = (ResizeRequest) invocation.getArguments()[0]; - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; + final ResizeRequest request = invocation.getArgument(1); + final ActionListener listener = invocation.getArgument(2); assertThat(request.getSourceIndex(), equalTo(sourceIndexMetadata.getIndex().getName())); assertThat(request.getTargetIndexRequest().aliases(), equalTo(Set.of())); @@ -133,16 +132,14 @@ public void testPerformAction() throws Exception { assertThat(request.getMaxPrimaryShardSize(), equalTo(step.getMaxPrimaryShardSize())); listener.onResponse(new CreateIndexResponse(true, true, sourceIndexMetadata.getIndex().getName())); return null; - }).when(indicesClient).resizeIndex(Mockito.any(), Mockito.any()); + }).when(projectClient).execute(Mockito.same(TransportResizeAction.TYPE), Mockito.any(), Mockito.any()); final var state = projectStateWithEmptyProject(); performActionAndWait(step, sourceIndexMetadata, state, null); Mockito.verify(client).projectClient(state.projectId()); - Mockito.verify(projectClient).admin(); - Mockito.verifyNoMoreInteractions(client); - Mockito.verify(adminClient, Mockito.only()).indices(); - Mockito.verify(indicesClient, Mockito.only()).resizeIndex(Mockito.any(), Mockito.any()); + Mockito.verify(projectClient).execute(Mockito.same(TransportResizeAction.TYPE), Mockito.any(), Mockito.any()); + Mockito.verifyNoMoreInteractions(client, projectClient); } public void testPerformActionShrunkenIndexExists() throws Exception { @@ -195,23 +192,20 @@ public void testPerformActionIsCompleteForUnAckedRequests() throws Exception { ResizeIndexStep step = createRandomInstance(); Mockito.doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; + final ActionListener listener = invocation.getArgument(2); listener.onResponse(new CreateIndexResponse(false, false, indexMetadata.getIndex().getName())); return null; - }).when(indicesClient).resizeIndex(Mockito.any(), Mockito.any()); + }).when(projectClient).execute(Mockito.same(TransportResizeAction.TYPE), Mockito.any(), Mockito.any()); final var state = projectStateWithEmptyProject(); performActionAndWait(step, indexMetadata, state, null); Mockito.verify(client).projectClient(state.projectId()); - Mockito.verify(projectClient).admin(); - Mockito.verifyNoMoreInteractions(client); - Mockito.verify(adminClient, Mockito.only()).indices(); - Mockito.verify(indicesClient, Mockito.only()).resizeIndex(Mockito.any(), Mockito.any()); + Mockito.verify(projectClient).execute(Mockito.same(TransportResizeAction.TYPE), Mockito.any(), Mockito.any()); + Mockito.verifyNoMoreInteractions(client, projectClient); } - public void testPerformActionFailure() throws Exception { + public void testPerformActionFailure() { LifecycleExecutionState.Builder lifecycleState = LifecycleExecutionState.builder(); lifecycleState.setIndexCreationDate(randomNonNegativeLong()); IndexMetadata indexMetadata = IndexMetadata.builder(randomAlphaOfLength(10)) @@ -224,20 +218,17 @@ public void testPerformActionFailure() throws Exception { ResizeIndexStep step = createRandomInstance(); Mockito.doAnswer(invocation -> { - @SuppressWarnings("unchecked") - ActionListener listener = (ActionListener) invocation.getArguments()[1]; + final ActionListener listener = invocation.getArgument(2); listener.onFailure(exception); return null; - }).when(indicesClient).resizeIndex(Mockito.any(), Mockito.any()); + }).when(projectClient).execute(Mockito.same(TransportResizeAction.TYPE), Mockito.any(), Mockito.any()); final var state = projectStateWithEmptyProject(); assertSame(exception, expectThrows(Exception.class, () -> performActionAndWait(step, indexMetadata, state, null))); Mockito.verify(client).projectClient(state.projectId()); - Mockito.verify(projectClient).admin(); - Mockito.verifyNoMoreInteractions(client); - Mockito.verify(adminClient, Mockito.only()).indices(); - Mockito.verify(indicesClient, Mockito.only()).resizeIndex(Mockito.any(), Mockito.any()); + Mockito.verify(projectClient).execute(Mockito.same(TransportResizeAction.TYPE), Mockito.any(), Mockito.any()); + Mockito.verifyNoMoreInteractions(client, projectClient); } } diff --git a/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsIndexingIT.java b/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsIndexingIT.java index bea3b7343de3b..d52b364d56888 100644 --- a/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsIndexingIT.java +++ b/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsIndexingIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.get.GetRequest; @@ -37,6 +38,7 @@ import java.util.List; import java.util.UUID; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.resizeRequest; import static org.elasticsearch.index.mapper.DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; @@ -233,12 +235,9 @@ public void testShrink() throws Exception { client().bulk(bulkRequest).actionGet(); client().admin().indices().prepareFlush("my-logs").get(); client().admin().indices().prepareUpdateSettings("my-logs").setSettings(Settings.builder().put("index.blocks.write", true)).get(); - client().admin() - .indices() - .prepareResizeIndex("my-logs", "shrink-my-logs") - .setResizeType(ResizeType.SHRINK) - .setSettings(indexSettings(1, 0).build()) - .get(); + + client().execute(TransportResizeAction.TYPE, resizeRequest(ResizeType.SHRINK, "my-logs", "shrink-my-logs", indexSettings(1, 0))) + .actionGet(); assertNoFailures(client().admin().indices().prepareForceMerge("shrink-my-logs").setMaxNumSegments(1).setFlush(true).get()); } diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java index 5c86ca4649194..83bb975131805 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/FrozenSearchableSnapshotsIntegTests.java @@ -62,6 +62,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.executeResize; import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram; @@ -384,15 +385,15 @@ public void testCreateAndRestorePartialSearchableSnapshot() throws Exception { final String clonedIndexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); assertAcked( - indicesAdmin().prepareResizeIndex(restoredIndexName, clonedIndexName) - .setResizeType(ResizeType.CLONE) - .setSettings( - Settings.builder() - .putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()) - .putNull(IndexModule.INDEX_RECOVERY_TYPE_SETTING.getKey()) - .put(DataTier.TIER_PREFERENCE, DataTier.DATA_HOT) - .build() - ) + executeResize( + ResizeType.CLONE, + restoredIndexName, + clonedIndexName, + Settings.builder() + .putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()) + .putNull(IndexModule.INDEX_RECOVERY_TYPE_SETTING.getKey()) + .put(DataTier.TIER_PREFERENCE, DataTier.DATA_HOT) + ) ); ensureGreen(clonedIndexName); assertTotalHits(clonedIndexName, originalAllHits, originalBarHits); diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java index 0fe63df106862..69c3792a0f665 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java @@ -81,6 +81,7 @@ import java.util.stream.IntStream; import static org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplanationUtils.getClusterAllocationExplanation; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.executeResize; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING; import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; @@ -330,14 +331,14 @@ public void testCreateAndRestoreSearchableSnapshot() throws Exception { final String clonedIndexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); assertAcked( - indicesAdmin().prepareResizeIndex(restoredIndexName, clonedIndexName) - .setResizeType(ResizeType.CLONE) - .setSettings( - Settings.builder() - .putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()) - .putNull(IndexModule.INDEX_RECOVERY_TYPE_SETTING.getKey()) - .build() - ) + executeResize( + ResizeType.CLONE, + restoredIndexName, + clonedIndexName, + Settings.builder() + .putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()) + .putNull(IndexModule.INDEX_RECOVERY_TYPE_SETTING.getKey()) + ) ); ensureGreen(clonedIndexName); assertTotalHits(clonedIndexName, originalAllHits, originalBarHits); diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsResizeIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsResizeIntegTests.java index 0a0cce5dd87ed..d6db625fa02c8 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsResizeIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsResizeIntegTests.java @@ -18,6 +18,7 @@ import java.util.List; +import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.executeResize; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING; import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING; import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; @@ -55,13 +56,14 @@ public void tearDown() throws Exception { super.tearDown(); } + private static void runResizeAction(String targetIndexName, ResizeType resizeType, Settings.Builder settings) { + assertAcked(executeResize(resizeType, "mounted-index", targetIndexName, settings)); + } + public void testShrinkSearchableSnapshotIndex() { final IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> indicesAdmin().prepareResizeIndex("mounted-index", "shrunk-index") - .setResizeType(ResizeType.SHRINK) - .setSettings(indexSettingsNoReplicas(1).build()) - .get() + () -> runResizeAction("shrunk-index", ResizeType.SHRINK, indexSettingsNoReplicas(1)) ); assertThat(exception.getMessage(), equalTo("can't shrink searchable snapshot index [mounted-index]")); } @@ -69,10 +71,7 @@ public void testShrinkSearchableSnapshotIndex() { public void testSplitSearchableSnapshotIndex() { final IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> indicesAdmin().prepareResizeIndex("mounted-index", "split-index") - .setResizeType(ResizeType.SPLIT) - .setSettings(indexSettingsNoReplicas(4).build()) - .get() + () -> runResizeAction("split-index", ResizeType.SPLIT, indexSettingsNoReplicas(4)) ); assertThat(exception.getMessage(), equalTo("can't split searchable snapshot index [mounted-index]")); } @@ -80,7 +79,7 @@ public void testSplitSearchableSnapshotIndex() { public void testCloneSearchableSnapshotIndex() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, - () -> indicesAdmin().prepareResizeIndex("mounted-index", "cloned-index").setResizeType(ResizeType.CLONE).get() + () -> runResizeAction("cloned-index", ResizeType.CLONE, Settings.builder()) ); assertThat( exception.getMessage(), @@ -89,27 +88,25 @@ public void testCloneSearchableSnapshotIndex() { exception = expectThrows( IllegalArgumentException.class, - () -> indicesAdmin().prepareResizeIndex("mounted-index", "cloned-index") - .setResizeType(ResizeType.CLONE) - .setSettings(Settings.builder().putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()).build()) - .get() + () -> runResizeAction( + "cloned-index", + ResizeType.CLONE, + Settings.builder().putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()) + ) ); assertThat( exception.getMessage(), equalTo("can't clone searchable snapshot index [mounted-index]; setting [index.recovery.type] should be overridden") ); - assertAcked( - indicesAdmin().prepareResizeIndex("mounted-index", "cloned-index") - .setResizeType(ResizeType.CLONE) - .setSettings( - Settings.builder() - .putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()) - .putNull(IndexModule.INDEX_RECOVERY_TYPE_SETTING.getKey()) - .put(DataTier.TIER_PREFERENCE, DataTier.DATA_HOT) - .put(INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) - .build() - ) + runResizeAction( + "cloned-index", + ResizeType.CLONE, + Settings.builder() + .putNull(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()) + .putNull(IndexModule.INDEX_RECOVERY_TYPE_SETTING.getKey()) + .put(DataTier.TIER_PREFERENCE, DataTier.DATA_HOT) + .put(INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) ); ensureGreen("cloned-index"); assertAcked(indicesAdmin().prepareDelete("cloned-index")); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/ShrinkIndexWithSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/ShrinkIndexWithSecurityTests.java index 05dc4325a302a..3926a047c43aa 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/ShrinkIndexWithSecurityTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/ShrinkIndexWithSecurityTests.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.integration; +import org.elasticsearch.action.admin.indices.ResizeIndexTestUtils; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.TermsQueryBuilder; @@ -46,7 +48,8 @@ public void testShrinkIndex() throws Exception { // wait for green and then shrink ensureGreen(); - assertAcked(indicesAdmin().prepareResizeIndex("bigindex", "shrunk_bigindex").setSettings(indexSettings(1, 0).build())); + + assertAcked(ResizeIndexTestUtils.executeResize(ResizeType.SHRINK, "bigindex", "shrunk_bigindex", indexSettings(1, 0))); // verify all docs ensureGreen(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptorTests.java index e3b402d96d416..618d374984e41 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/interceptor/ResizeRequestInterceptorTests.java @@ -8,8 +8,9 @@ import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.shrink.ResizeAction; import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -111,7 +112,12 @@ public void checkResizeWithDlsFlsConfigured(boolean dlsFlsFeatureEnabled, String ); PlainActionFuture plainActionFuture = new PlainActionFuture<>(); - RequestInfo requestInfo = new RequestInfo(authentication, new ResizeRequest("bar", "foo"), ResizeAction.NAME, null); + RequestInfo requestInfo = new RequestInfo( + authentication, + new ResizeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, randomFrom(ResizeType.values()), "foo", "bar"), + TransportResizeAction.TYPE.name(), + null + ); AuthorizationEngine mockEngine = mock(AuthorizationEngine.class); doAnswer(invocationOnMock -> { ActionListener listener = (ActionListener) invocationOnMock.getArguments()[3]; @@ -147,7 +153,12 @@ public void testResizeRequestInterceptorThrowsWhenTargetHasGreaterPermissions() AuthorizationEngine mockEngine = mock(AuthorizationEngine.class); { PlainActionFuture plainActionFuture = new PlainActionFuture<>(); - RequestInfo requestInfo = new RequestInfo(authentication, new ResizeRequest("target", "source"), ResizeAction.NAME, null); + RequestInfo requestInfo = new RequestInfo( + authentication, + new ResizeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, randomFrom(ResizeType.values()), "source", "target"), + TransportResizeAction.TYPE.name(), + null + ); doAnswer(invocationOnMock -> { ActionListener listener = (ActionListener) invocationOnMock.getArguments()[3]; listener.onResponse(AuthorizationResult.deny()); @@ -172,7 +183,12 @@ public void testResizeRequestInterceptorThrowsWhenTargetHasGreaterPermissions() // swap target and source for success { PlainActionFuture plainActionFuture = new PlainActionFuture<>(); - RequestInfo requestInfo = new RequestInfo(authentication, new ResizeRequest("source", "target"), ResizeAction.NAME, null); + RequestInfo requestInfo = new RequestInfo( + authentication, + new ResizeRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, randomFrom(ResizeType.values()), "target", "source"), + TransportResizeAction.TYPE.name(), + null + ); doAnswer(invocationOnMock -> { ActionListener listener = (ActionListener) invocationOnMock.getArguments()[3]; listener.onResponse(AuthorizationResult.granted());