Skip to content

Commit dd09bf8

Browse files
carlosdelestcarlosdelest
committed
Merge remote-tracking branch 'origin/main' into non-issue/esql-vector-search-functions-basics
2 parents 7601522 + b16007b commit dd09bf8

File tree

307 files changed

+8868
-5904
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

307 files changed

+8868
-5904
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesAggregatorBenchmark.java‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,8 @@ static void selfTest() {
9595
try {
9696
for (String groups : ValuesAggregatorBenchmark.class.getField("groups").getAnnotationsByType(Param.class)[0].value()) {
9797
for (String dataType : ValuesAggregatorBenchmark.class.getField("dataType").getAnnotationsByType(Param.class)[0].value()) {
98-
run(Integer.parseInt(groups), dataType, 10);
98+
run(Integer.parseInt(groups), dataType, 10, 0);
99+
run(Integer.parseInt(groups), dataType, 10, 1);
99100
}
100101
}
101102
} catch (NoSuchFieldException e) {
@@ -113,7 +114,10 @@ static void selfTest() {
113114
@Param({ BYTES_REF, INT, LONG })
114115
public String dataType;
115116

116-
private static Operator operator(DriverContext driverContext, int groups, String dataType) {
117+
@Param({ "0", "1" })
118+
public int numOrdinalMerges;
119+
120+
private static Operator operator(DriverContext driverContext, int groups, String dataType, int numOrdinalMerges) {
117121
if (groups == 1) {
118122
return new AggregationOperator(
119123
List.of(supplier(dataType).aggregatorFactory(AggregatorMode.SINGLE, List.of(0)).apply(driverContext)),
@@ -125,7 +129,24 @@ private static Operator operator(DriverContext driverContext, int groups, String
125129
List.of(supplier(dataType).groupingAggregatorFactory(AggregatorMode.SINGLE, List.of(1))),
126130
() -> BlockHash.build(groupSpec, driverContext.blockFactory(), 16 * 1024, false),
127131
driverContext
128-
);
132+
) {
133+
@Override
134+
public Page getOutput() {
135+
mergeOrdinal();
136+
return super.getOutput();
137+
}
138+
139+
// simulate OrdinalsGroupingOperator
140+
void mergeOrdinal() {
141+
var merged = supplier(dataType).groupingAggregatorFactory(AggregatorMode.SINGLE, List.of(1)).apply(driverContext);
142+
for (int i = 0; i < numOrdinalMerges; i++) {
143+
for (int p = 0; p < groups; p++) {
144+
merged.addIntermediateRow(p, aggregators.getFirst(), p);
145+
}
146+
}
147+
aggregators.set(0, merged);
148+
}
149+
};
129150
}
130151

131152
private static AggregatorFunctionSupplier supplier(String dataType) {
@@ -331,12 +352,12 @@ private static Block groupingBlock(int groups) {
331352

332353
@Benchmark
333354
public void run() {
334-
run(groups, dataType, OP_COUNT);
355+
run(groups, dataType, OP_COUNT, numOrdinalMerges);
335356
}
336357

337-
private static void run(int groups, String dataType, int opCount) {
358+
private static void run(int groups, String dataType, int opCount, int numOrdinalMerges) {
338359
DriverContext driverContext = driverContext();
339-
try (Operator operator = operator(driverContext, groups, dataType)) {
360+
try (Operator operator = operator(driverContext, groups, dataType, numOrdinalMerges)) {
340361
Page page = page(groups, dataType);
341362
for (int i = 0; i < opCount; i++) {
342363
operator.addInput(page.shallowCopy());

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@
4141
import org.elasticsearch.compute.data.Page;
4242
import org.elasticsearch.compute.lucene.LuceneSourceOperator;
4343
import org.elasticsearch.compute.lucene.ShardRefCounted;
44-
import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator;
44+
import org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperator;
45+
import org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperatorStatus;
4546
import org.elasticsearch.compute.operator.topn.TopNOperator;
4647
import org.elasticsearch.core.IOUtils;
4748
import org.elasticsearch.index.IndexSettings;
@@ -343,7 +344,7 @@ public void benchmark() {
343344
);
344345
long sum = 0;
345346
for (Page page : pages) {
346-
op.addInput(page);
347+
op.addInput(page.shallowCopy());
347348
switch (name) {
348349
case "long" -> {
349350
LongVector values = op.getOutput().<LongBlock>getBlock(1).asVector();
@@ -411,7 +412,7 @@ public void benchmark() {
411412
throw new AssertionError("[" + layout + "][" + name + "] expected [" + expected + "] but was [" + sum + "]");
412413
}
413414
boolean foundStoredFieldLoader = false;
414-
ValuesSourceReaderOperator.Status status = (ValuesSourceReaderOperator.Status) op.status();
415+
ValuesSourceReaderOperatorStatus status = (ValuesSourceReaderOperatorStatus) op.status();
415416
for (Map.Entry<String, Integer> e : status.readersBuilt().entrySet()) {
416417
if (e.getKey().indexOf("stored_fields") >= 0) {
417418
foundStoredFieldLoader = true;

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/Int7uScorerBenchmark.java‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
import org.apache.lucene.util.quantization.ScalarQuantizer;
2525
import org.elasticsearch.common.logging.LogConfigurator;
2626
import org.elasticsearch.core.IOUtils;
27+
import org.elasticsearch.logging.LogManager;
28+
import org.elasticsearch.logging.Logger;
2729
import org.elasticsearch.simdvec.VectorScorerFactory;
2830
import org.openjdk.jmh.annotations.Benchmark;
2931
import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -61,6 +63,10 @@ public class Int7uScorerBenchmark {
6163

6264
static {
6365
LogConfigurator.configureESLogging(); // native access requires logging to be initialized
66+
if (supportsHeapSegments() == false) {
67+
final Logger LOG = LogManager.getLogger(Int7uScorerBenchmark.class);
68+
LOG.warn("*Query targets cannot run on " + "JDK " + Runtime.version());
69+
}
6470
}
6571

6672
@Param({ "96", "768", "1024" })
@@ -129,15 +135,17 @@ public void setup() throws IOException {
129135
nativeSqrScorer = factory.getInt7SQVectorScorerSupplier(EUCLIDEAN, in, values, scoreCorrectionConstant).get().scorer();
130136
nativeSqrScorer.setScoringOrdinal(0);
131137

132-
// setup for getInt7SQVectorScorer / query vector scoring
133-
float[] queryVec = new float[dims];
134-
for (int i = 0; i < dims; i++) {
135-
queryVec[i] = ThreadLocalRandom.current().nextFloat();
138+
if (supportsHeapSegments()) {
139+
// setup for getInt7SQVectorScorer / query vector scoring
140+
float[] queryVec = new float[dims];
141+
for (int i = 0; i < dims; i++) {
142+
queryVec[i] = ThreadLocalRandom.current().nextFloat();
143+
}
144+
luceneDotScorerQuery = luceneScorer(values, VectorSimilarityFunction.DOT_PRODUCT, queryVec);
145+
nativeDotScorerQuery = factory.getInt7SQVectorScorer(VectorSimilarityFunction.DOT_PRODUCT, values, queryVec).get();
146+
luceneSqrScorerQuery = luceneScorer(values, VectorSimilarityFunction.EUCLIDEAN, queryVec);
147+
nativeSqrScorerQuery = factory.getInt7SQVectorScorer(VectorSimilarityFunction.EUCLIDEAN, values, queryVec).get();
136148
}
137-
luceneDotScorerQuery = luceneScorer(values, VectorSimilarityFunction.DOT_PRODUCT, queryVec);
138-
nativeDotScorerQuery = factory.getInt7SQVectorScorer(VectorSimilarityFunction.DOT_PRODUCT, values, queryVec).get();
139-
luceneSqrScorerQuery = luceneScorer(values, VectorSimilarityFunction.EUCLIDEAN, queryVec);
140-
nativeSqrScorerQuery = factory.getInt7SQVectorScorer(VectorSimilarityFunction.EUCLIDEAN, values, queryVec).get();
141149
}
142150

143151
@TearDown
@@ -208,6 +216,10 @@ public float squareDistanceNativeQuery() throws IOException {
208216
return nativeSqrScorerQuery.score(1);
209217
}
210218

219+
static boolean supportsHeapSegments() {
220+
return Runtime.version().feature() >= 22;
221+
}
222+
211223
QuantizedByteVectorValues vectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {
212224
var sq = new ScalarQuantizer(0.1f, 0.9f, (byte) 7);
213225
var slice = in.slice("values", 0, in.length());

‎benchmarks/src/test/java/org/elasticsearch/benchmark/vector/Int7uScorerBenchmarkTests.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,10 @@ public void testDotProduct() throws Exception {
4242
assertEquals(expected, bench.dotProductLucene(), delta);
4343
assertEquals(expected, bench.dotProductNative(), delta);
4444

45-
expected = bench.dotProductLuceneQuery();
46-
assertEquals(expected, bench.dotProductNativeQuery(), delta);
45+
if (Int7uScorerBenchmark.supportsHeapSegments()) {
46+
expected = bench.dotProductLuceneQuery();
47+
assertEquals(expected, bench.dotProductNativeQuery(), delta);
48+
}
4749
} finally {
4850
bench.teardown();
4951
}
@@ -60,8 +62,10 @@ public void testSquareDistance() throws Exception {
6062
assertEquals(expected, bench.squareDistanceLucene(), delta);
6163
assertEquals(expected, bench.squareDistanceNative(), delta);
6264

63-
expected = bench.squareDistanceLuceneQuery();
64-
assertEquals(expected, bench.squareDistanceNativeQuery(), delta);
65+
if (Int7uScorerBenchmark.supportsHeapSegments()) {
66+
expected = bench.squareDistanceLuceneQuery();
67+
assertEquals(expected, bench.squareDistanceNativeQuery(), delta);
68+
}
6569
} finally {
6670
bench.teardown();
6771
}

‎build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ java.nio.channels.SocketChannel#connect(java.net.SocketAddress)
3535
# org.elasticsearch.core.Booleans#parseBoolean(java.lang.String) directly on the string.
3636
@defaultMessage use org.elasticsearch.core.Booleans#parseBoolean(java.lang.String)
3737
java.lang.Boolean#getBoolean(java.lang.String)
38+
java.lang.Boolean#parseBoolean(java.lang.String)
39+
java.lang.Boolean#valueOf(java.lang.String)
3840

3941
org.apache.lucene.util.IOUtils @ use @org.elasticsearch.core.internal.io instead
4042

‎docs/changelog/128917.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 128917
2+
summary: Adopt a "LogicalPlan" approach to running multiple sub-queries (with INLINESTATS
3+
so far)
4+
area: ES|QL
5+
type: enhancement
6+
issues: []

‎docs/changelog/129872.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 129872
2+
summary: Run `TransportClusterStateAction` on local node
3+
area: Distributed
4+
type: enhancement
5+
issues: []

‎docs/changelog/130409.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 130409
2+
summary: Add Dependency Checker for `LogicalLocalPlanOptimizer`
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎docs/changelog/130576.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 130576
2+
summary: Avoid O(N^2) in VALUES with ordinals grouping
3+
area: ES|QL
4+
type: bug
5+
issues: []

‎docs/changelog/130594.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 130594
2+
summary: Add audit logging for stream content
3+
area: Network
4+
type: enhancement
5+
issues: []

0 commit comments

Comments
 (0)