Skip to content

Commit de218a6

Browse files
luigidellaquilaluigidellaquila
authored
Merge branch 'main' into esql/fix_bytesref2blockhash
2 parents 882bc2c + 31c2594 commit de218a6

File tree

447 files changed

+13629
-7216
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

447 files changed

+13629
-7216
lines changed

‎.buildkite/pipelines/periodic.template.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,17 @@ steps:
204204
image: family/elasticsearch-ubuntu-2404
205205
machineType: n2-standard-8
206206
buildDirectory: /dev/shm/bk
207+
- label: third-party / ms-graph
208+
command: |
209+
.ci/scripts/run-gradle.sh msGraphThirdPartyTest
210+
env:
211+
USE_3RD_PARTY_MS_GRAPH_CREDENTIALS: "true"
212+
timeout_in_minutes: 30
213+
agents:
214+
provider: gcp
215+
image: family/elasticsearch-ubuntu-2404
216+
machineType: n2-standard-8
217+
buildDirectory: /dev/shm/bk
207218
- group: lucene-compat
208219
steps:
209220
- label: "{{matrix.LUCENE_VERSION}} / lucene-compat"

‎.buildkite/pipelines/periodic.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -642,6 +642,17 @@ steps:
642642
image: family/elasticsearch-ubuntu-2404
643643
machineType: n2-standard-8
644644
buildDirectory: /dev/shm/bk
645+
- label: third-party / ms-graph
646+
command: |
647+
.ci/scripts/run-gradle.sh msGraphThirdPartyTest
648+
env:
649+
USE_3RD_PARTY_MS_GRAPH_CREDENTIALS: "true"
650+
timeout_in_minutes: 30
651+
agents:
652+
provider: gcp
653+
image: family/elasticsearch-ubuntu-2404
654+
machineType: n2-standard-8
655+
buildDirectory: /dev/shm/bk
645656
- group: lucene-compat
646657
steps:
647658
- label: "{{matrix.LUCENE_VERSION}} / lucene-compat"

‎.buildkite/scripts/third-party-test-credentials.sh‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,23 @@ if [[ "${USE_3RD_PARTY_GCS_CREDENTIALS:-}" == "true" ]]; then
4848
.buildkite/scripts/third-party-test-credentials.gcs.sh "$google_storage_service_account"
4949
fi
5050

51+
if [[ "${USE_3RD_PARTY_MS_GRAPH_CREDENTIALS:-}" == "true" ]]; then
52+
json=$(vault read -format=json secret/ci/elastic-elasticsearch/ms_graph_thirdparty_test_creds)
5153

54+
MS_GRAPH_TENANT_ID=$(echo "$json" | jq -r .data.tenant_id)
55+
export ms_graph_tenant_id="$MS_GRAPH_TENANT_ID"
56+
57+
MS_GRAPH_CLIENT_ID=$(echo "$json" | jq -r .data.client_id)
58+
export ms_graph_client_id="$MS_GRAPH_CLIENT_ID"
59+
60+
MS_GRAPH_CLIENT_SECRET=$(echo "$json" | jq -r .data.client_secret)
61+
export ms_graph_client_secret="$MS_GRAPH_CLIENT_SECRET"
62+
63+
MS_GRAPH_USERNAME=$(echo "$json" | jq -r .data.username)
64+
export ms_graph_username="$MS_GRAPH_USERNAME"
65+
66+
MS_GRAPH_GROUP_ID=$(echo "$json" | jq -r .data.group_id)
67+
export ms_graph_group_id="$MS_GRAPH_GROUP_ID"
68+
fi
5269

5370
unset json

‎.github/workflows/docs-preview-comment.yml‎

Lines changed: 0 additions & 71 deletions
This file was deleted.

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesAggregatorBenchmark.java‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,8 @@ static void selfTest() {
9595
try {
9696
for (String groups : ValuesAggregatorBenchmark.class.getField("groups").getAnnotationsByType(Param.class)[0].value()) {
9797
for (String dataType : ValuesAggregatorBenchmark.class.getField("dataType").getAnnotationsByType(Param.class)[0].value()) {
98-
run(Integer.parseInt(groups), dataType, 10);
98+
run(Integer.parseInt(groups), dataType, 10, 0);
99+
run(Integer.parseInt(groups), dataType, 10, 1);
99100
}
100101
}
101102
} catch (NoSuchFieldException e) {
@@ -113,7 +114,10 @@ static void selfTest() {
113114
@Param({ BYTES_REF, INT, LONG })
114115
public String dataType;
115116

116-
private static Operator operator(DriverContext driverContext, int groups, String dataType) {
117+
@Param({ "0", "1" })
118+
public int numOrdinalMerges;
119+
120+
private static Operator operator(DriverContext driverContext, int groups, String dataType, int numOrdinalMerges) {
117121
if (groups == 1) {
118122
return new AggregationOperator(
119123
List.of(supplier(dataType).aggregatorFactory(AggregatorMode.SINGLE, List.of(0)).apply(driverContext)),
@@ -125,7 +129,24 @@ private static Operator operator(DriverContext driverContext, int groups, String
125129
List.of(supplier(dataType).groupingAggregatorFactory(AggregatorMode.SINGLE, List.of(1))),
126130
() -> BlockHash.build(groupSpec, driverContext.blockFactory(), 16 * 1024, false),
127131
driverContext
128-
);
132+
) {
133+
@Override
134+
public Page getOutput() {
135+
mergeOrdinal();
136+
return super.getOutput();
137+
}
138+
139+
// simulate OrdinalsGroupingOperator
140+
void mergeOrdinal() {
141+
var merged = supplier(dataType).groupingAggregatorFactory(AggregatorMode.SINGLE, List.of(1)).apply(driverContext);
142+
for (int i = 0; i < numOrdinalMerges; i++) {
143+
for (int p = 0; p < groups; p++) {
144+
merged.addIntermediateRow(p, aggregators.getFirst(), p);
145+
}
146+
}
147+
aggregators.set(0, merged);
148+
}
149+
};
129150
}
130151

131152
private static AggregatorFunctionSupplier supplier(String dataType) {
@@ -331,12 +352,12 @@ private static Block groupingBlock(int groups) {
331352

332353
@Benchmark
333354
public void run() {
334-
run(groups, dataType, OP_COUNT);
355+
run(groups, dataType, OP_COUNT, numOrdinalMerges);
335356
}
336357

337-
private static void run(int groups, String dataType, int opCount) {
358+
private static void run(int groups, String dataType, int opCount, int numOrdinalMerges) {
338359
DriverContext driverContext = driverContext();
339-
try (Operator operator = operator(driverContext, groups, dataType)) {
360+
try (Operator operator = operator(driverContext, groups, dataType, numOrdinalMerges)) {
340361
Page page = page(groups, dataType);
341362
for (int i = 0; i < opCount; i++) {
342363
operator.addInput(page.shallowCopy());

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@
4141
import org.elasticsearch.compute.data.Page;
4242
import org.elasticsearch.compute.lucene.LuceneSourceOperator;
4343
import org.elasticsearch.compute.lucene.ShardRefCounted;
44-
import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator;
44+
import org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperator;
45+
import org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperatorStatus;
4546
import org.elasticsearch.compute.operator.topn.TopNOperator;
4647
import org.elasticsearch.core.IOUtils;
4748
import org.elasticsearch.index.IndexSettings;
@@ -343,7 +344,7 @@ public void benchmark() {
343344
);
344345
long sum = 0;
345346
for (Page page : pages) {
346-
op.addInput(page);
347+
op.addInput(page.shallowCopy());
347348
switch (name) {
348349
case "long" -> {
349350
LongVector values = op.getOutput().<LongBlock>getBlock(1).asVector();
@@ -411,7 +412,7 @@ public void benchmark() {
411412
throw new AssertionError("[" + layout + "][" + name + "] expected [" + expected + "] but was [" + sum + "]");
412413
}
413414
boolean foundStoredFieldLoader = false;
414-
ValuesSourceReaderOperator.Status status = (ValuesSourceReaderOperator.Status) op.status();
415+
ValuesSourceReaderOperatorStatus status = (ValuesSourceReaderOperatorStatus) op.status();
415416
for (Map.Entry<String, Integer> e : status.readersBuilt().entrySet()) {
416417
if (e.getKey().indexOf("stored_fields") >= 0) {
417418
foundStoredFieldLoader = true;

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/Int7uScorerBenchmark.java‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
import org.apache.lucene.util.quantization.ScalarQuantizer;
2525
import org.elasticsearch.common.logging.LogConfigurator;
2626
import org.elasticsearch.core.IOUtils;
27+
import org.elasticsearch.logging.LogManager;
28+
import org.elasticsearch.logging.Logger;
2729
import org.elasticsearch.simdvec.VectorScorerFactory;
2830
import org.openjdk.jmh.annotations.Benchmark;
2931
import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -61,6 +63,10 @@ public class Int7uScorerBenchmark {
6163

6264
static {
6365
LogConfigurator.configureESLogging(); // native access requires logging to be initialized
66+
if (supportsHeapSegments() == false) {
67+
final Logger LOG = LogManager.getLogger(Int7uScorerBenchmark.class);
68+
LOG.warn("*Query targets cannot run on " + "JDK " + Runtime.version());
69+
}
6470
}
6571

6672
@Param({ "96", "768", "1024" })
@@ -129,15 +135,17 @@ public void setup() throws IOException {
129135
nativeSqrScorer = factory.getInt7SQVectorScorerSupplier(EUCLIDEAN, in, values, scoreCorrectionConstant).get().scorer();
130136
nativeSqrScorer.setScoringOrdinal(0);
131137

132-
// setup for getInt7SQVectorScorer / query vector scoring
133-
float[] queryVec = new float[dims];
134-
for (int i = 0; i < dims; i++) {
135-
queryVec[i] = ThreadLocalRandom.current().nextFloat();
138+
if (supportsHeapSegments()) {
139+
// setup for getInt7SQVectorScorer / query vector scoring
140+
float[] queryVec = new float[dims];
141+
for (int i = 0; i < dims; i++) {
142+
queryVec[i] = ThreadLocalRandom.current().nextFloat();
143+
}
144+
luceneDotScorerQuery = luceneScorer(values, VectorSimilarityFunction.DOT_PRODUCT, queryVec);
145+
nativeDotScorerQuery = factory.getInt7SQVectorScorer(VectorSimilarityFunction.DOT_PRODUCT, values, queryVec).get();
146+
luceneSqrScorerQuery = luceneScorer(values, VectorSimilarityFunction.EUCLIDEAN, queryVec);
147+
nativeSqrScorerQuery = factory.getInt7SQVectorScorer(VectorSimilarityFunction.EUCLIDEAN, values, queryVec).get();
136148
}
137-
luceneDotScorerQuery = luceneScorer(values, VectorSimilarityFunction.DOT_PRODUCT, queryVec);
138-
nativeDotScorerQuery = factory.getInt7SQVectorScorer(VectorSimilarityFunction.DOT_PRODUCT, values, queryVec).get();
139-
luceneSqrScorerQuery = luceneScorer(values, VectorSimilarityFunction.EUCLIDEAN, queryVec);
140-
nativeSqrScorerQuery = factory.getInt7SQVectorScorer(VectorSimilarityFunction.EUCLIDEAN, values, queryVec).get();
141149
}
142150

143151
@TearDown
@@ -208,6 +216,10 @@ public float squareDistanceNativeQuery() throws IOException {
208216
return nativeSqrScorerQuery.score(1);
209217
}
210218

219+
static boolean supportsHeapSegments() {
220+
return Runtime.version().feature() >= 22;
221+
}
222+
211223
QuantizedByteVectorValues vectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {
212224
var sq = new ScalarQuantizer(0.1f, 0.9f, (byte) 7);
213225
var slice = in.slice("values", 0, in.length());

‎benchmarks/src/test/java/org/elasticsearch/benchmark/vector/Int7uScorerBenchmarkTests.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,10 @@ public void testDotProduct() throws Exception {
4242
assertEquals(expected, bench.dotProductLucene(), delta);
4343
assertEquals(expected, bench.dotProductNative(), delta);
4444

45-
expected = bench.dotProductLuceneQuery();
46-
assertEquals(expected, bench.dotProductNativeQuery(), delta);
45+
if (Int7uScorerBenchmark.supportsHeapSegments()) {
46+
expected = bench.dotProductLuceneQuery();
47+
assertEquals(expected, bench.dotProductNativeQuery(), delta);
48+
}
4749
} finally {
4850
bench.teardown();
4951
}
@@ -60,8 +62,10 @@ public void testSquareDistance() throws Exception {
6062
assertEquals(expected, bench.squareDistanceLucene(), delta);
6163
assertEquals(expected, bench.squareDistanceNative(), delta);
6264

63-
expected = bench.squareDistanceLuceneQuery();
64-
assertEquals(expected, bench.squareDistanceNativeQuery(), delta);
65+
if (Int7uScorerBenchmark.supportsHeapSegments()) {
66+
expected = bench.squareDistanceLuceneQuery();
67+
assertEquals(expected, bench.squareDistanceNativeQuery(), delta);
68+
}
6569
} finally {
6670
bench.teardown();
6771
}

‎build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ java.nio.channels.SocketChannel#connect(java.net.SocketAddress)
3535
# org.elasticsearch.core.Booleans#parseBoolean(java.lang.String) directly on the string.
3636
@defaultMessage use org.elasticsearch.core.Booleans#parseBoolean(java.lang.String)
3737
java.lang.Boolean#getBoolean(java.lang.String)
38+
java.lang.Boolean#parseBoolean(java.lang.String)
39+
java.lang.Boolean#valueOf(java.lang.String)
3840

3941
org.apache.lucene.util.IOUtils @ use @org.elasticsearch.core.internal.io instead
4042

‎docs/changelog/128917.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 128917
2+
summary: Adopt a "LogicalPlan" approach to running multiple sub-queries (with INLINESTATS
3+
so far)
4+
area: ES|QL
5+
type: enhancement
6+
issues: []

0 commit comments

Comments
 (0)