Skip to content

Commit 72bda3b

Browse files
mhl-bmhl-b
authored
Merge branch 'main' into nit-empty-incremental-bulk
2 parents feb03da + de98e5d commit 72bda3b

File tree

754 files changed

+13200
-3818
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

754 files changed

+13200
-3818
lines changed

‎.buildkite/pipelines/periodic-java-ea.template.yml‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,12 @@ steps:
3030
buildDirectory: /dev/shm/bk
3131
- group: java-matrix
3232
steps:
33-
- label: "{{matrix.GRADLE_TASK}} / java-ea"
33+
- label: "{{matrix.GRADLE_TASK}} / {{matrix.image}} "
3434
command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true $$GRADLE_TASK
3535
timeout_in_minutes: 300
3636
matrix:
3737
setup:
3838
image:
39-
- windows-2025
4039
- ubuntu-2404
4140
GRADLE_TASK:
4241
- checkPart1
@@ -53,6 +52,30 @@ steps:
5352
buildDirectory: /dev/shm/bk
5453
env:
5554
GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
55+
- label: "{{matrix.GRADLE_TASK}} / {{matrix.image}} "
56+
command: |
57+
.\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh
58+
timeout_in_minutes: 300
59+
matrix:
60+
setup:
61+
image:
62+
- windows-2025
63+
GRADLE_TASK:
64+
- checkPart1
65+
- checkPart2
66+
- checkPart3
67+
- checkPart4
68+
- checkPart5
69+
- checkPart6
70+
- checkRestCompat
71+
agents:
72+
provider: gcp
73+
image: family/elasticsearch-{{matrix.image}}
74+
machineType: n1-standard-32
75+
buildDirectory: /dev/shm/bk
76+
env:
77+
GRADLE_PARAMS: "-Druntime.java=$$JAVA_EA_VERSION"
78+
GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
5679
- label: "{{matrix.BWC_VERSION}} / matrix-bwc"
5780
command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v$$BWC_VERSION#bwcTest
5881
timeout_in_minutes: 300

‎.buildkite/pipelines/periodic-java-ea.yml‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -468,13 +468,12 @@ steps:
468468
buildDirectory: /dev/shm/bk
469469
- group: java-matrix
470470
steps:
471-
- label: "{{matrix.GRADLE_TASK}} / java-ea"
471+
- label: "{{matrix.GRADLE_TASK}} / {{matrix.image}} "
472472
command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true $$GRADLE_TASK
473473
timeout_in_minutes: 300
474474
matrix:
475475
setup:
476476
image:
477-
- windows-2025
478477
- ubuntu-2404
479478
GRADLE_TASK:
480479
- checkPart1
@@ -491,6 +490,30 @@ steps:
491490
buildDirectory: /dev/shm/bk
492491
env:
493492
GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
493+
- label: "{{matrix.GRADLE_TASK}} / {{matrix.image}} "
494+
command: |
495+
.\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh
496+
timeout_in_minutes: 300
497+
matrix:
498+
setup:
499+
image:
500+
- windows-2025
501+
GRADLE_TASK:
502+
- checkPart1
503+
- checkPart2
504+
- checkPart3
505+
- checkPart4
506+
- checkPart5
507+
- checkPart6
508+
- checkRestCompat
509+
agents:
510+
provider: gcp
511+
image: family/elasticsearch-{{matrix.image}}
512+
machineType: n1-standard-32
513+
buildDirectory: /dev/shm/bk
514+
env:
515+
GRADLE_PARAMS: "-Druntime.java=${JAVA_EA_VERSION}"
516+
GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
494517
- label: "{{matrix.BWC_VERSION}} / matrix-bwc"
495518
command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v$$BWC_VERSION#bwcTest
496519
timeout_in_minutes: 300

‎.buildkite/scripts/windows-run-gradle.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@
22

33
set -euo pipefail
44

5-
.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $GRADLE_TASK
5+
.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true ${GRADLE_PARAMS:-} $GRADLE_TASK

‎.github/CODEOWNERS‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,11 @@ x-pack/plugin/apm-data/src/yamlRestTest/resources @elastic/obs-ds-intake-service
3434
x-pack/plugin/otel-data/src/main/resources @elastic/obs-ds-intake-services
3535
x-pack/plugin/otel-data/src/yamlRestTest/resources @elastic/obs-ds-intake-services
3636

37+
# Storage Engine
38+
x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp @elastic/es-storage-engine
39+
x-pack/plugin/otel-data/src/test/java/org/elasticsearch/xpack/oteldata/otlp @elastic/es-storage-engine
40+
x-pack/plugin/otel-data/src/javaRestTest/java/org/elasticsearch/xpack/oteldata/otlp @elastic/es-storage-engine
41+
3742
# Delivery
3843
gradle @elastic/es-delivery
3944
build-conventions @elastic/es-delivery

‎benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ private ExponentialHistogram asCompressedHistogram(ExponentialHistogram histogra
130130
CompressedExponentialHistogram.writeHistogramBytes(histoBytes, histogram.scale(), negativeBuckets, positiveBuckets);
131131
CompressedExponentialHistogram result = new CompressedExponentialHistogram();
132132
BytesRef data = histoBytes.bytes().toBytesRef();
133-
result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), histogram.min(), data);
133+
result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), histogram.min(), histogram.max(), data);
134134
return result;
135135
} catch (IOException e) {
136136
throw new RuntimeException(e);

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OSQScorerBenchmark.java‎

Lines changed: 88 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,10 @@
1414
import org.apache.lucene.store.IndexInput;
1515
import org.apache.lucene.store.IndexOutput;
1616
import org.apache.lucene.store.MMapDirectory;
17-
import org.apache.lucene.util.VectorUtil;
17+
import org.apache.lucene.store.NIOFSDirectory;
1818
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
1919
import org.elasticsearch.common.logging.LogConfigurator;
20+
import org.elasticsearch.core.IOUtils;
2021
import org.elasticsearch.simdvec.ES91OSQVectorsScorer;
2122
import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
2223
import org.openjdk.jmh.annotations.Benchmark;
@@ -29,6 +30,7 @@
2930
import org.openjdk.jmh.annotations.Scope;
3031
import org.openjdk.jmh.annotations.Setup;
3132
import org.openjdk.jmh.annotations.State;
33+
import org.openjdk.jmh.annotations.TearDown;
3234
import org.openjdk.jmh.annotations.Warmup;
3335
import org.openjdk.jmh.infra.Blackhole;
3436

@@ -66,9 +68,14 @@ public class OSQScorerBenchmark {
6668
float centroidDp;
6769

6870
byte[] scratch;
69-
ES91OSQVectorsScorer scorer;
71+
ES91OSQVectorsScorer scorerMmap;
72+
ES91OSQVectorsScorer scorerNfios;
7073

71-
IndexInput in;
74+
Directory dirMmap;
75+
IndexInput inMmap;
76+
77+
Directory dirNiofs;
78+
IndexInput inNiofs;
7279

7380
float[] scratchScores;
7481
float[] corrections;
@@ -84,18 +91,24 @@ public void setup() throws IOException {
8491
random.nextBytes(binaryVector);
8592
}
8693

87-
Directory dir = new MMapDirectory(Files.createTempDirectory("vectorData"));
88-
IndexOutput out = dir.createOutput("vectors", IOContext.DEFAULT);
94+
dirMmap = new MMapDirectory(Files.createTempDirectory("vectorDataMmap"));
95+
dirNiofs = new NIOFSDirectory(Files.createTempDirectory("vectorDataNFIOS"));
96+
IndexOutput outMmap = dirMmap.createOutput("vectors", IOContext.DEFAULT);
97+
IndexOutput outNfios = dirNiofs.createOutput("vectors", IOContext.DEFAULT);
8998
byte[] correctionBytes = new byte[14 * ES91OSQVectorsScorer.BULK_SIZE];
9099
for (int i = 0; i < numVectors; i += ES91OSQVectorsScorer.BULK_SIZE) {
91100
for (int j = 0; j < ES91OSQVectorsScorer.BULK_SIZE; j++) {
92-
out.writeBytes(binaryVectors[i + j], 0, binaryVectors[i + j].length);
101+
outMmap.writeBytes(binaryVectors[i + j], 0, binaryVectors[i + j].length);
102+
outNfios.writeBytes(binaryVectors[i + j], 0, binaryVectors[i + j].length);
93103
}
94104
random.nextBytes(correctionBytes);
95-
out.writeBytes(correctionBytes, 0, correctionBytes.length);
105+
outMmap.writeBytes(correctionBytes, 0, correctionBytes.length);
106+
outNfios.writeBytes(correctionBytes, 0, correctionBytes.length);
96107
}
97-
out.close();
98-
in = dir.openInput("vectors", IOContext.DEFAULT);
108+
outMmap.close();
109+
outNfios.close();
110+
inMmap = dirMmap.openInput("vectors", IOContext.DEFAULT);
111+
inNiofs = dirNiofs.openInput("vectors", IOContext.DEFAULT);
99112

100113
binaryQueries = new byte[numVectors][4 * length];
101114
for (byte[] binaryVector : binaryVectors) {
@@ -110,42 +123,40 @@ public void setup() throws IOException {
110123
centroidDp = random.nextFloat();
111124

112125
scratch = new byte[length];
113-
scorer = ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(in, dims);
126+
scorerMmap = ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(inMmap, dims);
127+
scorerNfios = ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(inNiofs, dims);
114128
scratchScores = new float[16];
115129
corrections = new float[3];
116130
}
117131

132+
@TearDown
133+
public void teardown() throws IOException {
134+
IOUtils.close(dirMmap, inMmap, dirNiofs, inNiofs);
135+
}
136+
137+
@Benchmark
138+
public void scoreFromMemorySegmentOnlyVectorMmapScalar(Blackhole bh) throws IOException {
139+
scoreFromMemorySegmentOnlyVector(bh, inMmap, scorerMmap);
140+
}
141+
118142
@Benchmark
119143
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
120-
public void scoreFromArray(Blackhole bh) throws IOException {
121-
for (int j = 0; j < numQueries; j++) {
122-
in.seek(0);
123-
for (int i = 0; i < numVectors; i++) {
124-
in.readBytes(scratch, 0, length);
125-
float qDist = VectorUtil.int4BitDotProduct(binaryQueries[j], scratch);
126-
in.readFloats(corrections, 0, corrections.length);
127-
int addition = Short.toUnsignedInt(in.readShort());
128-
float score = scorer.score(
129-
result.lowerInterval(),
130-
result.upperInterval(),
131-
result.quantizedComponentSum(),
132-
result.additionalCorrection(),
133-
VectorSimilarityFunction.EUCLIDEAN,
134-
centroidDp,
135-
corrections[0],
136-
corrections[1],
137-
addition,
138-
corrections[2],
139-
qDist
140-
);
141-
bh.consume(score);
142-
}
143-
}
144+
public void scoreFromMemorySegmentOnlyVectorMmapVect(Blackhole bh) throws IOException {
145+
scoreFromMemorySegmentOnlyVector(bh, inMmap, scorerMmap);
146+
}
147+
148+
@Benchmark
149+
public void scoreFromMemorySegmentOnlyVectorNiofsScalar(Blackhole bh) throws IOException {
150+
scoreFromMemorySegmentOnlyVector(bh, inNiofs, scorerNfios);
144151
}
145152

146153
@Benchmark
147154
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
148-
public void scoreFromMemorySegmentOnlyVector(Blackhole bh) throws IOException {
155+
public void scoreFromMemorySegmentOnlyVectorNiofsVect(Blackhole bh) throws IOException {
156+
scoreFromMemorySegmentOnlyVector(bh, inNiofs, scorerNfios);
157+
}
158+
159+
private void scoreFromMemorySegmentOnlyVector(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
149160
for (int j = 0; j < numQueries; j++) {
150161
in.seek(0);
151162
for (int i = 0; i < numVectors; i++) {
@@ -170,9 +181,29 @@ public void scoreFromMemorySegmentOnlyVector(Blackhole bh) throws IOException {
170181
}
171182
}
172183

184+
@Benchmark
185+
public void scoreFromMemorySegmentOnlyVectorBulkMmapScalar(Blackhole bh) throws IOException {
186+
scoreFromMemorySegmentOnlyVectorBulk(bh, inMmap, scorerMmap);
187+
}
188+
189+
@Benchmark
190+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
191+
public void scoreFromMemorySegmentOnlyVectorBulkMmapVect(Blackhole bh) throws IOException {
192+
scoreFromMemorySegmentOnlyVectorBulk(bh, inMmap, scorerMmap);
193+
}
194+
195+
@Benchmark
196+
public void scoreFromMemorySegmentOnlyVectorBulkNiofsScalar(Blackhole bh) throws IOException {
197+
scoreFromMemorySegmentOnlyVectorBulk(bh, inNiofs, scorerNfios);
198+
}
199+
173200
@Benchmark
174201
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
175-
public void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh) throws IOException {
202+
public void scoreFromMemorySegmentOnlyVectorBulkNiofsVect(Blackhole bh) throws IOException {
203+
scoreFromMemorySegmentOnlyVectorBulk(bh, inNiofs, scorerNfios);
204+
}
205+
206+
private void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
176207
for (int j = 0; j < numQueries; j++) {
177208
in.seek(0);
178209
for (int i = 0; i < numVectors; i += 16) {
@@ -199,9 +230,29 @@ public void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh) throws IOExceptio
199230
}
200231
}
201232

233+
@Benchmark
234+
public void scoreFromMemorySegmentAllBulkMmapScalar(Blackhole bh) throws IOException {
235+
scoreFromMemorySegmentAllBulk(bh, inMmap, scorerMmap);
236+
}
237+
238+
@Benchmark
239+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
240+
public void scoreFromMemorySegmentAllBulkMmapVect(Blackhole bh) throws IOException {
241+
scoreFromMemorySegmentAllBulk(bh, inMmap, scorerMmap);
242+
}
243+
244+
@Benchmark
245+
public void scoreFromMemorySegmentAllBulkNiofsScalar(Blackhole bh) throws IOException {
246+
scoreFromMemorySegmentAllBulk(bh, inNiofs, scorerNfios);
247+
}
248+
202249
@Benchmark
203250
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
204-
public void scoreFromMemorySegmentAllBulk(Blackhole bh) throws IOException {
251+
public void scoreFromMemorySegmentAllBulkNiofsVect(Blackhole bh) throws IOException {
252+
scoreFromMemorySegmentAllBulk(bh, inNiofs, scorerNfios);
253+
}
254+
255+
private void scoreFromMemorySegmentAllBulk(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
205256
for (int j = 0; j < numQueries; j++) {
206257
in.seek(0);
207258
for (int i = 0; i < numVectors; i += 16) {

‎branches.json‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,29 @@
11
{
2-
"notice": "This file is not maintained outside of the main branch and should only be used for tooling.",
3-
"branches": [
2+
"notice" : "This file is not maintained outside of the main branch and should only be used for tooling.",
3+
"branches" : [
44
{
5-
"branch": "main"
5+
"branch" : "main",
6+
"version" : "9.2.0"
67
},
78
{
8-
"branch": "9.1"
9+
"branch" : "9.1",
10+
"version" : "9.1.4"
911
},
1012
{
11-
"branch": "9.0"
13+
"branch" : "9.0",
14+
"version" : "9.0.7"
1215
},
1316
{
14-
"branch": "8.19"
17+
"branch" : "8.19",
18+
"version" : "8.19.4"
1519
},
1620
{
17-
"branch": "8.18"
21+
"branch" : "8.18",
22+
"version" : "8.18.7"
1823
},
1924
{
20-
"branch": "7.17"
25+
"branch" : "7.17",
26+
"version" : "7.17.30"
2127
}
2228
]
2329
}

‎build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesPlugin.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818

1919
public class VersionPropertiesPlugin implements Plugin<Project> {
2020

21+
public static final String VERSIONS_EXT = "versions";
22+
2123
@Override
2224
public void apply(Project project) {
2325
File workspaceDir = Util.locateElasticsearchWorkspace(project.getGradle());
@@ -28,6 +30,6 @@ public void apply(Project project) {
2830
.registerIfAbsent("versions", VersionPropertiesBuildService.class, spec -> {
2931
spec.getParameters().getInfoPath().set(infoPath);
3032
});
31-
project.getExtensions().add("versions", serviceProvider.get().getProperties());
33+
project.getExtensions().add(VERSIONS_EXT, serviceProvider.get().getProperties());
3234
}
3335
}

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,17 @@ abstract class AbstractGitAwareGradleFuncTest extends AbstractGradleFuncTest {
2626
execute("git clone ${remoteGitRepo.absolutePath} cloned", testProjectDir.root)
2727
buildFile = new File(testProjectDir.root, 'cloned/build.gradle')
2828
settingsFile = new File(testProjectDir.root, 'cloned/settings.gradle')
29+
versionPropertiesFile = new File(testProjectDir.root, 'cloned/build-tools-internal/version.properties')
30+
versionPropertiesFile.text = """
31+
elasticsearch = 9.1.0
32+
lucene = 10.2.2
33+
34+
bundled_jdk_vendor = openjdk
35+
bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90
36+
minimumJdkVersion = 21
37+
minimumRuntimeJava = 21
38+
minimumCompilerJava = 21
39+
"""
2940
}
3041

3142
File setupGitRemote() {

0 commit comments

Comments
 (0)