Skip to content

Commit 04a8ac1

Browse files
Mikep86Mikep86
committed
Merge branch 'main' into semantic-text_semantic-query-multi-index-support
2 parents d7f877b + de98e5d commit 04a8ac1

File tree

108 files changed

+2985
-608
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

108 files changed

+2985
-608
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OSQScorerBenchmark.java‎

Lines changed: 88 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,10 @@
1414
import org.apache.lucene.store.IndexInput;
1515
import org.apache.lucene.store.IndexOutput;
1616
import org.apache.lucene.store.MMapDirectory;
17-
import org.apache.lucene.util.VectorUtil;
17+
import org.apache.lucene.store.NIOFSDirectory;
1818
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
1919
import org.elasticsearch.common.logging.LogConfigurator;
20+
import org.elasticsearch.core.IOUtils;
2021
import org.elasticsearch.simdvec.ES91OSQVectorsScorer;
2122
import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
2223
import org.openjdk.jmh.annotations.Benchmark;
@@ -29,6 +30,7 @@
2930
import org.openjdk.jmh.annotations.Scope;
3031
import org.openjdk.jmh.annotations.Setup;
3132
import org.openjdk.jmh.annotations.State;
33+
import org.openjdk.jmh.annotations.TearDown;
3234
import org.openjdk.jmh.annotations.Warmup;
3335
import org.openjdk.jmh.infra.Blackhole;
3436

@@ -66,9 +68,14 @@ public class OSQScorerBenchmark {
6668
float centroidDp;
6769

6870
byte[] scratch;
69-
ES91OSQVectorsScorer scorer;
71+
ES91OSQVectorsScorer scorerMmap;
72+
ES91OSQVectorsScorer scorerNfios;
7073

71-
IndexInput in;
74+
Directory dirMmap;
75+
IndexInput inMmap;
76+
77+
Directory dirNiofs;
78+
IndexInput inNiofs;
7279

7380
float[] scratchScores;
7481
float[] corrections;
@@ -84,18 +91,24 @@ public void setup() throws IOException {
8491
random.nextBytes(binaryVector);
8592
}
8693

87-
Directory dir = new MMapDirectory(Files.createTempDirectory("vectorData"));
88-
IndexOutput out = dir.createOutput("vectors", IOContext.DEFAULT);
94+
dirMmap = new MMapDirectory(Files.createTempDirectory("vectorDataMmap"));
95+
dirNiofs = new NIOFSDirectory(Files.createTempDirectory("vectorDataNFIOS"));
96+
IndexOutput outMmap = dirMmap.createOutput("vectors", IOContext.DEFAULT);
97+
IndexOutput outNfios = dirNiofs.createOutput("vectors", IOContext.DEFAULT);
8998
byte[] correctionBytes = new byte[14 * ES91OSQVectorsScorer.BULK_SIZE];
9099
for (int i = 0; i < numVectors; i += ES91OSQVectorsScorer.BULK_SIZE) {
91100
for (int j = 0; j < ES91OSQVectorsScorer.BULK_SIZE; j++) {
92-
out.writeBytes(binaryVectors[i + j], 0, binaryVectors[i + j].length);
101+
outMmap.writeBytes(binaryVectors[i + j], 0, binaryVectors[i + j].length);
102+
outNfios.writeBytes(binaryVectors[i + j], 0, binaryVectors[i + j].length);
93103
}
94104
random.nextBytes(correctionBytes);
95-
out.writeBytes(correctionBytes, 0, correctionBytes.length);
105+
outMmap.writeBytes(correctionBytes, 0, correctionBytes.length);
106+
outNfios.writeBytes(correctionBytes, 0, correctionBytes.length);
96107
}
97-
out.close();
98-
in = dir.openInput("vectors", IOContext.DEFAULT);
108+
outMmap.close();
109+
outNfios.close();
110+
inMmap = dirMmap.openInput("vectors", IOContext.DEFAULT);
111+
inNiofs = dirNiofs.openInput("vectors", IOContext.DEFAULT);
99112

100113
binaryQueries = new byte[numVectors][4 * length];
101114
for (byte[] binaryVector : binaryVectors) {
@@ -110,42 +123,40 @@ public void setup() throws IOException {
110123
centroidDp = random.nextFloat();
111124

112125
scratch = new byte[length];
113-
scorer = ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(in, dims);
126+
scorerMmap = ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(inMmap, dims);
127+
scorerNfios = ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(inNiofs, dims);
114128
scratchScores = new float[16];
115129
corrections = new float[3];
116130
}
117131

132+
@TearDown
133+
public void teardown() throws IOException {
134+
IOUtils.close(dirMmap, inMmap, dirNiofs, inNiofs);
135+
}
136+
137+
@Benchmark
138+
public void scoreFromMemorySegmentOnlyVectorMmapScalar(Blackhole bh) throws IOException {
139+
scoreFromMemorySegmentOnlyVector(bh, inMmap, scorerMmap);
140+
}
141+
118142
@Benchmark
119143
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
120-
public void scoreFromArray(Blackhole bh) throws IOException {
121-
for (int j = 0; j < numQueries; j++) {
122-
in.seek(0);
123-
for (int i = 0; i < numVectors; i++) {
124-
in.readBytes(scratch, 0, length);
125-
float qDist = VectorUtil.int4BitDotProduct(binaryQueries[j], scratch);
126-
in.readFloats(corrections, 0, corrections.length);
127-
int addition = Short.toUnsignedInt(in.readShort());
128-
float score = scorer.score(
129-
result.lowerInterval(),
130-
result.upperInterval(),
131-
result.quantizedComponentSum(),
132-
result.additionalCorrection(),
133-
VectorSimilarityFunction.EUCLIDEAN,
134-
centroidDp,
135-
corrections[0],
136-
corrections[1],
137-
addition,
138-
corrections[2],
139-
qDist
140-
);
141-
bh.consume(score);
142-
}
143-
}
144+
public void scoreFromMemorySegmentOnlyVectorMmapVect(Blackhole bh) throws IOException {
145+
scoreFromMemorySegmentOnlyVector(bh, inMmap, scorerMmap);
146+
}
147+
148+
@Benchmark
149+
public void scoreFromMemorySegmentOnlyVectorNiofsScalar(Blackhole bh) throws IOException {
150+
scoreFromMemorySegmentOnlyVector(bh, inNiofs, scorerNfios);
144151
}
145152

146153
@Benchmark
147154
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
148-
public void scoreFromMemorySegmentOnlyVector(Blackhole bh) throws IOException {
155+
public void scoreFromMemorySegmentOnlyVectorNiofsVect(Blackhole bh) throws IOException {
156+
scoreFromMemorySegmentOnlyVector(bh, inNiofs, scorerNfios);
157+
}
158+
159+
private void scoreFromMemorySegmentOnlyVector(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
149160
for (int j = 0; j < numQueries; j++) {
150161
in.seek(0);
151162
for (int i = 0; i < numVectors; i++) {
@@ -170,9 +181,29 @@ public void scoreFromMemorySegmentOnlyVector(Blackhole bh) throws IOException {
170181
}
171182
}
172183

184+
@Benchmark
185+
public void scoreFromMemorySegmentOnlyVectorBulkMmapScalar(Blackhole bh) throws IOException {
186+
scoreFromMemorySegmentOnlyVectorBulk(bh, inMmap, scorerMmap);
187+
}
188+
189+
@Benchmark
190+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
191+
public void scoreFromMemorySegmentOnlyVectorBulkMmapVect(Blackhole bh) throws IOException {
192+
scoreFromMemorySegmentOnlyVectorBulk(bh, inMmap, scorerMmap);
193+
}
194+
195+
@Benchmark
196+
public void scoreFromMemorySegmentOnlyVectorBulkNiofsScalar(Blackhole bh) throws IOException {
197+
scoreFromMemorySegmentOnlyVectorBulk(bh, inNiofs, scorerNfios);
198+
}
199+
173200
@Benchmark
174201
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
175-
public void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh) throws IOException {
202+
public void scoreFromMemorySegmentOnlyVectorBulkNiofsVect(Blackhole bh) throws IOException {
203+
scoreFromMemorySegmentOnlyVectorBulk(bh, inNiofs, scorerNfios);
204+
}
205+
206+
private void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
176207
for (int j = 0; j < numQueries; j++) {
177208
in.seek(0);
178209
for (int i = 0; i < numVectors; i += 16) {
@@ -199,9 +230,29 @@ public void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh) throws IOExceptio
199230
}
200231
}
201232

233+
@Benchmark
234+
public void scoreFromMemorySegmentAllBulkMmapScalar(Blackhole bh) throws IOException {
235+
scoreFromMemorySegmentAllBulk(bh, inMmap, scorerMmap);
236+
}
237+
238+
@Benchmark
239+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
240+
public void scoreFromMemorySegmentAllBulkMmapVect(Blackhole bh) throws IOException {
241+
scoreFromMemorySegmentAllBulk(bh, inMmap, scorerMmap);
242+
}
243+
244+
@Benchmark
245+
public void scoreFromMemorySegmentAllBulkNiofsScalar(Blackhole bh) throws IOException {
246+
scoreFromMemorySegmentAllBulk(bh, inNiofs, scorerNfios);
247+
}
248+
202249
@Benchmark
203250
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
204-
public void scoreFromMemorySegmentAllBulk(Blackhole bh) throws IOException {
251+
public void scoreFromMemorySegmentAllBulkNiofsVect(Blackhole bh) throws IOException {
252+
scoreFromMemorySegmentAllBulk(bh, inNiofs, scorerNfios);
253+
}
254+
255+
private void scoreFromMemorySegmentAllBulk(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
205256
for (int j = 0; j < numQueries; j++) {
206257
in.seek(0);
207258
for (int i = 0; i < numVectors; i += 16) {

‎build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesPlugin.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818

1919
public class VersionPropertiesPlugin implements Plugin<Project> {
2020

21+
public static final String VERSIONS_EXT = "versions";
22+
2123
@Override
2224
public void apply(Project project) {
2325
File workspaceDir = Util.locateElasticsearchWorkspace(project.getGradle());
@@ -28,6 +30,6 @@ public void apply(Project project) {
2830
.registerIfAbsent("versions", VersionPropertiesBuildService.class, spec -> {
2931
spec.getParameters().getInfoPath().set(infoPath);
3032
});
31-
project.getExtensions().add("versions", serviceProvider.get().getProperties());
33+
project.getExtensions().add(VERSIONS_EXT, serviceProvider.get().getProperties());
3234
}
3335
}

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,17 @@ abstract class AbstractGitAwareGradleFuncTest extends AbstractGradleFuncTest {
2626
execute("git clone ${remoteGitRepo.absolutePath} cloned", testProjectDir.root)
2727
buildFile = new File(testProjectDir.root, 'cloned/build.gradle')
2828
settingsFile = new File(testProjectDir.root, 'cloned/settings.gradle')
29+
versionPropertiesFile = new File(testProjectDir.root, 'cloned/build-tools-internal/version.properties')
30+
versionPropertiesFile.text = """
31+
elasticsearch = 9.1.0
32+
lucene = 10.2.2
33+
34+
bundled_jdk_vendor = openjdk
35+
bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90
36+
minimumJdkVersion = 21
37+
minimumRuntimeJava = 21
38+
minimumCompilerJava = 21
39+
"""
2940
}
3041

3142
File setupGitRemote() {

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -51,23 +51,23 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
5151
assertOutputContains(result.output, "[$bwcDistVersion] > Task :distribution:archives:darwin-tar:${expectedAssembleTaskName}")
5252

5353
where:
54-
bwcDistVersion | bwcProject | expectedAssembleTaskName
55-
"8.4.0" | "minor" | "extractedAssemble"
56-
"8.3.0" | "staged" | "extractedAssemble"
57-
"8.2.1" | "bugfix" | "extractedAssemble"
58-
"8.1.3" | "bugfix2" | "extractedAssemble"
54+
bwcDistVersion | bwcProject | expectedAssembleTaskName
55+
"8.4.0" | "major1" | "extractedAssemble"
56+
"8.3.0" | "major2" | "extractedAssemble"
57+
"8.2.1" | "major3" | "extractedAssemble"
58+
"8.1.3" | "major4" | "extractedAssemble"
5959
}
6060

6161
@Unroll
6262
def "supports #platform aarch distributions"() {
6363
when:
64-
def result = gradleRunner(":distribution:bwc:minor:buildBwc${platform.capitalize()}Aarch64Tar",
64+
def result = gradleRunner(":distribution:bwc:major1:buildBwc${platform.capitalize()}Aarch64Tar",
6565
"-DtestRemoteRepo=" + remoteGitRepo,
6666
"-Dbwc.remote=origin",
6767
"-Dbwc.dist.version=${bwcDistVersion}-SNAPSHOT")
6868
.build()
6969
then:
70-
result.task(":distribution:bwc:minor:buildBwc${platform.capitalize()}Aarch64Tar").outcome == TaskOutcome.SUCCESS
70+
result.task(":distribution:bwc:major1:buildBwc${platform.capitalize()}Aarch64Tar").outcome == TaskOutcome.SUCCESS
7171

7272
and: "assemble tasks triggered"
7373
assertOutputContains(result.output, "[$bwcDistVersion] > Task :distribution:archives:${platform}-aarch64-tar:extractedAssemble")
@@ -87,7 +87,7 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
8787
}
8888
8989
dependencies {
90-
expandedDist project(path: ":distribution:bwc:minor", configuration:"expanded-darwin-tar")
90+
expandedDist project(path: ":distribution:bwc:major1", configuration:"expanded-darwin-tar")
9191
}
9292
9393
tasks.register("resolveExpandedDistribution") {
@@ -109,12 +109,12 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
109109
.build()
110110
then:
111111
result.task(":resolveExpandedDistribution").outcome == TaskOutcome.SUCCESS
112-
result.task(":distribution:bwc:minor:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS
112+
result.task(":distribution:bwc:major1:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS
113113
and: "assemble task triggered"
114114
result.output.contains("[8.4.0] > Task :distribution:archives:darwin-tar:extractedAssemble")
115-
result.output.contains("expandedRootPath /distribution/bwc/minor/build/bwc/checkout-8.x/" +
115+
result.output.contains("expandedRootPath /distribution/bwc/major1/build/bwc/checkout-8.x/" +
116116
"distribution/archives/darwin-tar/build/install")
117-
result.output.contains("nested folder /distribution/bwc/minor/build/bwc/checkout-8.x/" +
117+
result.output.contains("nested folder /distribution/bwc/major1/build/bwc/checkout-8.x/" +
118118
"distribution/archives/darwin-tar/build/install/elasticsearch-8.4.0-SNAPSHOT")
119119
}
120120
}

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
5151
def "resolves expanded bwc versions from source"() {
5252
given:
5353
internalBuild()
54-
bwcMinorProjectSetup()
54+
bwcMajor1ProjectSetup()
5555
buildFile << """
5656
apply plugin: 'elasticsearch.internal-distribution-download'
5757
@@ -72,16 +72,16 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
7272

7373
def result = gradleRunner("setupDistro").build()
7474
then:
75-
result.task(":distribution:bwc:minor:buildBwcExpandedTask").outcome == TaskOutcome.SUCCESS
75+
result.task(":distribution:bwc:major1:buildBwcExpandedTask").outcome == TaskOutcome.SUCCESS
7676
result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
77-
assertExtractedDistroIsCreated("distribution/bwc/minor/build/install/elastic-distro",
77+
assertExtractedDistroIsCreated("distribution/bwc/major1/build/install/elastic-distro",
7878
'bwc-marker.txt')
7979
}
8080

8181
def "fails on resolving bwc versions with no bundled jdk"() {
8282
given:
8383
internalBuild()
84-
bwcMinorProjectSetup()
84+
bwcMajor1ProjectSetup()
8585
buildFile << """
8686
apply plugin: 'elasticsearch.internal-distribution-download'
8787
@@ -105,12 +105,12 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
105105
"without a bundled JDK is not supported.")
106106
}
107107

108-
private void bwcMinorProjectSetup() {
108+
private void bwcMajor1ProjectSetup() {
109109
settingsFile << """
110-
include ':distribution:bwc:minor'
110+
include ':distribution:bwc:major1'
111111
"""
112-
def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "bwc", "minor")
113-
new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=minor"
112+
def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "bwc", "major1")
113+
new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=major1"
114114
new File(bwcSubProjectFolder, 'build.gradle') << """
115115
apply plugin:'base'
116116

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
4040
given:
4141
internalBuild()
4242

43-
subProject(":distribution:bwc:minor") << """
43+
subProject(":distribution:bwc:major1") << """
4444
configurations { checkout }
4545
artifacts {
4646
checkout(new File(projectDir, "checkoutDir"))
@@ -61,11 +61,11 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
6161
result.task(transformTask).outcome == TaskOutcome.NO_SOURCE
6262
}
6363

64-
def "yamlRestCompatTest executes and copies api and transforms tests from :bwc:minor"() {
64+
def "yamlRestCompatTest executes and copies api and transforms tests from :bwc:major1"() {
6565
given:
6666
internalBuild()
6767

68-
subProject(":distribution:bwc:minor") << """
68+
subProject(":distribution:bwc:major1") << """
6969
configurations { checkout }
7070
artifacts {
7171
checkout(new File(projectDir, "checkoutDir"))
@@ -98,8 +98,8 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
9898
String api = "foo.json"
9999
String test = "10_basic.yml"
100100
//add the compatible test and api files, these are the prior version's normal yaml rest tests
101-
file("distribution/bwc/minor/checkoutDir/rest-api-spec/src/main/resources/rest-api-spec/api/" + api) << ""
102-
file("distribution/bwc/minor/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/" + test) << ""
101+
file("distribution/bwc/major1/checkoutDir/rest-api-spec/src/main/resources/rest-api-spec/api/" + api) << ""
102+
file("distribution/bwc/major1/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/" + test) << ""
103103

104104
when:
105105
def result = gradleRunner("yamlRestCompatTest").build()
@@ -145,7 +145,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
145145
given:
146146
internalBuild()
147147
withVersionCatalogue()
148-
subProject(":distribution:bwc:minor") << """
148+
subProject(":distribution:bwc:major1") << """
149149
configurations { checkout }
150150
artifacts {
151151
checkout(new File(projectDir, "checkoutDir"))
@@ -186,7 +186,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
186186
given:
187187
internalBuild()
188188

189-
subProject(":distribution:bwc:minor") << """
189+
subProject(":distribution:bwc:major1") << """
190190
configurations { checkout }
191191
artifacts {
192192
checkout(new File(projectDir, "checkoutDir"))
@@ -230,7 +230,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe
230230

231231
setupRestResources([], [])
232232

233-
file("distribution/bwc/minor/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/test.yml" ) << """
233+
file("distribution/bwc/major1/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/test.yml" ) << """
234234
"one":
235235
- do:
236236
do_.some.key_to_replace:

0 commit comments

Comments
 (0)