Skip to content

Commit ee803a4

Browse files
carlosdelestcarlosdelest
authored
Merge branch 'main' into non-issue/esql-dense-vector-byte-element-support
2 parents 64ca563 + f23029f commit ee803a4

File tree

147 files changed

+3083
-995
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

147 files changed

+3083
-995
lines changed

‎.buildkite/pull-requests.json‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,19 @@
1515
"trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*((buildkite|@elastic(search)?machine)\\s*)?test\\s+this(\\s+please)?)",
1616
"cancel_intermediate_builds": true,
1717
"cancel_intermediate_builds_on_comment": false
18+
},
19+
{
20+
"enabled": true,
21+
"pipeline_slug": "elasticsearch-performance-esbench-pr",
22+
"allow_org_users": true,
23+
"allowed_repo_permissions": [
24+
"admin",
25+
"write"
26+
],
27+
"set_commit_status": false,
28+
"build_on_commit": false,
29+
"build_on_comment": true,
30+
"trigger_comment_regex": "^(buildkite|@elastic(search)?machine) benchmark this with (?<benchmark>\\w+)( please)?$"
1831
}
1932
]
2033
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/PackAsBinaryBenchmark.java‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
package org.elasticsearch.benchmark.vector;
10+
11+
import org.elasticsearch.common.logging.LogConfigurator;
12+
import org.elasticsearch.index.codec.vectors.BQVectorUtils;
13+
import org.openjdk.jmh.annotations.Benchmark;
14+
import org.openjdk.jmh.annotations.BenchmarkMode;
15+
import org.openjdk.jmh.annotations.Fork;
16+
import org.openjdk.jmh.annotations.Measurement;
17+
import org.openjdk.jmh.annotations.Mode;
18+
import org.openjdk.jmh.annotations.OutputTimeUnit;
19+
import org.openjdk.jmh.annotations.Param;
20+
import org.openjdk.jmh.annotations.Scope;
21+
import org.openjdk.jmh.annotations.Setup;
22+
import org.openjdk.jmh.annotations.State;
23+
import org.openjdk.jmh.annotations.Warmup;
24+
import org.openjdk.jmh.infra.Blackhole;
25+
26+
import java.io.IOException;
27+
import java.util.Random;
28+
import java.util.concurrent.TimeUnit;
29+
30+
@BenchmarkMode(Mode.Throughput)
31+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
32+
@State(Scope.Benchmark)
33+
// first iteration is complete garbage, so make sure we really warmup
34+
@Warmup(iterations = 4, time = 1)
35+
// real iterations. not useful to spend tons of time here, better to fork more
36+
@Measurement(iterations = 5, time = 1)
37+
// engage some noise reduction
38+
@Fork(value = 1)
39+
public class PackAsBinaryBenchmark {
40+
41+
static {
42+
LogConfigurator.configureESLogging(); // native access requires logging to be initialized
43+
}
44+
45+
@Param({ "384", "782", "1024" })
46+
int dims;
47+
48+
int length;
49+
50+
int numVectors = 1000;
51+
52+
int[][] qVectors;
53+
byte[] packed;
54+
55+
@Setup
56+
public void setup() throws IOException {
57+
Random random = new Random(123);
58+
59+
this.length = BQVectorUtils.discretize(dims, 64) / 8;
60+
this.packed = new byte[length];
61+
62+
qVectors = new int[numVectors][dims];
63+
for (int[] qVector : qVectors) {
64+
for (int i = 0; i < dims; i++) {
65+
qVector[i] = random.nextInt(2);
66+
}
67+
}
68+
}
69+
70+
@Benchmark
71+
public void packAsBinary(Blackhole bh) {
72+
for (int i = 0; i < numVectors; i++) {
73+
BQVectorUtils.packAsBinary(qVectors[i], packed);
74+
bh.consume(packed);
75+
}
76+
}
77+
78+
@Benchmark
79+
public void packAsBinaryLegacy(Blackhole bh) {
80+
for (int i = 0; i < numVectors; i++) {
81+
BQVectorUtils.packAsBinaryLegacy(qVectors[i], packed);
82+
bh.consume(packed);
83+
}
84+
}
85+
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/TransposeHalfByteBenchmark.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
package org.elasticsearch.benchmark.vector;
10+
11+
import org.elasticsearch.common.logging.LogConfigurator;
12+
import org.elasticsearch.index.codec.vectors.BQSpaceUtils;
13+
import org.elasticsearch.index.codec.vectors.BQVectorUtils;
14+
import org.openjdk.jmh.annotations.Benchmark;
15+
import org.openjdk.jmh.annotations.BenchmarkMode;
16+
import org.openjdk.jmh.annotations.Fork;
17+
import org.openjdk.jmh.annotations.Measurement;
18+
import org.openjdk.jmh.annotations.Mode;
19+
import org.openjdk.jmh.annotations.OutputTimeUnit;
20+
import org.openjdk.jmh.annotations.Param;
21+
import org.openjdk.jmh.annotations.Scope;
22+
import org.openjdk.jmh.annotations.Setup;
23+
import org.openjdk.jmh.annotations.State;
24+
import org.openjdk.jmh.annotations.Warmup;
25+
import org.openjdk.jmh.infra.Blackhole;
26+
27+
import java.io.IOException;
28+
import java.util.Random;
29+
import java.util.concurrent.TimeUnit;
30+
31+
@BenchmarkMode(Mode.Throughput)
32+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
33+
@State(Scope.Benchmark)
34+
// first iteration is complete garbage, so make sure we really warmup
35+
@Warmup(iterations = 4, time = 1)
36+
// real iterations. not useful to spend tons of time here, better to fork more
37+
@Measurement(iterations = 5, time = 1)
38+
// engage some noise reduction
39+
@Fork(value = 1)
40+
public class TransposeHalfByteBenchmark {
41+
42+
static {
43+
LogConfigurator.configureESLogging(); // native access requires logging to be initialized
44+
}
45+
46+
@Param({ "384", "782", "1024" })
47+
int dims;
48+
49+
int length;
50+
51+
int numVectors = 1000;
52+
53+
int[][] qVectors;
54+
byte[] packed;
55+
56+
@Setup
57+
public void setup() throws IOException {
58+
Random random = new Random(123);
59+
60+
this.length = 4 * BQVectorUtils.discretize(dims, 64) / 8;
61+
this.packed = new byte[length];
62+
63+
qVectors = new int[numVectors][dims];
64+
for (int[] qVector : qVectors) {
65+
for (int i = 0; i < dims; i++) {
66+
qVector[i] = random.nextInt(16);
67+
}
68+
}
69+
}
70+
71+
@Benchmark
72+
public void transposeHalfByte(Blackhole bh) {
73+
for (int i = 0; i < numVectors; i++) {
74+
BQSpaceUtils.transposeHalfByte(qVectors[i], packed);
75+
bh.consume(packed);
76+
}
77+
}
78+
79+
@Benchmark
80+
public void transposeHalfByteLegacy(Blackhole bh) {
81+
for (int i = 0; i < numVectors; i++) {
82+
BQSpaceUtils.transposeHalfByteLegacy(qVectors[i], packed);
83+
bh.consume(packed);
84+
}
85+
}
86+
}

‎build-tools-internal/build.gradle‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -220,13 +220,13 @@ gradlePlugin {
220220
id = 'elasticsearch.internal-yaml-rest-test'
221221
implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin'
222222
}
223-
transportVersionManagementPlugin {
224-
id = 'elasticsearch.transport-version-management'
225-
implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin'
223+
transportVersionReferencesPlugin {
224+
id = 'elasticsearch.transport-version-references'
225+
implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionReferencesPlugin'
226226
}
227-
globalTransportVersionManagementPlugin {
228-
id = 'elasticsearch.global-transport-version-management'
229-
implementationClass = 'org.elasticsearch.gradle.internal.transport.GlobalTransportVersionManagementPlugin'
227+
transportVersionResourcesPlugin {
228+
id = 'elasticsearch.transport-version-resources'
229+
implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionResourcesPlugin'
230230
}
231231
}
232232
}

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPluginFuncTest.groovy‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,8 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
9292

9393
file("myserver/build.gradle") << """
9494
apply plugin: 'java-library'
95-
apply plugin: 'elasticsearch.transport-version-management'
96-
apply plugin: 'elasticsearch.global-transport-version-management'
95+
apply plugin: 'elasticsearch.transport-version-references'
96+
apply plugin: 'elasticsearch.transport-version-resources'
9797
"""
9898
definedTransportVersion("existing_91", "8012000")
9999
definedTransportVersion("existing_92", "8123000,8012001")
@@ -112,7 +112,7 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
112112

113113
file("myplugin/build.gradle") << """
114114
apply plugin: 'java-library'
115-
apply plugin: 'elasticsearch.transport-version-management'
115+
apply plugin: 'elasticsearch.transport-version-references'
116116
117117
dependencies {
118118
implementation project(":myserver")

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
import org.elasticsearch.gradle.internal.info.BuildParameterExtension;
1616
import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin;
1717
import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin;
18-
import org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin;
18+
import org.elasticsearch.gradle.internal.transport.TransportVersionReferencesPlugin;
1919
import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
2020
import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
2121
import org.elasticsearch.gradle.util.GradleUtils;
@@ -37,7 +37,7 @@ public void apply(Project project) {
3737
project.getPluginManager().apply(JarHellPrecommitPlugin.class);
3838
project.getPluginManager().apply(ElasticsearchJavaPlugin.class);
3939
project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class);
40-
project.getPluginManager().apply(TransportVersionManagementPlugin.class);
40+
project.getPluginManager().apply(TransportVersionReferencesPlugin.class);
4141
boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).getCi();
4242
// Clear default dependencies added by public PluginBuildPlugin as we add our
4343
// own project dependencies for internal builds

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ public abstract class CollectTransportVersionReferencesTask extends DefaultTask
6161

6262
@TaskAction
6363
public void checkTransportVersion() throws IOException {
64-
var results = new HashSet<TransportVersionUtils.TransportVersionReference>();
64+
var results = new HashSet<TransportVersionReference>();
6565

6666
for (var cpElement : getClassPath()) {
6767
Path file = cpElement.toPath();
@@ -74,8 +74,7 @@ public void checkTransportVersion() throws IOException {
7474
Files.writeString(outputFile, String.join("\n", results.stream().map(Object::toString).sorted().toList()));
7575
}
7676

77-
private void addNamesFromClassesDirectory(Set<TransportVersionUtils.TransportVersionReference> results, Path basePath)
78-
throws IOException {
77+
private void addNamesFromClassesDirectory(Set<TransportVersionReference> results, Path basePath) throws IOException {
7978
Files.walkFileTree(basePath, new SimpleFileVisitor<>() {
8079
@Override
8180
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
@@ -90,8 +89,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
9089
});
9190
}
9291

93-
private void addNamesFromClass(Set<TransportVersionUtils.TransportVersionReference> results, InputStream classBytes, String classname)
94-
throws IOException {
92+
private void addNamesFromClass(Set<TransportVersionReference> results, InputStream classBytes, String classname) throws IOException {
9593
ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) {
9694
@Override
9795
public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
@@ -111,7 +109,7 @@ public void visitMethodInsn(int opcode, String owner, String name, String descri
111109
if (abstractInstruction instanceof LdcInsnNode ldcInsnNode
112110
&& ldcInsnNode.cst instanceof String tvName
113111
&& tvName.isEmpty() == false) {
114-
results.add(new TransportVersionUtils.TransportVersionReference(tvName, location));
112+
results.add(new TransportVersionReference(tvName, location));
115113
} else {
116114
// The instruction is not a LDC with a String constant (or an empty String), which is not allowed.
117115
throw new RuntimeException(

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionDefinition.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.gradle.internal.transport;
11+
12+
import java.util.ArrayList;
13+
import java.util.List;
14+
15+
record TransportVersionDefinition(String name, List<TransportVersionId> ids) {
16+
public static TransportVersionDefinition fromString(String filename, String contents) {
17+
assert filename.endsWith(".csv");
18+
String name = filename.substring(0, filename.length() - 4);
19+
List<TransportVersionId> ids = new ArrayList<>();
20+
21+
if (contents.isEmpty() == false) {
22+
for (String rawId : contents.split(",")) {
23+
try {
24+
ids.add(TransportVersionId.fromString(rawId));
25+
} catch (NumberFormatException e) {
26+
throw new IllegalStateException("Failed to parse id " + rawId + " in " + filename, e);
27+
}
28+
}
29+
}
30+
31+
return new TransportVersionDefinition(name, ids);
32+
}
33+
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionId.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.gradle.internal.transport;
11+
12+
record TransportVersionId(int complete, int major, int server, int subsidiary, int patch) implements Comparable<TransportVersionId> {
13+
14+
static TransportVersionId fromString(String s) {
15+
int complete = Integer.parseInt(s);
16+
int patch = complete % 100;
17+
int subsidiary = (complete / 100) % 10;
18+
int server = (complete / 1000) % 1000;
19+
int major = complete / 1000000;
20+
return new TransportVersionId(complete, major, server, subsidiary, patch);
21+
}
22+
23+
@Override
24+
public int compareTo(TransportVersionId o) {
25+
return Integer.compare(complete, o.complete);
26+
}
27+
28+
@Override
29+
public String toString() {
30+
return Integer.toString(complete);
31+
}
32+
33+
public int base() {
34+
return (complete / 1000) * 1000;
35+
}
36+
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionLatest.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.gradle.internal.transport;
11+
12+
record TransportVersionLatest(String branch, String name, TransportVersionId id) {
13+
public static TransportVersionLatest fromString(String filename, String contents) {
14+
assert filename.endsWith(".csv");
15+
String branch = filename.substring(0, filename.length() - 4);
16+
17+
String[] parts = contents.split(",");
18+
if (parts.length != 2) {
19+
throw new IllegalStateException("Invalid transport version latest file [" + filename + "]: " + contents);
20+
}
21+
22+
return new TransportVersionLatest(branch, parts[0], TransportVersionId.fromString(parts[1]));
23+
}
24+
}

0 commit comments

Comments
 (0)