Skip to content

Commit 8fecee3

Browse files
author
elasticsearchmachine
committed
Merge remote-tracking branch 'origin/main' into lucene_snapshot
2 parents 349f4c0 + 2c85c37 commit 8fecee3

File tree

149 files changed

+2963
-847
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

149 files changed

+2963
-847
lines changed

‎.buildkite/pull-requests.json‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,19 @@
1515
"trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*((buildkite|@elastic(search)?machine)\\s*)?test\\s+this(\\s+please)?)",
1616
"cancel_intermediate_builds": true,
1717
"cancel_intermediate_builds_on_comment": false
18+
},
19+
{
20+
"enabled": true,
21+
"pipeline_slug": "elasticsearch-performance-esbench-pr",
22+
"allow_org_users": true,
23+
"allowed_repo_permissions": [
24+
"admin",
25+
"write"
26+
],
27+
"set_commit_status": false,
28+
"build_on_commit": false,
29+
"build_on_comment": true,
30+
"trigger_comment_regex": "^(buildkite|@elastic(search)?machine) benchmark this with (?<benchmark>\\w+)( please)?$"
1831
}
1932
]
2033
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/PackAsBinaryBenchmark.java‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
package org.elasticsearch.benchmark.vector;
10+
11+
import org.elasticsearch.common.logging.LogConfigurator;
12+
import org.elasticsearch.index.codec.vectors.BQVectorUtils;
13+
import org.openjdk.jmh.annotations.Benchmark;
14+
import org.openjdk.jmh.annotations.BenchmarkMode;
15+
import org.openjdk.jmh.annotations.Fork;
16+
import org.openjdk.jmh.annotations.Measurement;
17+
import org.openjdk.jmh.annotations.Mode;
18+
import org.openjdk.jmh.annotations.OutputTimeUnit;
19+
import org.openjdk.jmh.annotations.Param;
20+
import org.openjdk.jmh.annotations.Scope;
21+
import org.openjdk.jmh.annotations.Setup;
22+
import org.openjdk.jmh.annotations.State;
23+
import org.openjdk.jmh.annotations.Warmup;
24+
import org.openjdk.jmh.infra.Blackhole;
25+
26+
import java.io.IOException;
27+
import java.util.Random;
28+
import java.util.concurrent.TimeUnit;
29+
30+
@BenchmarkMode(Mode.Throughput)
31+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
32+
@State(Scope.Benchmark)
33+
// first iteration is complete garbage, so make sure we really warmup
34+
@Warmup(iterations = 4, time = 1)
35+
// real iterations. not useful to spend tons of time here, better to fork more
36+
@Measurement(iterations = 5, time = 1)
37+
// engage some noise reduction
38+
@Fork(value = 1)
39+
public class PackAsBinaryBenchmark {
40+
41+
static {
42+
LogConfigurator.configureESLogging(); // native access requires logging to be initialized
43+
}
44+
45+
@Param({ "384", "782", "1024" })
46+
int dims;
47+
48+
int length;
49+
50+
int numVectors = 1000;
51+
52+
int[][] qVectors;
53+
byte[] packed;
54+
55+
@Setup
56+
public void setup() throws IOException {
57+
Random random = new Random(123);
58+
59+
this.length = BQVectorUtils.discretize(dims, 64) / 8;
60+
this.packed = new byte[length];
61+
62+
qVectors = new int[numVectors][dims];
63+
for (int[] qVector : qVectors) {
64+
for (int i = 0; i < dims; i++) {
65+
qVector[i] = random.nextInt(2);
66+
}
67+
}
68+
}
69+
70+
@Benchmark
71+
public void packAsBinary(Blackhole bh) {
72+
for (int i = 0; i < numVectors; i++) {
73+
BQVectorUtils.packAsBinary(qVectors[i], packed);
74+
bh.consume(packed);
75+
}
76+
}
77+
78+
@Benchmark
79+
public void packAsBinaryLegacy(Blackhole bh) {
80+
for (int i = 0; i < numVectors; i++) {
81+
BQVectorUtils.packAsBinaryLegacy(qVectors[i], packed);
82+
bh.consume(packed);
83+
}
84+
}
85+
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/TransposeHalfByteBenchmark.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
package org.elasticsearch.benchmark.vector;
10+
11+
import org.elasticsearch.common.logging.LogConfigurator;
12+
import org.elasticsearch.index.codec.vectors.BQSpaceUtils;
13+
import org.elasticsearch.index.codec.vectors.BQVectorUtils;
14+
import org.openjdk.jmh.annotations.Benchmark;
15+
import org.openjdk.jmh.annotations.BenchmarkMode;
16+
import org.openjdk.jmh.annotations.Fork;
17+
import org.openjdk.jmh.annotations.Measurement;
18+
import org.openjdk.jmh.annotations.Mode;
19+
import org.openjdk.jmh.annotations.OutputTimeUnit;
20+
import org.openjdk.jmh.annotations.Param;
21+
import org.openjdk.jmh.annotations.Scope;
22+
import org.openjdk.jmh.annotations.Setup;
23+
import org.openjdk.jmh.annotations.State;
24+
import org.openjdk.jmh.annotations.Warmup;
25+
import org.openjdk.jmh.infra.Blackhole;
26+
27+
import java.io.IOException;
28+
import java.util.Random;
29+
import java.util.concurrent.TimeUnit;
30+
31+
@BenchmarkMode(Mode.Throughput)
32+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
33+
@State(Scope.Benchmark)
34+
// first iteration is complete garbage, so make sure we really warmup
35+
@Warmup(iterations = 4, time = 1)
36+
// real iterations. not useful to spend tons of time here, better to fork more
37+
@Measurement(iterations = 5, time = 1)
38+
// engage some noise reduction
39+
@Fork(value = 1)
40+
public class TransposeHalfByteBenchmark {
41+
42+
static {
43+
LogConfigurator.configureESLogging(); // native access requires logging to be initialized
44+
}
45+
46+
@Param({ "384", "782", "1024" })
47+
int dims;
48+
49+
int length;
50+
51+
int numVectors = 1000;
52+
53+
int[][] qVectors;
54+
byte[] packed;
55+
56+
@Setup
57+
public void setup() throws IOException {
58+
Random random = new Random(123);
59+
60+
this.length = 4 * BQVectorUtils.discretize(dims, 64) / 8;
61+
this.packed = new byte[length];
62+
63+
qVectors = new int[numVectors][dims];
64+
for (int[] qVector : qVectors) {
65+
for (int i = 0; i < dims; i++) {
66+
qVector[i] = random.nextInt(16);
67+
}
68+
}
69+
}
70+
71+
@Benchmark
72+
public void transposeHalfByte(Blackhole bh) {
73+
for (int i = 0; i < numVectors; i++) {
74+
BQSpaceUtils.transposeHalfByte(qVectors[i], packed);
75+
bh.consume(packed);
76+
}
77+
}
78+
79+
@Benchmark
80+
public void transposeHalfByteLegacy(Blackhole bh) {
81+
for (int i = 0; i < numVectors; i++) {
82+
BQSpaceUtils.transposeHalfByteLegacy(qVectors[i], packed);
83+
bh.consume(packed);
84+
}
85+
}
86+
}

‎build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
import groovy.util.Node;
1313
import nmcp.NmcpPlugin;
1414

15-
import com.github.jengelman.gradle.plugins.shadow.ShadowExtension;
1615
import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin;
1716

1817
import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
@@ -173,8 +172,7 @@ private void addNameAndDescriptionToPom(Project project, NamedDomainObjectSet<Ma
173172
}
174173

175174
private static void configureWithShadowPlugin(Project project, MavenPublication publication) {
176-
var shadow = project.getExtensions().getByType(ShadowExtension.class);
177-
shadow.component(publication);
175+
publication.from(project.getComponents().getByName("shadow"));
178176
publication.artifact(project.getTasks().named("javadocJar"));
179177
publication.artifact(project.getTasks().named("sourcesJar"));
180178
}

‎build-tools-internal/build.gradle‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -220,13 +220,13 @@ gradlePlugin {
220220
id = 'elasticsearch.internal-yaml-rest-test'
221221
implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin'
222222
}
223-
transportVersionManagementPlugin {
224-
id = 'elasticsearch.transport-version-management'
225-
implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin'
223+
transportVersionReferencesPlugin {
224+
id = 'elasticsearch.transport-version-references'
225+
implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionReferencesPlugin'
226226
}
227-
globalTransportVersionManagementPlugin {
228-
id = 'elasticsearch.global-transport-version-management'
229-
implementationClass = 'org.elasticsearch.gradle.internal.transport.GlobalTransportVersionManagementPlugin'
227+
transportVersionResourcesPlugin {
228+
id = 'elasticsearch.transport-version-resources'
229+
implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionResourcesPlugin'
230230
}
231231
}
232232
}

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPluginFuncTest.groovy‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,8 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
9292

9393
file("myserver/build.gradle") << """
9494
apply plugin: 'java-library'
95-
apply plugin: 'elasticsearch.transport-version-management'
96-
apply plugin: 'elasticsearch.global-transport-version-management'
95+
apply plugin: 'elasticsearch.transport-version-references'
96+
apply plugin: 'elasticsearch.transport-version-resources'
9797
"""
9898
definedTransportVersion("existing_91", "8012000")
9999
definedTransportVersion("existing_92", "8123000,8012001")
@@ -112,7 +112,7 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
112112

113113
file("myplugin/build.gradle") << """
114114
apply plugin: 'java-library'
115-
apply plugin: 'elasticsearch.transport-version-management'
115+
apply plugin: 'elasticsearch.transport-version-references'
116116
117117
dependencies {
118118
implementation project(":myserver")

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
import org.elasticsearch.gradle.internal.info.BuildParameterExtension;
1616
import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin;
1717
import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin;
18-
import org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin;
18+
import org.elasticsearch.gradle.internal.transport.TransportVersionReferencesPlugin;
1919
import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
2020
import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
2121
import org.elasticsearch.gradle.util.GradleUtils;
@@ -37,7 +37,7 @@ public void apply(Project project) {
3737
project.getPluginManager().apply(JarHellPrecommitPlugin.class);
3838
project.getPluginManager().apply(ElasticsearchJavaPlugin.class);
3939
project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class);
40-
project.getPluginManager().apply(TransportVersionManagementPlugin.class);
40+
project.getPluginManager().apply(TransportVersionReferencesPlugin.class);
4141
boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).getCi();
4242
// Clear default dependencies added by public PluginBuildPlugin as we add our
4343
// own project dependencies for internal builds

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/shadow/XmlClassRelocationTransformer.java‎

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,9 @@
99

1010
package org.elasticsearch.gradle.internal.shadow;
1111

12-
import com.github.jengelman.gradle.plugins.shadow.ShadowStats;
1312
import com.github.jengelman.gradle.plugins.shadow.relocation.RelocateClassContext;
1413
import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator;
15-
import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer;
14+
import com.github.jengelman.gradle.plugins.shadow.transformers.ResourceTransformer;
1615
import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext;
1716

1817
import org.apache.commons.io.IOUtils;
@@ -26,7 +25,7 @@
2625
import java.io.BufferedInputStream;
2726
import java.io.ByteArrayOutputStream;
2827
import java.io.IOException;
29-
import java.util.List;
28+
import java.util.Set;
3029

3130
import javax.xml.parsers.DocumentBuilder;
3231
import javax.xml.parsers.DocumentBuilderFactory;
@@ -35,7 +34,7 @@
3534
import javax.xml.transform.dom.DOMSource;
3635
import javax.xml.transform.stream.StreamResult;
3736

38-
public class XmlClassRelocationTransformer implements Transformer {
37+
public class XmlClassRelocationTransformer implements ResourceTransformer {
3938

4039
boolean hasTransformedResource = false;
4140

@@ -55,7 +54,7 @@ public boolean canTransformResource(FileTreeElement element) {
5554
@Override
5655
public void transform(TransformerContext context) {
5756
try {
58-
BufferedInputStream bis = new BufferedInputStream(context.getIs());
57+
BufferedInputStream bis = new BufferedInputStream(context.getInputStream());
5958
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
6059
DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
6160
doc = dBuilder.parse(bis);
@@ -66,17 +65,16 @@ public void transform(TransformerContext context) {
6665
this.doc = null;
6766
}
6867
} catch (Exception e) {
69-
throw new RuntimeException("Error parsing xml file in " + context.getIs(), e);
68+
throw new RuntimeException("Error parsing xml file in " + context.getInputStream(), e);
7069
}
7170
}
7271

7372
private static String getRelocatedClass(String className, TransformerContext context) {
74-
List<Relocator> relocators = context.getRelocators();
75-
ShadowStats stats = context.getStats();
73+
Set<Relocator> relocators = context.getRelocators();
7674
if (className != null && className.length() > 0 && relocators != null) {
7775
for (Relocator relocator : relocators) {
7876
if (relocator.canRelocateClass(className)) {
79-
RelocateClassContext relocateClassContext = new RelocateClassContext(className, stats);
77+
RelocateClassContext relocateClassContext = new RelocateClassContext(className);
8078
return relocator.relocateClass(relocateClassContext);
8179
}
8280
}
@@ -111,8 +109,6 @@ public boolean hasTransformedResource() {
111109
@Override
112110
public void modifyOutputStream(ZipOutputStream os, boolean preserveFileTimestamps) {
113111
ZipEntry entry = new ZipEntry(resource);
114-
entry.setTime(TransformerContext.getEntryTimestamp(preserveFileTimestamps, entry.getTime()));
115-
116112
try {
117113
// Write the content back to the XML file
118114
TransformerFactory transformerFactory = TransformerFactory.newInstance();

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ public abstract class CollectTransportVersionReferencesTask extends DefaultTask
6161

6262
@TaskAction
6363
public void checkTransportVersion() throws IOException {
64-
var results = new HashSet<TransportVersionUtils.TransportVersionReference>();
64+
var results = new HashSet<TransportVersionReference>();
6565

6666
for (var cpElement : getClassPath()) {
6767
Path file = cpElement.toPath();
@@ -74,8 +74,7 @@ public void checkTransportVersion() throws IOException {
7474
Files.writeString(outputFile, String.join("\n", results.stream().map(Object::toString).sorted().toList()));
7575
}
7676

77-
private void addNamesFromClassesDirectory(Set<TransportVersionUtils.TransportVersionReference> results, Path basePath)
78-
throws IOException {
77+
private void addNamesFromClassesDirectory(Set<TransportVersionReference> results, Path basePath) throws IOException {
7978
Files.walkFileTree(basePath, new SimpleFileVisitor<>() {
8079
@Override
8180
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
@@ -90,8 +89,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
9089
});
9190
}
9291

93-
private void addNamesFromClass(Set<TransportVersionUtils.TransportVersionReference> results, InputStream classBytes, String classname)
94-
throws IOException {
92+
private void addNamesFromClass(Set<TransportVersionReference> results, InputStream classBytes, String classname) throws IOException {
9593
ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) {
9694
@Override
9795
public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
@@ -111,7 +109,7 @@ public void visitMethodInsn(int opcode, String owner, String name, String descri
111109
if (abstractInstruction instanceof LdcInsnNode ldcInsnNode
112110
&& ldcInsnNode.cst instanceof String tvName
113111
&& tvName.isEmpty() == false) {
114-
results.add(new TransportVersionUtils.TransportVersionReference(tvName, location));
112+
results.add(new TransportVersionReference(tvName, location));
115113
} else {
116114
// The instruction is not a LDC with a String constant (or an empty String), which is not allowed.
117115
throw new RuntimeException(

0 commit comments

Comments
 (0)