Skip to content

Commit 5112eb4

Browse files
fang-xing-esqlfang-xing-esql
authored
Merge branch 'main' into pushdown-roundto-merge-range-query
2 parents 77ba749 + c9b45d8 commit 5112eb4

File tree

371 files changed

+10040
-2595
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

371 files changed

+10040
-2595
lines changed

‎.buildkite/pull-requests.json‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,19 @@
1515
"trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*((buildkite|@elastic(search)?machine)\\s*)?test\\s+this(\\s+please)?)",
1616
"cancel_intermediate_builds": true,
1717
"cancel_intermediate_builds_on_comment": false
18+
},
19+
{
20+
"enabled": true,
21+
"pipeline_slug": "elasticsearch-performance-esbench-pr",
22+
"allow_org_users": true,
23+
"allowed_repo_permissions": [
24+
"admin",
25+
"write"
26+
],
27+
"set_commit_status": false,
28+
"build_on_commit": false,
29+
"build_on_comment": true,
30+
"trigger_comment_regex": "^(buildkite|@elastic(search)?machine) benchmark this with (?<benchmark>\\w+)( please)?$"
1831
}
1932
]
2033
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramGenerationBench.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
package org.elasticsearch.benchmark.exponentialhistogram;
1111

12+
import org.elasticsearch.exponentialhistogram.ExponentialHistogramCircuitBreaker;
1213
import org.elasticsearch.exponentialhistogram.ExponentialHistogramGenerator;
1314
import org.openjdk.jmh.annotations.Benchmark;
1415
import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -59,7 +60,7 @@ public class ExponentialHistogramGenerationBench {
5960
@Setup
6061
public void setUp() {
6162
random = ThreadLocalRandom.current();
62-
histoGenerator = new ExponentialHistogramGenerator(bucketCount);
63+
histoGenerator = ExponentialHistogramGenerator.create(bucketCount, ExponentialHistogramCircuitBreaker.noop());
6364

6465
DoubleSupplier nextRandom = () -> distribution.equals("GAUSSIAN") ? random.nextGaussian() : random.nextDouble();
6566

‎benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
import org.elasticsearch.exponentialhistogram.BucketIterator;
1313
import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
14+
import org.elasticsearch.exponentialhistogram.ExponentialHistogramCircuitBreaker;
1415
import org.elasticsearch.exponentialhistogram.ExponentialHistogramGenerator;
1516
import org.elasticsearch.exponentialhistogram.ExponentialHistogramMerger;
1617
import org.openjdk.jmh.annotations.Benchmark;
@@ -56,13 +57,14 @@ public class ExponentialHistogramMergeBench {
5657
@Setup
5758
public void setUp() {
5859
random = ThreadLocalRandom.current();
59-
histoMerger = new ExponentialHistogramMerger(bucketCount);
60+
ExponentialHistogramCircuitBreaker breaker = ExponentialHistogramCircuitBreaker.noop();
61+
histoMerger = ExponentialHistogramMerger.create(bucketCount, breaker);
6062

61-
ExponentialHistogramGenerator initial = new ExponentialHistogramGenerator(bucketCount);
63+
ExponentialHistogramGenerator initialGenerator = ExponentialHistogramGenerator.create(bucketCount, breaker);
6264
for (int j = 0; j < bucketCount; j++) {
63-
initial.add(Math.pow(1.001, j));
65+
initialGenerator.add(Math.pow(1.001, j));
6466
}
65-
ExponentialHistogram initialHisto = initial.get();
67+
ExponentialHistogram initialHisto = initialGenerator.getAndClear();
6668
int cnt = getBucketCount(initialHisto);
6769
if (cnt < bucketCount) {
6870
throw new IllegalArgumentException("Expected bucket count to be " + bucketCount + ", but was " + cnt);
@@ -72,14 +74,14 @@ public void setUp() {
7274
int dataPointSize = (int) Math.round(bucketCount * mergedHistoSizeFactor);
7375

7476
for (int i = 0; i < toMerge.length; i++) {
75-
ExponentialHistogramGenerator generator = new ExponentialHistogramGenerator(dataPointSize);
77+
ExponentialHistogramGenerator generator = ExponentialHistogramGenerator.create(dataPointSize, breaker);
7678

7779
int bucketIndex = 0;
7880
for (int j = 0; j < dataPointSize; j++) {
7981
bucketIndex += 1 + random.nextInt(bucketCount) % (Math.max(1, bucketCount / dataPointSize));
8082
generator.add(Math.pow(1.001, bucketIndex));
8183
}
82-
toMerge[i] = generator.get();
84+
toMerge[i] = generator.getAndClear();
8385
cnt = getBucketCount(toMerge[i]);
8486
if (cnt < dataPointSize) {
8587
throw new IllegalArgumentException("Expected bucket count to be " + dataPointSize + ", but was " + cnt);

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/PackAsBinaryBenchmark.java‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
package org.elasticsearch.benchmark.vector;
10+
11+
import org.elasticsearch.common.logging.LogConfigurator;
12+
import org.elasticsearch.index.codec.vectors.BQVectorUtils;
13+
import org.openjdk.jmh.annotations.Benchmark;
14+
import org.openjdk.jmh.annotations.BenchmarkMode;
15+
import org.openjdk.jmh.annotations.Fork;
16+
import org.openjdk.jmh.annotations.Measurement;
17+
import org.openjdk.jmh.annotations.Mode;
18+
import org.openjdk.jmh.annotations.OutputTimeUnit;
19+
import org.openjdk.jmh.annotations.Param;
20+
import org.openjdk.jmh.annotations.Scope;
21+
import org.openjdk.jmh.annotations.Setup;
22+
import org.openjdk.jmh.annotations.State;
23+
import org.openjdk.jmh.annotations.Warmup;
24+
import org.openjdk.jmh.infra.Blackhole;
25+
26+
import java.io.IOException;
27+
import java.util.Random;
28+
import java.util.concurrent.TimeUnit;
29+
30+
@BenchmarkMode(Mode.Throughput)
31+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
32+
@State(Scope.Benchmark)
33+
// first iteration is complete garbage, so make sure we really warmup
34+
@Warmup(iterations = 4, time = 1)
35+
// real iterations. not useful to spend tons of time here, better to fork more
36+
@Measurement(iterations = 5, time = 1)
37+
// engage some noise reduction
38+
@Fork(value = 1)
39+
public class PackAsBinaryBenchmark {
40+
41+
static {
42+
LogConfigurator.configureESLogging(); // native access requires logging to be initialized
43+
}
44+
45+
@Param({ "384", "782", "1024" })
46+
int dims;
47+
48+
int length;
49+
50+
int numVectors = 1000;
51+
52+
int[][] qVectors;
53+
byte[] packed;
54+
55+
@Setup
56+
public void setup() throws IOException {
57+
Random random = new Random(123);
58+
59+
this.length = BQVectorUtils.discretize(dims, 64) / 8;
60+
this.packed = new byte[length];
61+
62+
qVectors = new int[numVectors][dims];
63+
for (int[] qVector : qVectors) {
64+
for (int i = 0; i < dims; i++) {
65+
qVector[i] = random.nextInt(2);
66+
}
67+
}
68+
}
69+
70+
@Benchmark
71+
public void packAsBinary(Blackhole bh) {
72+
for (int i = 0; i < numVectors; i++) {
73+
BQVectorUtils.packAsBinary(qVectors[i], packed);
74+
bh.consume(packed);
75+
}
76+
}
77+
78+
@Benchmark
79+
public void packAsBinaryLegacy(Blackhole bh) {
80+
for (int i = 0; i < numVectors; i++) {
81+
BQVectorUtils.packAsBinaryLegacy(qVectors[i], packed);
82+
bh.consume(packed);
83+
}
84+
}
85+
86+
@Benchmark
87+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
88+
public void packAsBinaryPanama(Blackhole bh) {
89+
for (int i = 0; i < numVectors; i++) {
90+
BQVectorUtils.packAsBinary(qVectors[i], packed);
91+
bh.consume(packed);
92+
}
93+
}
94+
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/TransposeHalfByteBenchmark.java‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
package org.elasticsearch.benchmark.vector;
10+
11+
import org.elasticsearch.common.logging.LogConfigurator;
12+
import org.elasticsearch.index.codec.vectors.BQSpaceUtils;
13+
import org.elasticsearch.index.codec.vectors.BQVectorUtils;
14+
import org.openjdk.jmh.annotations.Benchmark;
15+
import org.openjdk.jmh.annotations.BenchmarkMode;
16+
import org.openjdk.jmh.annotations.Fork;
17+
import org.openjdk.jmh.annotations.Measurement;
18+
import org.openjdk.jmh.annotations.Mode;
19+
import org.openjdk.jmh.annotations.OutputTimeUnit;
20+
import org.openjdk.jmh.annotations.Param;
21+
import org.openjdk.jmh.annotations.Scope;
22+
import org.openjdk.jmh.annotations.Setup;
23+
import org.openjdk.jmh.annotations.State;
24+
import org.openjdk.jmh.annotations.Warmup;
25+
import org.openjdk.jmh.infra.Blackhole;
26+
27+
import java.io.IOException;
28+
import java.util.Random;
29+
import java.util.concurrent.TimeUnit;
30+
31+
@BenchmarkMode(Mode.Throughput)
32+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
33+
@State(Scope.Benchmark)
34+
// first iteration is complete garbage, so make sure we really warmup
35+
@Warmup(iterations = 4, time = 1)
36+
// real iterations. not useful to spend tons of time here, better to fork more
37+
@Measurement(iterations = 5, time = 1)
38+
// engage some noise reduction
39+
@Fork(value = 1)
40+
public class TransposeHalfByteBenchmark {
41+
42+
static {
43+
LogConfigurator.configureESLogging(); // native access requires logging to be initialized
44+
}
45+
46+
@Param({ "384", "782", "1024" })
47+
int dims;
48+
49+
int length;
50+
51+
int numVectors = 1000;
52+
53+
int[][] qVectors;
54+
byte[] packed;
55+
56+
@Setup
57+
public void setup() throws IOException {
58+
Random random = new Random(123);
59+
60+
this.length = 4 * BQVectorUtils.discretize(dims, 64) / 8;
61+
this.packed = new byte[length];
62+
63+
qVectors = new int[numVectors][dims];
64+
for (int[] qVector : qVectors) {
65+
for (int i = 0; i < dims; i++) {
66+
qVector[i] = random.nextInt(16);
67+
}
68+
}
69+
}
70+
71+
@Benchmark
72+
public void transposeHalfByte(Blackhole bh) {
73+
for (int i = 0; i < numVectors; i++) {
74+
BQSpaceUtils.transposeHalfByte(qVectors[i], packed);
75+
bh.consume(packed);
76+
}
77+
}
78+
79+
@Benchmark
80+
public void transposeHalfByteLegacy(Blackhole bh) {
81+
for (int i = 0; i < numVectors; i++) {
82+
BQSpaceUtils.transposeHalfByteLegacy(qVectors[i], packed);
83+
bh.consume(packed);
84+
}
85+
}
86+
87+
@Benchmark
88+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
89+
public void transposeHalfBytePanama(Blackhole bh) {
90+
for (int i = 0; i < numVectors; i++) {
91+
BQSpaceUtils.transposeHalfByte(qVectors[i], packed);
92+
bh.consume(packed);
93+
}
94+
}
95+
}

‎build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
import groovy.util.Node;
1313
import nmcp.NmcpPlugin;
1414

15-
import com.github.jengelman.gradle.plugins.shadow.ShadowExtension;
1615
import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin;
1716

1817
import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
@@ -173,8 +172,7 @@ private void addNameAndDescriptionToPom(Project project, NamedDomainObjectSet<Ma
173172
}
174173

175174
private static void configureWithShadowPlugin(Project project, MavenPublication publication) {
176-
var shadow = project.getExtensions().getByType(ShadowExtension.class);
177-
shadow.component(publication);
175+
publication.from(project.getComponents().getByName("shadow"));
178176
publication.artifact(project.getTasks().named("javadocJar"));
179177
publication.artifact(project.getTasks().named("sourcesJar"));
180178
}

‎build-tools-internal/build.gradle‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -220,13 +220,13 @@ gradlePlugin {
220220
id = 'elasticsearch.internal-yaml-rest-test'
221221
implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin'
222222
}
223-
transportVersionManagementPlugin {
224-
id = 'elasticsearch.transport-version-management'
225-
implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin'
223+
transportVersionReferencesPlugin {
224+
id = 'elasticsearch.transport-version-references'
225+
implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionReferencesPlugin'
226226
}
227-
globalTransportVersionManagementPlugin {
228-
id = 'elasticsearch.global-transport-version-management'
229-
implementationClass = 'org.elasticsearch.gradle.internal.transport.GlobalTransportVersionManagementPlugin'
227+
transportVersionResourcesPlugin {
228+
id = 'elasticsearch.transport-version-resources'
229+
implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionResourcesPlugin'
230230
}
231231
}
232232
}

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPluginFuncTest.groovy‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,8 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
9292

9393
file("myserver/build.gradle") << """
9494
apply plugin: 'java-library'
95-
apply plugin: 'elasticsearch.transport-version-management'
96-
apply plugin: 'elasticsearch.global-transport-version-management'
95+
apply plugin: 'elasticsearch.transport-version-references'
96+
apply plugin: 'elasticsearch.transport-version-resources'
9797
"""
9898
definedTransportVersion("existing_91", "8012000")
9999
definedTransportVersion("existing_92", "8123000,8012001")
@@ -112,7 +112,7 @@ class TransportVersionManagementPluginFuncTest extends AbstractGradleFuncTest {
112112

113113
file("myplugin/build.gradle") << """
114114
apply plugin: 'java-library'
115-
apply plugin: 'elasticsearch.transport-version-management'
115+
apply plugin: 'elasticsearch.transport-version-references'
116116
117117
dependencies {
118118
implementation project(":myserver")

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
import org.elasticsearch.gradle.internal.info.BuildParameterExtension;
1616
import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin;
1717
import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin;
18-
import org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin;
18+
import org.elasticsearch.gradle.internal.transport.TransportVersionReferencesPlugin;
1919
import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
2020
import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
2121
import org.elasticsearch.gradle.util.GradleUtils;
@@ -37,7 +37,7 @@ public void apply(Project project) {
3737
project.getPluginManager().apply(JarHellPrecommitPlugin.class);
3838
project.getPluginManager().apply(ElasticsearchJavaPlugin.class);
3939
project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class);
40-
project.getPluginManager().apply(TransportVersionManagementPlugin.class);
40+
project.getPluginManager().apply(TransportVersionReferencesPlugin.class);
4141
boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).getCi();
4242
// Clear default dependencies added by public PluginBuildPlugin as we add our
4343
// own project dependencies for internal builds

0 commit comments

Comments
 (0)