Skip to content

Commit ef0ec37

Browse files
felixbarnyfelixbarny
committed
Merge remote-tracking branch 'origin/main' into otlp-tsdb-2
2 parents a46882e + d036f8b commit ef0ec37

File tree

316 files changed

+9529
-2184
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

316 files changed

+9529
-2184
lines changed

‎.buildkite/pipelines/pull-request/performance-benchmark.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
steps:
2+
- label: ":pipeline: TODO"
3+
command: echo TODO

‎.buildkite/pull-requests.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
},
1919
{
2020
"enabled": true,
21-
"pipeline_slug": "elasticsearch-performance-esbench-pr",
21+
"pipeline_slug": "elasticsearch-pull-request-performance-benchmark",
2222
"allow_org_users": true,
2323
"allowed_repo_permissions": [
2424
"admin",

‎.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ testfixtures_shared/
6969
# Generated
7070
checkstyle_ide.xml
7171
x-pack/plugin/esql/src/main/generated-src/generated/
72-
server/src/main/resources/transport/defined/manifest.txt
72+
server/src/main/resources/transport/definitions/manifest.txt
7373

7474
# JEnv
7575
.java-version

‎BUILDING.md

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ uses the changed dependencies. In most cases, `precommit` or `check` are good ca
9292
We prefer sha256 checksums as md5 and sha1 are not considered safe anymore these days. The generated entry
9393
will have the `origin` attribute been set to `Generated by Gradle`.
9494

95-
> [!Tip]
95+
> [!Tip]
9696
> A manual confirmation of the Gradle generated checksums is currently not mandatory.
9797
> If you want to add a level of verification you can manually confirm the checksum (e.g. by looking it up on the website of the library)
9898
> Please replace the content of the `origin` attribute by `official site` in that case.
@@ -186,6 +186,25 @@ dependencies {
186186

187187
To test an unreleased development version of a third party dependency you have several options.
188188

189+
### How do I test against java early access (ea) versions?
190+
191+
Currently only openjdk EA builds by oracle are supported.
192+
To test against an early access version java version you can pass the major
193+
java version appended with `-ea` as a system property (e.g. -Druntime.java=26-ea) to the Gradle build:
194+
195+
```
196+
./gradlew clean test -Druntime.java=26-ea
197+
```
198+
199+
This will run the tests using the JDK 26 EA version and pick the latest available build of the matching JDK EA version we expose
200+
in our custom jdk catalogue at `https://storage.googleapis.com/elasticsearch-jdk-archive/jdks/openjdk/latest.json`.
201+
202+
To run against a specific build number of the EA build you can pass a second system property (e.g. `-Druntime.java.build=6`):
203+
204+
```
205+
./gradlew clean test -Druntime.java=26-ea -Druntime.java.build=6
206+
```
207+
189208
#### How to use a Maven based third party dependency via `mavenlocal`?
190209

191210
1. Clone the third party repository locally
@@ -229,7 +248,7 @@ In addition to snapshot builds JitPack supports building Pull Requests. Simply u
229248
3. Run the Gradle build as needed. Keep in mind the initial resolution might take a bit longer as this needs to be built
230249
by JitPack in the background before we can resolve the adhoc built dependency.
231250

232-
> [!Note]
251+
> [!Note]
233252
> You should only use that approach locally or on a developer branch for production dependencies as we do
234253
not want to ship unreleased libraries into our releases.
235254

@@ -261,7 +280,7 @@ allprojects {
261280
```
262281
4. Run the Gradle build as needed with `--write-verification-metadata` to ensure the Gradle dependency verification does not fail on your custom dependency.
263282

264-
> [!Note]
283+
> [!Note]
265284
> As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
266285
flat directory repositories cannot be used to override artifacts with real meta-data from other repositories declared in the build.
267286
> For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository

‎benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramGenerationBench.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
package org.elasticsearch.benchmark.exponentialhistogram;
1111

12+
import org.elasticsearch.exponentialhistogram.ExponentialHistogramCircuitBreaker;
1213
import org.elasticsearch.exponentialhistogram.ExponentialHistogramGenerator;
1314
import org.openjdk.jmh.annotations.Benchmark;
1415
import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -59,7 +60,7 @@ public class ExponentialHistogramGenerationBench {
5960
@Setup
6061
public void setUp() {
6162
random = ThreadLocalRandom.current();
62-
histoGenerator = new ExponentialHistogramGenerator(bucketCount);
63+
histoGenerator = ExponentialHistogramGenerator.create(bucketCount, ExponentialHistogramCircuitBreaker.noop());
6364

6465
DoubleSupplier nextRandom = () -> distribution.equals("GAUSSIAN") ? random.nextGaussian() : random.nextDouble();
6566

‎benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
import org.elasticsearch.exponentialhistogram.BucketIterator;
1313
import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
14+
import org.elasticsearch.exponentialhistogram.ExponentialHistogramCircuitBreaker;
1415
import org.elasticsearch.exponentialhistogram.ExponentialHistogramGenerator;
1516
import org.elasticsearch.exponentialhistogram.ExponentialHistogramMerger;
1617
import org.openjdk.jmh.annotations.Benchmark;
@@ -56,13 +57,14 @@ public class ExponentialHistogramMergeBench {
5657
@Setup
5758
public void setUp() {
5859
random = ThreadLocalRandom.current();
59-
histoMerger = new ExponentialHistogramMerger(bucketCount);
60+
ExponentialHistogramCircuitBreaker breaker = ExponentialHistogramCircuitBreaker.noop();
61+
histoMerger = ExponentialHistogramMerger.create(bucketCount, breaker);
6062

61-
ExponentialHistogramGenerator initial = new ExponentialHistogramGenerator(bucketCount);
63+
ExponentialHistogramGenerator initialGenerator = ExponentialHistogramGenerator.create(bucketCount, breaker);
6264
for (int j = 0; j < bucketCount; j++) {
63-
initial.add(Math.pow(1.001, j));
65+
initialGenerator.add(Math.pow(1.001, j));
6466
}
65-
ExponentialHistogram initialHisto = initial.get();
67+
ExponentialHistogram initialHisto = initialGenerator.getAndClear();
6668
int cnt = getBucketCount(initialHisto);
6769
if (cnt < bucketCount) {
6870
throw new IllegalArgumentException("Expected bucket count to be " + bucketCount + ", but was " + cnt);
@@ -72,14 +74,14 @@ public void setUp() {
7274
int dataPointSize = (int) Math.round(bucketCount * mergedHistoSizeFactor);
7375

7476
for (int i = 0; i < toMerge.length; i++) {
75-
ExponentialHistogramGenerator generator = new ExponentialHistogramGenerator(dataPointSize);
77+
ExponentialHistogramGenerator generator = ExponentialHistogramGenerator.create(dataPointSize, breaker);
7678

7779
int bucketIndex = 0;
7880
for (int j = 0; j < dataPointSize; j++) {
7981
bucketIndex += 1 + random.nextInt(bucketCount) % (Math.max(1, bucketCount / dataPointSize));
8082
generator.add(Math.pow(1.001, bucketIndex));
8183
}
82-
toMerge[i] = generator.get();
84+
toMerge[i] = generator.getAndClear();
8385
cnt = getBucketCount(toMerge[i]);
8486
if (cnt < dataPointSize) {
8587
throw new IllegalArgumentException("Expected bucket count to be " + dataPointSize + ", but was " + cnt);

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/PackAsBinaryBenchmark.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,4 +82,13 @@ public void packAsBinaryLegacy(Blackhole bh) {
8282
bh.consume(packed);
8383
}
8484
}
85+
86+
@Benchmark
87+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
88+
public void packAsBinaryPanama(Blackhole bh) {
89+
for (int i = 0; i < numVectors; i++) {
90+
BQVectorUtils.packAsBinary(qVectors[i], packed);
91+
bh.consume(packed);
92+
}
93+
}
8594
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/TransposeHalfByteBenchmark.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,4 +83,13 @@ public void transposeHalfByteLegacy(Blackhole bh) {
8383
bh.consume(packed);
8484
}
8585
}
86+
87+
@Benchmark
88+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
89+
public void transposeHalfBytePanama(Blackhole bh) {
90+
for (int i = 0; i < numVectors; i++) {
91+
BQSpaceUtils.transposeHalfByte(qVectors[i], packed);
92+
bh.consume(packed);
93+
}
94+
}
8695
}

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
package org.elasticsearch.gradle.fixtures
1111

1212
import org.apache.commons.io.FileUtils
13-
import org.elasticsearch.gradle.internal.test.InternalAwareGradleRunner
1413
import org.gradle.testkit.runner.GradleRunner
1514
import org.junit.Rule
1615
import org.junit.rules.TemporaryFolder

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy

Lines changed: 36 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,18 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
3333
private static final String ADOPT_JDK_VERSION_15 = "15.0.2+7"
3434
private static final String AZUL_JDK_VERSION_8 = "8u302+b08"
3535
private static final String AZUL_8_DISTRO_VERSION = "8.56.0.23"
36+
private static final String CATALOG_EA_VERSION = "25-ea+30"
3637
private static final String OPEN_JDK_VERSION = "12.0.1+99@123456789123456789123456789abcde"
3738
private static final Pattern JDK_HOME_LOGLINE = Pattern.compile("JDK HOME: (.*)")
3839

40+
def setup() {
41+
configurationCacheCompatible = false // JDK class references configurations which break configuration cache
42+
}
43+
3944
@Unroll
40-
def "jdk #jdkVendor for #platform#suffix are downloaded and extracted"() {
45+
def "jdk #distributionVersion #jdkVendor for #platform#suffix are downloaded and extracted"() {
4146
given:
42-
def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform, arch);
47+
def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform, arch, distributionVersion);
4348
def mockedContent = filebytes(jdkVendor, platform)
4449
buildFile.text = """
4550
plugins {
@@ -77,21 +82,26 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
7782

7883
where:
7984
platform | arch | jdkVendor | jdkVersion | distributionVersion | expectedJavaBin | suffix
80-
"linux" | "x64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | null | "bin/java" | ""
81-
"linux" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | null | "bin/java" | ""
82-
"linux" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | null | "bin/java" | "(old version)"
83-
"windows" | "x64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | null | "bin/java" | ""
84-
"windows" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | null | "bin/java" | ""
85-
"windows" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | null | "bin/java" | "(old version)"
86-
"darwin" | "x64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | null | "Contents/Home/bin/java" | ""
87-
"darwin" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | null | "Contents/Home/bin/java" | ""
88-
"darwin" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | null | "Contents/Home/bin/java" | "(old version)"
89-
"mac" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | null | "Contents/Home/bin/java" | ""
90-
"mac" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | null | "Contents/Home/bin/java" | "(old version)"
91-
"darwin" | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | null | "Contents/Home/bin/java" | ""
92-
"linux" | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | null | "bin/java" | ""
93-
"linux" | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_11 | null | "bin/java" | "(jdk 11)"
94-
"linux" | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_15 | null | "bin/java" | "(jdk 15)"
85+
"linux" | "aarch64" | VENDOR_OPENJDK | CATALOG_EA_VERSION | "ea" | "bin/java" | ""
86+
"linux" | "x64" | VENDOR_OPENJDK | CATALOG_EA_VERSION | "ea" | "bin/java" | ""
87+
"darwin" | "aarch64" | VENDOR_OPENJDK | CATALOG_EA_VERSION | "ea" | "Contents/Home/bin/java" | ""
88+
"darwin" | "x64" | VENDOR_OPENJDK | CATALOG_EA_VERSION | "ea" | "Contents/Home/bin/java" | ""
89+
"windows" | "x64" | VENDOR_OPENJDK | CATALOG_EA_VERSION | "ea" | "bin/java" | ""
90+
"linux" | "x64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | "" | "bin/java" | ""
91+
"linux" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "" | "bin/java" | ""
92+
"linux" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "" | "bin/java" | "(old version)"
93+
"windows" | "x64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | "" | "bin/java" | ""
94+
"windows" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "" | "bin/java" | ""
95+
"windows" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "" | "bin/java" | "(old version)"
96+
"darwin" | "x64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | "" | "Contents/Home/bin/java" | ""
97+
"darwin" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "" | "Contents/Home/bin/java" | ""
98+
"darwin" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "" | "Contents/Home/bin/java" | "(old version)"
99+
"mac" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "" | "Contents/Home/bin/java" | ""
100+
"mac" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "" | "Contents/Home/bin/java" | "(old version)"
101+
"darwin" | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | "" | "Contents/Home/bin/java" | ""
102+
"linux" | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION | "" | "bin/java" | ""
103+
"linux" | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_11 | "" | "bin/java" | "(jdk 11)"
104+
"linux" | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_15 | "" | "bin/java" | "(jdk 15)"
95105
"darwin" | "aarch64" | VENDOR_ZULU | AZUL_JDK_VERSION_8 | AZUL_8_DISTRO_VERSION | "Contents/Home/bin/java" | "(jdk 8)"
96106
}
97107

@@ -214,13 +224,19 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
214224
private static String urlPath(final String vendor,
215225
final String version,
216226
final String platform,
217-
final String arch = 'x64') {
218-
if (vendor.equals(VENDOR_ADOPTIUM)) {
227+
final String arch = 'x64',
228+
final String distributedVersion = '') {
229+
final boolean isOld = version.equals(OPENJDK_VERSION_OLD);
230+
231+
if (distributedVersion.equals("ea")) {
232+
def effectivePlatform = isMac(platform) ? "macos" : platform;
233+
def fileExtension = platform.toLowerCase().equals("windows") ? "zip" : "tar.gz";
234+
return "/jdks/openjdk/25/openjdk-${version}/openjdk-${version}_$effectivePlatform-${arch}_bin.$fileExtension";
235+
} else if (vendor.equals(VENDOR_ADOPTIUM)) {
219236
final String module = isMac(platform) ? "mac" : platform;
220237
return "/jdk-" + version + "/" + module + "/${arch}/jdk/hotspot/normal/adoptium";
221238
} else if (vendor.equals(VENDOR_OPENJDK)) {
222239
final String effectivePlatform = isMac(platform) ? "macos" : platform;
223-
final boolean isOld = version.equals(OPENJDK_VERSION_OLD);
224240
final String versionPath = isOld ? "jdk1/99" : "jdk12.0.1/123456789123456789123456789abcde/99";
225241
final String filename = "openjdk-" + (isOld ? "1" : "12.0.1") + "_" + effectivePlatform + "-x64_bin." + extension(platform);
226242
return "/java/GA/" + versionPath + "/GPL/" + filename;

0 commit comments

Comments
 (0)