Skip to content

Commit e9c9208

Browse files
julian-elasticjulian-elastic
committed
Merge branch 'main' into joinTypeRefector
2 parents 34277cc + 1e1bf28 commit e9c9208

File tree

724 files changed

+21753
-7212
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

724 files changed

+21753
-7212
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/RecyclerBytesStreamOutputBenchmark.java‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.benchmark.bytes;
11+
12+
import org.apache.lucene.util.BytesRef;
13+
import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput;
14+
import org.elasticsearch.common.recycler.Recycler;
15+
import org.openjdk.jmh.annotations.Benchmark;
16+
import org.openjdk.jmh.annotations.BenchmarkMode;
17+
import org.openjdk.jmh.annotations.Fork;
18+
import org.openjdk.jmh.annotations.Measurement;
19+
import org.openjdk.jmh.annotations.Mode;
20+
import org.openjdk.jmh.annotations.OutputTimeUnit;
21+
import org.openjdk.jmh.annotations.Scope;
22+
import org.openjdk.jmh.annotations.Setup;
23+
import org.openjdk.jmh.annotations.State;
24+
import org.openjdk.jmh.annotations.Warmup;
25+
26+
import java.io.IOException;
27+
import java.util.concurrent.ThreadLocalRandom;
28+
import java.util.concurrent.TimeUnit;
29+
import java.util.concurrent.atomic.AtomicReference;
30+
31+
@Warmup(iterations = 3)
32+
@Measurement(iterations = 3)
33+
@BenchmarkMode(Mode.AverageTime)
34+
@OutputTimeUnit(TimeUnit.NANOSECONDS)
35+
@State(Scope.Thread)
36+
@Fork(value = 1)
37+
public class RecyclerBytesStreamOutputBenchmark {
38+
39+
private final AtomicReference<BytesRef> bytesRef = new AtomicReference<>(new BytesRef(16384));
40+
private RecyclerBytesStreamOutput streamOutput;
41+
private String shortString;
42+
private String longString;
43+
private String nonAsciiString;
44+
private String veryLongString;
45+
private byte[] bytes1;
46+
private byte[] bytes2;
47+
private byte[] bytes3;
48+
private byte[] multiPageBytes;
49+
private int[] vints;
50+
51+
@Setup
52+
public void initResults() throws IOException {
53+
streamOutput = new RecyclerBytesStreamOutput(new BenchmarkRecycler(bytesRef));
54+
ThreadLocalRandom random = ThreadLocalRandom.current();
55+
56+
bytes1 = new byte[327];
57+
bytes2 = new byte[712];
58+
bytes3 = new byte[1678];
59+
multiPageBytes = new byte[16387 * 4];
60+
random.nextBytes(bytes1);
61+
random.nextBytes(bytes2);
62+
random.nextBytes(bytes3);
63+
random.nextBytes(multiPageBytes);
64+
65+
// We use weights to generate certain sized UTF-8 characters and vInts. However, there is still some non-determinism which could
66+
// impact direct comparisons run-to-run
67+
68+
shortString = generateAsciiString(20);
69+
longString = generateAsciiString(100);
70+
nonAsciiString = generateUtf8String(200);
71+
veryLongString = generateAsciiString(800);
72+
// vint values for benchmarking
73+
vints = new int[1000];
74+
for (int i = 0; i < vints.length; i++) {
75+
if (random.nextBoolean()) {
76+
// 1-byte 50% of the time
77+
vints[i] = random.nextInt(128);
78+
} else if (random.nextBoolean()) {
79+
// 2-byte 25% of the time
80+
vints[i] = random.nextInt(128, 16384);
81+
} else {
82+
if (random.nextBoolean()) {
83+
// 3-byte vints
84+
vints[i] = random.nextInt(16384, 2097152);
85+
} else {
86+
// All vint variants
87+
vints[i] = random.nextInt();
88+
}
89+
}
90+
}
91+
}
92+
93+
@Benchmark
94+
public void writeByte() throws IOException {
95+
streamOutput.seek(1);
96+
for (byte item : bytes1) {
97+
streamOutput.writeByte(item);
98+
}
99+
for (byte item : bytes2) {
100+
streamOutput.writeByte(item);
101+
}
102+
for (byte item : bytes3) {
103+
streamOutput.writeByte(item);
104+
}
105+
}
106+
107+
@Benchmark
108+
public void writeBytes() throws IOException {
109+
streamOutput.seek(1);
110+
streamOutput.writeBytes(bytes1, 0, bytes1.length);
111+
streamOutput.writeBytes(bytes2, 0, bytes2.length);
112+
streamOutput.writeBytes(bytes3, 0, bytes3.length);
113+
}
114+
115+
@Benchmark
116+
public void writeBytesAcrossPageBoundary() throws IOException {
117+
streamOutput.seek(16384 - 1000);
118+
streamOutput.writeBytes(bytes1, 0, bytes1.length);
119+
streamOutput.writeBytes(bytes2, 0, bytes2.length);
120+
streamOutput.writeBytes(bytes3, 0, bytes3.length);
121+
}
122+
123+
@Benchmark
124+
public void writeBytesMultiPage() throws IOException {
125+
streamOutput.seek(16384 - 1000);
126+
streamOutput.writeBytes(multiPageBytes, 0, multiPageBytes.length);
127+
}
128+
129+
@Benchmark
130+
public void writeString() throws IOException {
131+
streamOutput.seek(1);
132+
streamOutput.writeString(shortString);
133+
streamOutput.writeString(longString);
134+
streamOutput.writeString(nonAsciiString);
135+
streamOutput.writeString(veryLongString);
136+
}
137+
138+
@Benchmark
139+
public void writeVInt() throws IOException {
140+
streamOutput.seek(1);
141+
for (int vint : vints) {
142+
streamOutput.writeVInt(vint);
143+
}
144+
}
145+
146+
public static String generateAsciiString(int n) {
147+
ThreadLocalRandom random = ThreadLocalRandom.current();
148+
StringBuilder sb = new StringBuilder(n);
149+
150+
for (int i = 0; i < n; i++) {
151+
int ascii = random.nextInt(128);
152+
sb.append((char) ascii);
153+
}
154+
155+
return sb.toString();
156+
}
157+
158+
public static String generateUtf8String(int n) {
159+
ThreadLocalRandom random = ThreadLocalRandom.current();
160+
StringBuilder sb = new StringBuilder(n);
161+
162+
for (int i = 0; i < n; i++) {
163+
int codePoint;
164+
int probability = random.nextInt(100);
165+
166+
if (probability < 85) {
167+
// 1-byte UTF-8 (ASCII range)
168+
// 0x0000 to 0x007F
169+
codePoint = random.nextInt(0x0080);
170+
} else if (probability < 95) {
171+
// 2-byte UTF-8
172+
// 0x0080 to 0x07FF
173+
codePoint = random.nextInt(0x0080, 0x0800);
174+
} else {
175+
// 3-byte UTF-8
176+
// 0x0800 to 0xFFFF
177+
do {
178+
codePoint = random.nextInt(0x0800, 0x10000);
179+
// Skip surrogate pairs (0xD800-0xDFFF)
180+
} while (codePoint >= 0xD800 && codePoint <= 0xDFFF);
181+
}
182+
183+
sb.appendCodePoint(codePoint);
184+
}
185+
186+
return sb.toString();
187+
}
188+
189+
private record BenchmarkRecycler(AtomicReference<BytesRef> bytesRef) implements Recycler<BytesRef> {
190+
191+
@Override
192+
public V<BytesRef> obtain() {
193+
BytesRef recycledBytesRef = bytesRef.getAndSet(null);
194+
final BytesRef localBytesRef;
195+
final boolean recycled;
196+
if (recycledBytesRef != null) {
197+
recycled = true;
198+
localBytesRef = recycledBytesRef;
199+
} else {
200+
recycled = false;
201+
localBytesRef = new BytesRef(16384);
202+
}
203+
return new V<>() {
204+
@Override
205+
public BytesRef v() {
206+
return localBytesRef;
207+
}
208+
209+
@Override
210+
public boolean isRecycled() {
211+
return recycled;
212+
}
213+
214+
@Override
215+
public void close() {
216+
if (recycled) {
217+
bytesRef.set(localBytesRef);
218+
}
219+
}
220+
};
221+
}
222+
223+
@Override
224+
public int pageSize() {
225+
return 16384;
226+
}
227+
}
228+
}

‎build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,23 @@ public void apply(Project project) {
5252
// Spotless resolves required dependencies from project repositories, so we need maven central
5353
project.getRepositories().mavenCentral();
5454

55+
// we cannot update to a newer spotless plugin version yet but we want to use the latest eclipse formatter to be compatible
56+
// with latest java versions
57+
project.getConfigurations().matching(it -> it.getName().startsWith("spotless")).configureEach(conf -> {
58+
project.getDependencies().constraints(constraints -> {
59+
constraints.add(conf.getName(), "org.eclipse.jdt:org.eclipse.jdt.core:3.42.0", dependencyConstraint -> {
60+
dependencyConstraint.because(
61+
"We want to use a recent version of the Eclipse formatter libraries to support latest Java"
62+
);
63+
});
64+
constraints.add(conf.getName(), "org.eclipse.jdt:ecj:3.42.0", dependencyConstraint -> {
65+
dependencyConstraint.because(
66+
"We want to use a recent version of the Eclipse formatter libraries to support latest Java"
67+
);
68+
});
69+
});
70+
});
71+
5572
project.getExtensions().getByType(SpotlessExtension.class).java(java -> {
5673
File elasticsearchWorkspace = Util.locateElasticsearchWorkspace(project.getGradle());
5774
String importOrderPath = "build-conventions/elastic.importorder";
@@ -74,7 +91,7 @@ public void apply(Project project) {
7491
// When running build benchmarks we alter the source in some scenarios.
7592
// The gradle-profiler unfortunately does not generate compliant formatted
7693
// sources so we ignore that altered file when running build benchmarks
77-
if(Boolean.getBoolean("BUILD_PERFORMANCE_TEST") && project.getPath().equals(":server")) {
94+
if (Boolean.getBoolean("BUILD_PERFORMANCE_TEST") && project.getPath().equals(":server")) {
7895
java.targetExclude("src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java");
7996
}
8097
});

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/ResolveTransportVersionConflictFuncTest.groovy‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,4 +87,13 @@ class ResolveTransportVersionConflictFuncTest extends AbstractTransportVersionFu
8787
assertUpperBound("9.1", "new_tv,8012002")
8888
assertUpperBound("8.19", "new_tv,7123002")
8989
}
90+
91+
def "no new transport version is idempotent"() {
92+
when:
93+
def result = runResolveAndValidateTask().build()
94+
95+
then:
96+
assertResolveAndValidateSuccess(result)
97+
assertUpperBound("9.2", "existing_92,8123000")
98+
}
9099
}

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionGenerationFuncTest.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -428,7 +428,7 @@ class TransportVersionGenerationFuncTest extends AbstractTransportVersionFuncTes
428428

429429
def "upper bounds files must exist for backport branches"() {
430430
when:
431-
def result = runGenerateTask("--backport-branches=9.1,8.13,7.17,6.0").buildAndFail()
431+
def result = runGenerateTask("--name", "new_tv", "--backport-branches=9.1,8.13,7.17,6.0").buildAndFail()
432432

433433
then:
434434
assertGenerateFailure(result, "Missing upper bounds files for branches [6.0, 7.17, 8.13], known branches are [8.19, 9.0, 9.1, 9.2]")

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,4 +294,21 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes
294294
then:
295295
result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
296296
}
297+
298+
def "only current upper bound validated on release branch"() {
299+
given:
300+
file("myserver/build.gradle") << """
301+
tasks.named('validateTransportVersionResources') {
302+
currentUpperBoundName = '9.0'
303+
}
304+
"""
305+
referableAndReferencedTransportVersion("some_tv", "8124000,8012004")
306+
transportVersionUpperBound("9.1", "some_tv", "8012004")
307+
308+
when:
309+
def result = gradleRunner("validateTransportVersionResources").build()
310+
311+
then:
312+
result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
313+
}
297314
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,9 @@ private void registerInternalDistributionResolutions(List<DistributionResolution
113113
String versionProperty = System.getProperty("tests.bwc.main.version");
114114
// We use this phony version as a placeholder for the real version
115115
if (distribution.getVersion().equals("0.0.0")) {
116+
if (versionProperty == null) {
117+
throw new GradleException("System property 'tests.bwc.main.version' expected for building bwc version.");
118+
}
116119
BwcVersions.UnreleasedVersionInfo unreleasedVersionInfo = new BwcVersions.UnreleasedVersionInfo(
117120
Version.fromString(versionProperty),
118121
"main",

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,14 @@
4444
import org.gradle.api.file.FileCollection;
4545
import org.gradle.api.file.FileTree;
4646
import org.gradle.api.internal.artifacts.dependencies.ProjectDependencyInternal;
47+
import org.gradle.api.plugins.JvmToolchainsPlugin;
4748
import org.gradle.api.provider.ProviderFactory;
4849
import org.gradle.api.tasks.ClasspathNormalizer;
4950
import org.gradle.api.tasks.PathSensitivity;
5051
import org.gradle.api.tasks.util.PatternFilterable;
52+
import org.gradle.jvm.toolchain.JavaLanguageVersion;
53+
import org.gradle.jvm.toolchain.JavaToolchainService;
54+
import org.gradle.jvm.toolchain.JvmVendorSpec;
5155

5256
import java.util.Collection;
5357
import java.util.Iterator;
@@ -59,6 +63,7 @@
5963
import javax.inject.Inject;
6064

6165
import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams;
66+
import static org.elasticsearch.gradle.util.OsUtils.jdkIsIncompatibleWithOS;
6267

6368
/**
6469
* Base plugin used for wiring up build tasks to REST testing tasks using new JUnit rule-based test clusters framework.
@@ -94,6 +99,7 @@ public RestTestBasePlugin(ProviderFactory providerFactory) {
9499
public void apply(Project project) {
95100
project.getPluginManager().apply(ElasticsearchJavaBasePlugin.class);
96101
project.getPluginManager().apply(InternalDistributionDownloadPlugin.class);
102+
project.getPluginManager().apply(JvmToolchainsPlugin.class);
97103
var bwcVersions = loadBuildParams(project).get().getBwcVersions();
98104

99105
// Register integ-test and default distributions
@@ -236,6 +242,17 @@ public Void call(Object... args) {
236242
String versionString = version.toString();
237243
ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString);
238244

245+
if (jdkIsIncompatibleWithOS(Version.fromString(versionString))) {
246+
var toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
247+
var fallbackJdk17Launcher = toolChainService.launcherFor(spec -> {
248+
spec.getVendor().set(JvmVendorSpec.ADOPTIUM);
249+
spec.getLanguageVersion().set(JavaLanguageVersion.of(17));
250+
});
251+
task.environment(
252+
"ES_FALLBACK_JAVA_HOME",
253+
fallbackJdk17Launcher.get().getMetadata().getInstallationPath().getAsFile().getPath()
254+
);
255+
}
239256
task.dependsOn(bwcDistro);
240257
registerDistributionInputs(task, bwcDistro);
241258

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -98,14 +98,14 @@ public void run() throws IOException {
9898
List<String> changedDefinitionNames = resources.getChangedReferableDefinitionNames();
9999
String targetDefinitionName = getTargetDefinitionName(resources, referencedNames, changedDefinitionNames);
100100

101-
List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromUpstream();
102-
Set<String> targetUpperBoundNames = getTargetUpperBoundNames(resources, upstreamUpperBounds, targetDefinitionName);
103-
104101
getLogger().lifecycle("Generating transport version name: " + targetDefinitionName);
105102
if (targetDefinitionName.isEmpty()) {
106103
// TODO: resetting upper bounds needs to be done locally, otherwise it pulls in some (incomplete) changes from upstream main
107104
// resetAllUpperBounds(resources);
108105
} else {
106+
List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromUpstream();
107+
Set<String> targetUpperBoundNames = getTargetUpperBoundNames(resources, upstreamUpperBounds, targetDefinitionName);
108+
109109
List<TransportVersionId> ids = updateUpperBounds(resources, upstreamUpperBounds, targetUpperBoundNames, targetDefinitionName);
110110
// (Re)write the definition file.
111111
resources.writeDefinition(new TransportVersionDefinition(targetDefinitionName, ids, true));

0 commit comments

Comments
 (0)