Skip to content

Commit dfd6ec1

Browse files
benchaplinbenchaplin
authored
Merge branch 'main' into fix_batched_doc_error_count
2 parents a91a426 + 5aebcce commit dfd6ec1

File tree

208 files changed

+8029
-2006
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

208 files changed

+8029
-2006
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/RecyclerBytesStreamOutputBenchmark.java‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.benchmark.bytes;
11+
12+
import org.apache.lucene.util.BytesRef;
13+
import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput;
14+
import org.elasticsearch.common.recycler.Recycler;
15+
import org.openjdk.jmh.annotations.Benchmark;
16+
import org.openjdk.jmh.annotations.BenchmarkMode;
17+
import org.openjdk.jmh.annotations.Fork;
18+
import org.openjdk.jmh.annotations.Measurement;
19+
import org.openjdk.jmh.annotations.Mode;
20+
import org.openjdk.jmh.annotations.OutputTimeUnit;
21+
import org.openjdk.jmh.annotations.Scope;
22+
import org.openjdk.jmh.annotations.Setup;
23+
import org.openjdk.jmh.annotations.State;
24+
import org.openjdk.jmh.annotations.Warmup;
25+
26+
import java.io.IOException;
27+
import java.util.concurrent.ThreadLocalRandom;
28+
import java.util.concurrent.TimeUnit;
29+
import java.util.concurrent.atomic.AtomicReference;
30+
31+
@Warmup(iterations = 3)
32+
@Measurement(iterations = 3)
33+
@BenchmarkMode(Mode.AverageTime)
34+
@OutputTimeUnit(TimeUnit.NANOSECONDS)
35+
@State(Scope.Thread)
36+
@Fork(value = 1)
37+
public class RecyclerBytesStreamOutputBenchmark {
38+
39+
private final AtomicReference<BytesRef> bytesRef = new AtomicReference<>(new BytesRef(16384));
40+
private RecyclerBytesStreamOutput streamOutput;
41+
private String shortString;
42+
private String longString;
43+
private String nonAsciiString;
44+
private String veryLongString;
45+
private byte[] bytes1;
46+
private byte[] bytes2;
47+
private byte[] bytes3;
48+
private byte[] multiPageBytes;
49+
private int[] vints;
50+
51+
@Setup
52+
public void initResults() throws IOException {
53+
streamOutput = new RecyclerBytesStreamOutput(new BenchmarkRecycler(bytesRef));
54+
ThreadLocalRandom random = ThreadLocalRandom.current();
55+
56+
bytes1 = new byte[327];
57+
bytes2 = new byte[712];
58+
bytes3 = new byte[1678];
59+
multiPageBytes = new byte[16387 * 4];
60+
random.nextBytes(bytes1);
61+
random.nextBytes(bytes2);
62+
random.nextBytes(bytes3);
63+
random.nextBytes(multiPageBytes);
64+
65+
// We use weights to generate certain sized UTF-8 characters and vInts. However, there is still some non-determinism which could
66+
// impact direct comparisons run-to-run
67+
68+
shortString = generateAsciiString(20);
69+
longString = generateAsciiString(100);
70+
nonAsciiString = generateUtf8String(200);
71+
veryLongString = generateAsciiString(800);
72+
// vint values for benchmarking
73+
vints = new int[1000];
74+
for (int i = 0; i < vints.length; i++) {
75+
if (random.nextBoolean()) {
76+
// 1-byte 50% of the time
77+
vints[i] = random.nextInt(128);
78+
} else if (random.nextBoolean()) {
79+
// 2-byte 25% of the time
80+
vints[i] = random.nextInt(128, 16384);
81+
} else {
82+
if (random.nextBoolean()) {
83+
// 3-byte vints
84+
vints[i] = random.nextInt(16384, 2097152);
85+
} else {
86+
// All vint variants
87+
vints[i] = random.nextInt();
88+
}
89+
}
90+
}
91+
}
92+
93+
@Benchmark
94+
public void writeByte() throws IOException {
95+
streamOutput.seek(1);
96+
for (byte item : bytes1) {
97+
streamOutput.writeByte(item);
98+
}
99+
for (byte item : bytes2) {
100+
streamOutput.writeByte(item);
101+
}
102+
for (byte item : bytes3) {
103+
streamOutput.writeByte(item);
104+
}
105+
}
106+
107+
@Benchmark
108+
public void writeBytes() throws IOException {
109+
streamOutput.seek(1);
110+
streamOutput.writeBytes(bytes1, 0, bytes1.length);
111+
streamOutput.writeBytes(bytes2, 0, bytes2.length);
112+
streamOutput.writeBytes(bytes3, 0, bytes3.length);
113+
}
114+
115+
@Benchmark
116+
public void writeBytesAcrossPageBoundary() throws IOException {
117+
streamOutput.seek(16384 - 1000);
118+
streamOutput.writeBytes(bytes1, 0, bytes1.length);
119+
streamOutput.writeBytes(bytes2, 0, bytes2.length);
120+
streamOutput.writeBytes(bytes3, 0, bytes3.length);
121+
}
122+
123+
@Benchmark
124+
public void writeBytesMultiPage() throws IOException {
125+
streamOutput.seek(16384 - 1000);
126+
streamOutput.writeBytes(multiPageBytes, 0, multiPageBytes.length);
127+
}
128+
129+
@Benchmark
130+
public void writeString() throws IOException {
131+
streamOutput.seek(1);
132+
streamOutput.writeString(shortString);
133+
streamOutput.writeString(longString);
134+
streamOutput.writeString(nonAsciiString);
135+
streamOutput.writeString(veryLongString);
136+
}
137+
138+
@Benchmark
139+
public void writeVInt() throws IOException {
140+
streamOutput.seek(1);
141+
for (int vint : vints) {
142+
streamOutput.writeVInt(vint);
143+
}
144+
}
145+
146+
public static String generateAsciiString(int n) {
147+
ThreadLocalRandom random = ThreadLocalRandom.current();
148+
StringBuilder sb = new StringBuilder(n);
149+
150+
for (int i = 0; i < n; i++) {
151+
int ascii = random.nextInt(128);
152+
sb.append((char) ascii);
153+
}
154+
155+
return sb.toString();
156+
}
157+
158+
public static String generateUtf8String(int n) {
159+
ThreadLocalRandom random = ThreadLocalRandom.current();
160+
StringBuilder sb = new StringBuilder(n);
161+
162+
for (int i = 0; i < n; i++) {
163+
int codePoint;
164+
int probability = random.nextInt(100);
165+
166+
if (probability < 85) {
167+
// 1-byte UTF-8 (ASCII range)
168+
// 0x0000 to 0x007F
169+
codePoint = random.nextInt(0x0080);
170+
} else if (probability < 95) {
171+
// 2-byte UTF-8
172+
// 0x0080 to 0x07FF
173+
codePoint = random.nextInt(0x0080, 0x0800);
174+
} else {
175+
// 3-byte UTF-8
176+
// 0x0800 to 0xFFFF
177+
do {
178+
codePoint = random.nextInt(0x0800, 0x10000);
179+
// Skip surrogate pairs (0xD800-0xDFFF)
180+
} while (codePoint >= 0xD800 && codePoint <= 0xDFFF);
181+
}
182+
183+
sb.appendCodePoint(codePoint);
184+
}
185+
186+
return sb.toString();
187+
}
188+
189+
private record BenchmarkRecycler(AtomicReference<BytesRef> bytesRef) implements Recycler<BytesRef> {
190+
191+
@Override
192+
public V<BytesRef> obtain() {
193+
BytesRef recycledBytesRef = bytesRef.getAndSet(null);
194+
final BytesRef localBytesRef;
195+
final boolean recycled;
196+
if (recycledBytesRef != null) {
197+
recycled = true;
198+
localBytesRef = recycledBytesRef;
199+
} else {
200+
recycled = false;
201+
localBytesRef = new BytesRef(16384);
202+
}
203+
return new V<>() {
204+
@Override
205+
public BytesRef v() {
206+
return localBytesRef;
207+
}
208+
209+
@Override
210+
public boolean isRecycled() {
211+
return recycled;
212+
}
213+
214+
@Override
215+
public void close() {
216+
if (recycled) {
217+
bytesRef.set(localBytesRef);
218+
}
219+
}
220+
};
221+
}
222+
223+
@Override
224+
public int pageSize() {
225+
return 16384;
226+
}
227+
}
228+
}

‎build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,23 @@ public void apply(Project project) {
5252
// Spotless resolves required dependencies from project repositories, so we need maven central
5353
project.getRepositories().mavenCentral();
5454

55+
// we cannot update to a newer spotless plugin version yet but we want to use the latest eclipse formatter to be compatible
56+
// with latest java versions
57+
project.getConfigurations().matching(it -> it.getName().startsWith("spotless")).configureEach(conf -> {
58+
project.getDependencies().constraints(constraints -> {
59+
constraints.add(conf.getName(), "org.eclipse.jdt:org.eclipse.jdt.core:3.42.0", dependencyConstraint -> {
60+
dependencyConstraint.because(
61+
"We want to use a recent version of the Eclipse formatter libraries to support latest Java"
62+
);
63+
});
64+
constraints.add(conf.getName(), "org.eclipse.jdt:ecj:3.42.0", dependencyConstraint -> {
65+
dependencyConstraint.because(
66+
"We want to use a recent version of the Eclipse formatter libraries to support latest Java"
67+
);
68+
});
69+
});
70+
});
71+
5572
project.getExtensions().getByType(SpotlessExtension.class).java(java -> {
5673
File elasticsearchWorkspace = Util.locateElasticsearchWorkspace(project.getGradle());
5774
String importOrderPath = "build-conventions/elastic.importorder";
@@ -74,7 +91,7 @@ public void apply(Project project) {
7491
// When running build benchmarks we alter the source in some scenarios.
7592
// The gradle-profiler unfortunately does not generate compliant formatted
7693
// sources so we ignore that altered file when running build benchmarks
77-
if(Boolean.getBoolean("BUILD_PERFORMANCE_TEST") && project.getPath().equals(":server")) {
94+
if (Boolean.getBoolean("BUILD_PERFORMANCE_TEST") && project.getPath().equals(":server")) {
7895
java.targetExclude("src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java");
7996
}
8097
});

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,9 @@ private void registerInternalDistributionResolutions(List<DistributionResolution
113113
String versionProperty = System.getProperty("tests.bwc.main.version");
114114
// We use this phony version as a placeholder for the real version
115115
if (distribution.getVersion().equals("0.0.0")) {
116+
if (versionProperty == null) {
117+
throw new GradleException("System property 'tests.bwc.main.version' expected for building bwc version.");
118+
}
116119
BwcVersions.UnreleasedVersionInfo unreleasedVersionInfo = new BwcVersions.UnreleasedVersionInfo(
117120
Version.fromString(versionProperty),
118121
"main",

‎build-tools/src/main/java/org/elasticsearch/gradle/util/OsUtils.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ private OsUtils() {}
3838
* This method returns true if the given version of the JDK is known to be incompatible
3939
*/
4040
public static boolean jdkIsIncompatibleWithOS(Version version) {
41-
return version.onOrBefore("8.10.4") && isUbuntu2404OrLater();
41+
return version.after("0.0.0") && version.onOrBefore("8.10.4") && isUbuntu2404OrLater();
4242
}
4343

4444
private static boolean isUbuntu2404OrLater() {

‎build.gradle‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -338,18 +338,13 @@ allprojects {
338338
tasks.register('resolveAllDependencies', ResolveAllDependencies) {
339339
def ignoredPrefixes = [DistributionDownloadPlugin.ES_DISTRO_CONFIG_PREFIX, "jdbcDriver"]
340340
configs = project.configurations.matching { config -> ignoredPrefixes.any { config.name.startsWith(it) } == false }
341+
341342
if (project.path == ':') {
342343
resolveJavaToolChain = true
343-
344-
// ensure we have best possible caching of bwc builds
345-
dependsOn ":distribution:bwc:major1:buildBwcLinuxTar"
346-
dependsOn ":distribution:bwc:major2:buildBwcLinuxTar"
347-
dependsOn ":distribution:bwc:major3:buildBwcLinuxTar"
348-
dependsOn ":distribution:bwc:major4:buildBwcLinuxTar"
349-
dependsOn ":distribution:bwc:minor1:buildBwcLinuxTar"
350-
dependsOn ":distribution:bwc:minor2:buildBwcLinuxTar"
351-
dependsOn ":distribution:bwc:minor3:buildBwcLinuxTar"
352-
dependsOn ":distribution:bwc:minor4:buildBwcLinuxTar"
344+
}
345+
// ensure we have best possible caching of bwc builds
346+
if(project.path.startsWith(":distribution:bwc:")) {
347+
dependsOn project.tasks.matching { it.name == 'buildBwcLinuxTar' }
353348
}
354349
if (project.path.contains("fixture")) {
355350
dependsOn tasks.withType(ComposePull)

‎docs/changelog/133630.yaml‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎docs/changelog/134673.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 134673
2+
summary: Gracefully shutdown model deployment when node is removed from assignment
3+
routing
4+
area: Machine Learning
5+
type: bug
6+
issues: []

‎docs/changelog/134963.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 134963
2+
summary: Fix a bug in the GET _transform API that incorrectly claims some Transform configurations are missing
3+
area: Transform
4+
type: bug
5+
issues:
6+
- 134263

‎docs/changelog/135096.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 135096
2+
summary: Tolerate mixed types in datafeed stats sort
3+
area: Machine Learning
4+
type: bug
5+
issues: []

‎docs/changelog/135244.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 135244
2+
summary: Add reload listener to `SslProfile`
3+
area: TLS
4+
type: enhancement
5+
issues: []

0 commit comments

Comments
 (0)