Skip to content

Commit d82b288

Browse files
pabloempabloem
authored
Merge branch 'main' into pem-clamp-function
2 parents 10d640d + b7237d3 commit d82b288

File tree

530 files changed

+7434
-5503
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

530 files changed

+7434
-5503
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
import org.elasticsearch.index.IndexVersion;
5252
import org.elasticsearch.index.mapper.BlockLoader;
5353
import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
54+
import org.elasticsearch.index.mapper.IndexType;
5455
import org.elasticsearch.index.mapper.KeywordFieldMapper;
5556
import org.elasticsearch.index.mapper.MappedFieldType;
5657
import org.elasticsearch.index.mapper.NumberFieldMapper;
@@ -290,9 +291,8 @@ private static BlockLoader numericBlockLoader(WhereAndBaseName w, NumberFieldMap
290291
return new NumberFieldMapper.NumberFieldType(
291292
w.name,
292293
numberType,
293-
true,
294+
IndexType.points(true, docValues),
294295
stored,
295-
docValues,
296296
true,
297297
null,
298298
Map.of(),

‎benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/RecyclerBytesStreamOutputBenchmark.java‎

Lines changed: 5 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
package org.elasticsearch.benchmark.bytes;
1111

1212
import org.apache.lucene.util.BytesRef;
13+
import org.elasticsearch.benchmark.common.util.UTF8StringBytesBenchmark;
1314
import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput;
1415
import org.elasticsearch.common.recycler.Recycler;
1516
import org.openjdk.jmh.annotations.Benchmark;
@@ -65,10 +66,10 @@ public void initResults() throws IOException {
6566
// We use weights to generate certain sized UTF-8 characters and vInts. However, there is still some non-determinism which could
6667
// impact direct comparisons run-to-run
6768

68-
shortString = generateAsciiString(20);
69-
longString = generateAsciiString(100);
70-
nonAsciiString = generateUtf8String(200);
71-
veryLongString = generateAsciiString(800);
69+
shortString = UTF8StringBytesBenchmark.generateAsciiString(20);
70+
longString = UTF8StringBytesBenchmark.generateAsciiString(100);
71+
nonAsciiString = UTF8StringBytesBenchmark.generateUTF8String(200);
72+
veryLongString = UTF8StringBytesBenchmark.generateAsciiString(800);
7273
// vint values for benchmarking
7374
vints = new int[1000];
7475
for (int i = 0; i < vints.length; i++) {
@@ -143,49 +144,6 @@ public void writeVInt() throws IOException {
143144
}
144145
}
145146

146-
public static String generateAsciiString(int n) {
147-
ThreadLocalRandom random = ThreadLocalRandom.current();
148-
StringBuilder sb = new StringBuilder(n);
149-
150-
for (int i = 0; i < n; i++) {
151-
int ascii = random.nextInt(128);
152-
sb.append((char) ascii);
153-
}
154-
155-
return sb.toString();
156-
}
157-
158-
public static String generateUtf8String(int n) {
159-
ThreadLocalRandom random = ThreadLocalRandom.current();
160-
StringBuilder sb = new StringBuilder(n);
161-
162-
for (int i = 0; i < n; i++) {
163-
int codePoint;
164-
int probability = random.nextInt(100);
165-
166-
if (probability < 85) {
167-
// 1-byte UTF-8 (ASCII range)
168-
// 0x0000 to 0x007F
169-
codePoint = random.nextInt(0x0080);
170-
} else if (probability < 95) {
171-
// 2-byte UTF-8
172-
// 0x0080 to 0x07FF
173-
codePoint = random.nextInt(0x0080, 0x0800);
174-
} else {
175-
// 3-byte UTF-8
176-
// 0x0800 to 0xFFFF
177-
do {
178-
codePoint = random.nextInt(0x0800, 0x10000);
179-
// Skip surrogate pairs (0xD800-0xDFFF)
180-
} while (codePoint >= 0xD800 && codePoint <= 0xDFFF);
181-
}
182-
183-
sb.appendCodePoint(codePoint);
184-
}
185-
186-
return sb.toString();
187-
}
188-
189147
private record BenchmarkRecycler(AtomicReference<BytesRef> bytesRef) implements Recycler<BytesRef> {
190148

191149
@Override

‎benchmarks/src/main/java/org/elasticsearch/benchmark/common/util/UTF8StringBytesBenchmark.java‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.benchmark.common.util;
11+
12+
import org.apache.lucene.util.BytesRef;
13+
import org.apache.lucene.util.UnicodeUtil;
14+
import org.elasticsearch.common.UUIDs;
15+
import org.openjdk.jmh.annotations.Benchmark;
16+
import org.openjdk.jmh.annotations.BenchmarkMode;
17+
import org.openjdk.jmh.annotations.Fork;
18+
import org.openjdk.jmh.annotations.Measurement;
19+
import org.openjdk.jmh.annotations.Mode;
20+
import org.openjdk.jmh.annotations.OutputTimeUnit;
21+
import org.openjdk.jmh.annotations.Param;
22+
import org.openjdk.jmh.annotations.Scope;
23+
import org.openjdk.jmh.annotations.Setup;
24+
import org.openjdk.jmh.annotations.State;
25+
import org.openjdk.jmh.annotations.Warmup;
26+
27+
import java.nio.ByteBuffer;
28+
import java.nio.charset.StandardCharsets;
29+
import java.util.concurrent.ThreadLocalRandom;
30+
import java.util.concurrent.TimeUnit;
31+
32+
@Warmup(iterations = 3)
33+
@Measurement(iterations = 3)
34+
@BenchmarkMode(Mode.AverageTime)
35+
@OutputTimeUnit(TimeUnit.NANOSECONDS)
36+
@Fork(value = 1)
37+
public class UTF8StringBytesBenchmark {
38+
39+
@State(Scope.Thread)
40+
public static class StringState {
41+
@Param({ "uuid", "short", "long", "nonAscii", "veryLong" })
42+
String stringType;
43+
44+
String string;
45+
BytesRef bytes;
46+
47+
@Setup
48+
public void setup() {
49+
string = switch (stringType) {
50+
case "uuid" -> UUIDs.base64UUID();
51+
case "short" -> generateAsciiString(20);
52+
case "long" -> generateAsciiString(100);
53+
case "nonAscii" -> generateUTF8String(200);
54+
case "veryLong" -> generateAsciiString(1000);
55+
default -> throw new IllegalArgumentException("Unknown stringType: " + stringType);
56+
};
57+
bytes = getBytes(string);
58+
}
59+
}
60+
61+
@Benchmark
62+
public BytesRef getBytesJDK(StringState state) {
63+
byte[] bytes = state.string.getBytes(StandardCharsets.UTF_8);
64+
return new BytesRef(bytes, 0, bytes.length);
65+
}
66+
67+
@Benchmark
68+
public BytesRef getBytesUnicodeUtils(StringState state) {
69+
String string = state.string;
70+
int length = string.length();
71+
int size = UnicodeUtil.calcUTF16toUTF8Length(string, 0, length);
72+
byte[] out = new byte[size];
73+
UnicodeUtil.UTF16toUTF8(string, 0, length, out, 0);
74+
return new BytesRef(out, 0, out.length);
75+
}
76+
77+
@Benchmark
78+
public BytesRef getBytesByteBufferEncoder(StringState state) {
79+
var byteBuff = StandardCharsets.UTF_8.encode(state.string);
80+
assert byteBuff.hasArray();
81+
return new BytesRef(byteBuff.array(), byteBuff.arrayOffset() + byteBuff.position(), byteBuff.remaining());
82+
}
83+
84+
@Benchmark
85+
public String getStringJDK(StringState state) {
86+
BytesRef bytes = state.bytes;
87+
return new String(bytes.bytes, bytes.offset, bytes.length, StandardCharsets.UTF_8);
88+
}
89+
90+
@Benchmark
91+
public String getStringByteBufferDecoder(StringState state) {
92+
BytesRef bytes = state.bytes;
93+
var byteBuff = ByteBuffer.wrap(bytes.bytes, bytes.offset, bytes.length);
94+
return StandardCharsets.UTF_8.decode(byteBuff).toString();
95+
}
96+
97+
private static BytesRef getBytes(String string) {
98+
int before = ThreadLocalRandom.current().nextInt(0, 50);
99+
int after = ThreadLocalRandom.current().nextInt(0, 50);
100+
byte[] stringBytes = string.getBytes(StandardCharsets.UTF_8);
101+
byte[] finalBytes = new byte[before + after + stringBytes.length];
102+
System.arraycopy(stringBytes, 0, finalBytes, before, stringBytes.length);
103+
return new BytesRef(finalBytes, before, stringBytes.length);
104+
}
105+
106+
public static String generateAsciiString(int n) {
107+
ThreadLocalRandom random = ThreadLocalRandom.current();
108+
StringBuilder sb = new StringBuilder(n);
109+
110+
for (int i = 0; i < n; i++) {
111+
int ascii = random.nextInt(128);
112+
sb.append((char) ascii);
113+
}
114+
115+
return sb.toString();
116+
}
117+
118+
public static String generateUTF8String(int n) {
119+
ThreadLocalRandom random = ThreadLocalRandom.current();
120+
StringBuilder sb = new StringBuilder(n);
121+
122+
for (int i = 0; i < n; i++) {
123+
int codePoint;
124+
int probability = random.nextInt(100);
125+
126+
if (probability < 85) {
127+
// 1-byte UTF-8 (ASCII range)
128+
// 0x0000 to 0x007F
129+
codePoint = random.nextInt(0x0080);
130+
} else if (probability < 95) {
131+
// 2-byte UTF-8
132+
// 0x0080 to 0x07FF
133+
codePoint = random.nextInt(0x0080, 0x0800);
134+
} else {
135+
// 3-byte UTF-8
136+
// 0x0800 to 0xFFFF
137+
do {
138+
codePoint = random.nextInt(0x0800, 0x10000);
139+
// Skip surrogate pairs (0xD800-0xDFFF)
140+
} while (codePoint >= 0xD800 && codePoint <= 0xDFFF);
141+
}
142+
143+
sb.appendCodePoint(codePoint);
144+
}
145+
146+
return sb.toString();
147+
}
148+
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.elasticsearch.index.fielddata.FieldDataContext;
3030
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
3131
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
32+
import org.elasticsearch.index.mapper.IndexType;
3233
import org.elasticsearch.index.mapper.MappedFieldType;
3334
import org.elasticsearch.index.mapper.MappingLookup;
3435
import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType;
@@ -85,7 +86,23 @@ public class ScriptScoreBenchmark {
8586
private final ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, pluginsService.filterPlugins(ScriptPlugin.class).toList());
8687

8788
private final Map<String, MappedFieldType> fieldTypes = Map.ofEntries(
88-
Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of(), null, false, null, null, false))
89+
Map.entry(
90+
"n",
91+
new NumberFieldType(
92+
"n",
93+
NumberType.LONG,
94+
IndexType.docValuesOnly(),
95+
false,
96+
true,
97+
null,
98+
Map.of(),
99+
null,
100+
false,
101+
null,
102+
null,
103+
false
104+
)
105+
)
89106
);
90107
private final IndexFieldDataCache fieldDataCache = new IndexFieldDataCache.None();
91108
private final CircuitBreakerService breakerService = new NoneCircuitBreakerService();

‎build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1407,7 +1407,7 @@ private void createConfiguration() {
14071407
// Limit the number of allocated processors for all nodes in the cluster by default.
14081408
// This is to ensure that the tests run consistently across different environments.
14091409
String processorCount = shouldConfigureTestClustersWithOneProcessor() ? "1" : "2";
1410-
if (getVersion().onOrAfter("7.6.0")) {
1410+
if (getVersion().onOrAfter("7.4.0")) {
14111411
baseConfig.put("node.processors", processorCount);
14121412
} else {
14131413
baseConfig.put("processors", processorCount);

‎distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
# Extract Elasticsearch artifact
2626
################################################################################
2727
28-
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:1c77fc945beb2ce254d83931ca15ef770c8c8fec7fb8750d9f8a9b94f7ce71b7 AS builder
28+
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:a380543ec0c09cf294408e0f42c653ee416d3c38e841cbd7267684b6aae51715 AS builder
2929
3030
# Install required packages to extract the Elasticsearch distribution
3131
RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -104,7 +104,7 @@ WORKDIR /usr/share/elasticsearch/config
104104
# Add entrypoint
105105
################################################################################
106106

107-
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:1c77fc945beb2ce254d83931ca15ef770c8c8fec7fb8750d9f8a9b94f7ce71b7
107+
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:a380543ec0c09cf294408e0f42c653ee416d3c38e841cbd7267684b6aae51715
108108

109109
RUN <%= retry.loop(package_manager,
110110
"export DEBIAN_FRONTEND=noninteractive && \n" +

‎distribution/docker/src/docker/dockerfiles/wolfi/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
# Extract Elasticsearch artifact
2626
################################################################################
2727
28-
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:00fa283d3647165f65fa64324d60f09d29f0ad1ade717af349131b2fc65f8f8f AS builder
28+
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:b08320d6d7e9eb8054e8781a3b624f8e77e90d4aeb1756fd7665bb2f96f36fc8 AS builder
2929
3030
# Install required packages to extract the Elasticsearch distribution
3131
RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -80,7 +80,7 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
8080
# Add entrypoint
8181
################################################################################
8282

83-
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:00fa283d3647165f65fa64324d60f09d29f0ad1ade717af349131b2fc65f8f8f
83+
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:b08320d6d7e9eb8054e8781a3b624f8e77e90d4aeb1756fd7665bb2f96f36fc8
8484

8585
RUN <%= retry.loop(package_manager,
8686
"export DEBIAN_FRONTEND=noninteractive && \n" +

‎docs/changelog/127053.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 127053
2+
summary: Add missing fields to the Logstash Stack Monitoring Template
3+
area: Monitoring
4+
type: enhancement
5+
issues:
6+
- 125499

‎docs/changelog/136086.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 136086
2+
summary: Delay automaton creation in `BinaryDvConfirmedQuery` to avoid OOM on queries
3+
against `WildCard` fields
4+
area: Search
5+
type: bug
6+
issues: []

‎docs/changelog/136103.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 136103
2+
summary: Enable the TEXT_EMBEDDING function in non-snapshot build
3+
area: ES|QL
4+
type: feature
5+
issues: []

0 commit comments

Comments
 (0)