Skip to content

Commit 39ad594

Browse files
ywangdywangd
committed
Merge remote-tracking branch 'origin/main' into shard-snapshot-context-refactor
2 parents 5e59730 + ef9e390 commit 39ad594

File tree

323 files changed

+14658
-8165
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

323 files changed

+14658
-8165
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/DistanceBulkBenchmark.java‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
package org.elasticsearch.benchmark.vector;
10+
11+
import org.apache.lucene.util.VectorUtil;
12+
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
13+
import org.elasticsearch.common.logging.LogConfigurator;
14+
import org.elasticsearch.simdvec.ESVectorUtil;
15+
import org.openjdk.jmh.annotations.Benchmark;
16+
import org.openjdk.jmh.annotations.BenchmarkMode;
17+
import org.openjdk.jmh.annotations.Fork;
18+
import org.openjdk.jmh.annotations.Measurement;
19+
import org.openjdk.jmh.annotations.Mode;
20+
import org.openjdk.jmh.annotations.OutputTimeUnit;
21+
import org.openjdk.jmh.annotations.Param;
22+
import org.openjdk.jmh.annotations.Scope;
23+
import org.openjdk.jmh.annotations.Setup;
24+
import org.openjdk.jmh.annotations.State;
25+
import org.openjdk.jmh.annotations.Warmup;
26+
import org.openjdk.jmh.infra.Blackhole;
27+
28+
import java.io.IOException;
29+
import java.util.Random;
30+
import java.util.concurrent.TimeUnit;
31+
32+
@BenchmarkMode(Mode.Throughput)
33+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
34+
@State(Scope.Benchmark)
35+
// first iteration is complete garbage, so make sure we really warmup
36+
@Warmup(iterations = 4, time = 1)
37+
// real iterations. not useful to spend tons of time here, better to fork more
38+
@Measurement(iterations = 5, time = 1)
39+
// engage some noise reduction
40+
@Fork(value = 1)
41+
public class DistanceBulkBenchmark {
42+
43+
static {
44+
LogConfigurator.configureESLogging(); // native access requires logging to be initialized
45+
}
46+
47+
@Param({ "384", "782", "1024" })
48+
int dims;
49+
50+
int length;
51+
52+
int numVectors = 4 * 100;
53+
int numQueries = 10;
54+
55+
float[][] vectors;
56+
float[][] queries;
57+
float[] distances = new float[4];
58+
59+
@Setup
60+
public void setup() throws IOException {
61+
Random random = new Random(123);
62+
63+
this.length = OptimizedScalarQuantizer.discretize(dims, 64) / 8;
64+
65+
vectors = new float[numVectors][dims];
66+
for (float[] vector : vectors) {
67+
for (int i = 0; i < dims; i++) {
68+
vector[i] = random.nextFloat();
69+
}
70+
}
71+
72+
queries = new float[numQueries][dims];
73+
for (float[] query : queries) {
74+
for (int i = 0; i < dims; i++) {
75+
query[i] = random.nextFloat();
76+
}
77+
}
78+
}
79+
80+
@Benchmark
81+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
82+
public void squareDistance(Blackhole bh) {
83+
for (int j = 0; j < numQueries; j++) {
84+
float[] query = queries[j];
85+
for (int i = 0; i < numVectors; i++) {
86+
float[] vector = vectors[i];
87+
float distance = VectorUtil.squareDistance(query, vector);
88+
bh.consume(distance);
89+
}
90+
}
91+
}
92+
93+
@Benchmark
94+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
95+
public void soarDistance(Blackhole bh) {
96+
for (int j = 0; j < numQueries; j++) {
97+
float[] query = queries[j];
98+
for (int i = 0; i < numVectors; i++) {
99+
float[] vector = vectors[i];
100+
float distance = ESVectorUtil.soarDistance(query, vector, vector, 1.0f, 1.0f);
101+
bh.consume(distance);
102+
}
103+
}
104+
}
105+
106+
@Benchmark
107+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
108+
public void squareDistanceBulk(Blackhole bh) {
109+
for (int j = 0; j < numQueries; j++) {
110+
float[] query = queries[j];
111+
for (int i = 0; i < numVectors; i += 4) {
112+
ESVectorUtil.squareDistanceBulk(query, vectors[i], vectors[i + 1], vectors[i + 2], vectors[i + 3], distances);
113+
for (float distance : distances) {
114+
bh.consume(distance);
115+
}
116+
117+
}
118+
}
119+
}
120+
121+
@Benchmark
122+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
123+
public void soarDistanceBulk(Blackhole bh) {
124+
for (int j = 0; j < numQueries; j++) {
125+
float[] query = queries[j];
126+
for (int i = 0; i < numVectors; i += 4) {
127+
ESVectorUtil.soarDistanceBulk(
128+
query,
129+
vectors[i],
130+
vectors[i + 1],
131+
vectors[i + 2],
132+
vectors[i + 3],
133+
vectors[i],
134+
1.0f,
135+
1.0f,
136+
distances
137+
);
138+
for (float distance : distances) {
139+
bh.consume(distance);
140+
}
141+
142+
}
143+
}
144+
}
145+
}

‎benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/JsonParserBenchmark.java‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.benchmark.xcontent;
11+
12+
import org.elasticsearch.common.bytes.BytesReference;
13+
import org.elasticsearch.common.io.Streams;
14+
import org.elasticsearch.xcontent.XContentParser;
15+
import org.elasticsearch.xcontent.XContentParserConfiguration;
16+
import org.elasticsearch.xcontent.XContentType;
17+
import org.openjdk.jmh.annotations.Benchmark;
18+
import org.openjdk.jmh.annotations.BenchmarkMode;
19+
import org.openjdk.jmh.annotations.Fork;
20+
import org.openjdk.jmh.annotations.Level;
21+
import org.openjdk.jmh.annotations.Measurement;
22+
import org.openjdk.jmh.annotations.Mode;
23+
import org.openjdk.jmh.annotations.OutputTimeUnit;
24+
import org.openjdk.jmh.annotations.Scope;
25+
import org.openjdk.jmh.annotations.Setup;
26+
import org.openjdk.jmh.annotations.State;
27+
import org.openjdk.jmh.annotations.Warmup;
28+
import org.openjdk.jmh.infra.Blackhole;
29+
30+
import java.io.IOException;
31+
import java.util.Arrays;
32+
import java.util.Collections;
33+
import java.util.List;
34+
import java.util.Map;
35+
import java.util.Random;
36+
import java.util.concurrent.TimeUnit;
37+
import java.util.stream.Collectors;
38+
39+
@Fork(1)
40+
@Warmup(iterations = 5)
41+
@Measurement(iterations = 10)
42+
@BenchmarkMode(Mode.AverageTime)
43+
@OutputTimeUnit(TimeUnit.NANOSECONDS)
44+
@State(Scope.Thread)
45+
public class JsonParserBenchmark {
46+
private Map<String, BytesReference> sourceBytes;
47+
private BytesReference source;
48+
private Random random;
49+
private List<String> sourcesRandomized;
50+
51+
final String[] sources = new String[] { "monitor_cluster_stats.json", "monitor_index_stats.json", "monitor_node_stats.json" };
52+
53+
@Setup(Level.Iteration)
54+
public void randomizeSource() {
55+
sourcesRandomized = Arrays.asList(sources);
56+
Collections.shuffle(sourcesRandomized, random);
57+
}
58+
59+
@Setup(Level.Trial)
60+
public void setup() throws IOException {
61+
random = new Random();
62+
sourceBytes = Arrays.stream(sources).collect(Collectors.toMap(s -> s, s -> {
63+
try {
64+
return readSource(s);
65+
} catch (IOException e) {
66+
throw new RuntimeException(e);
67+
}
68+
}));
69+
}
70+
71+
@Benchmark
72+
public void parseJson(Blackhole bh) throws IOException {
73+
sourcesRandomized.forEach(source -> {
74+
try {
75+
final XContentParser parser = XContentType.JSON.xContent()
76+
.createParser(XContentParserConfiguration.EMPTY, sourceBytes.get(source).streamInput());
77+
bh.consume(parser.mapOrdered());
78+
} catch (IOException e) {
79+
throw new RuntimeException(e);
80+
}
81+
});
82+
}
83+
84+
private BytesReference readSource(String fileName) throws IOException {
85+
return Streams.readFully(JsonParserBenchmark.class.getResourceAsStream(fileName));
86+
}
87+
}

‎distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
# Extract Elasticsearch artifact
2525
################################################################################
2626
27-
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:ea864ffab2b828ec157b2480c8b8dbd6f27da682c59a93d4b75a46163922b237 AS builder
27+
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:4a8fe3e7390fcf11a88b216b4608f1db322c327e801898386808551ebbdc7523 AS builder
2828
2929
# Install required packages to extract the Elasticsearch distribution
3030
RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -103,7 +103,7 @@ WORKDIR /usr/share/elasticsearch/config
103103
# Add entrypoint
104104
################################################################################
105105

106-
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:ea864ffab2b828ec157b2480c8b8dbd6f27da682c59a93d4b75a46163922b237
106+
FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:4a8fe3e7390fcf11a88b216b4608f1db322c327e801898386808551ebbdc7523
107107

108108
RUN <%= retry.loop(package_manager,
109109
"export DEBIAN_FRONTEND=noninteractive && \n" +

‎distribution/docker/src/docker/dockerfiles/wolfi/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
# Extract Elasticsearch artifact
2525
################################################################################
2626
27-
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:9ded4d2364e7f263cada56b0b9ca3ef643e8dac958a79df3d18c2a9f0a33fbc7 AS builder
27+
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:442a5663000b3d66d565e61d400b30a4638383a72d90494cfc3104b34dfb3211 AS builder
2828
2929
# Install required packages to extract the Elasticsearch distribution
3030
RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -79,7 +79,7 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
7979
# Add entrypoint
8080
################################################################################
8181

82-
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:9ded4d2364e7f263cada56b0b9ca3ef643e8dac958a79df3d18c2a9f0a33fbc7
82+
FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:442a5663000b3d66d565e61d400b30a4638383a72d90494cfc3104b34dfb3211
8383

8484
RUN <%= retry.loop(package_manager,
8585
"export DEBIAN_FRONTEND=noninteractive && \n" +

‎docs/changelog/131341.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 131341
2+
summary: Consider min/max from predicates when transform date_trunc/bucket to `round_to`
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎docs/changelog/131536.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
pr: 131536
2-
summary: "Component Templates: Add `{created,modified}_date`"
2+
summary: "Component Templates: Add created and modified date"
33
area: Ingest Node
44
type: enhancement
55
issues: []

‎docs/changelog/132083.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 132083
2+
summary: "Index template: Add created_date and modified_date"
3+
area: Ingest Node
4+
type: enhancement
5+
issues: []

‎docs/changelog/132138.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 132138
2+
summary: Fix lookup index resolution when field-caps returns empty mapping
3+
area: ES|QL
4+
type: bug
5+
issues:
6+
- 132105

‎docs/changelog/132143.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 132143
2+
summary: Consider min/max from predicates when transform date_trunc/bucket to `round_to`
3+
option 2
4+
area: ES|QL
5+
type: enhancement
6+
issues: []

‎docs/changelog/132167.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 132167
2+
summary: Deal with internally created IN in a different way for EQL
3+
area: EQL
4+
type: bug
5+
issues:
6+
- 118621

0 commit comments

Comments
 (0)