Skip to content

Commit 270eb6b

Browse files
ivanceaivancea
committed
Merge branch 'main' into esql-top-n-agg-grouping
2 parents c778b3b + c72d00f commit 270eb6b

File tree

657 files changed

+20371
-9715
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

657 files changed

+20371
-9715
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OptimizedScalarQuantizerBenchmark.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.benchmark.vector;
11+
12+
import org.apache.lucene.index.VectorSimilarityFunction;
13+
import org.elasticsearch.common.logging.LogConfigurator;
14+
import org.elasticsearch.index.codec.vectors.OptimizedScalarQuantizer;
15+
import org.openjdk.jmh.annotations.Benchmark;
16+
import org.openjdk.jmh.annotations.BenchmarkMode;
17+
import org.openjdk.jmh.annotations.Fork;
18+
import org.openjdk.jmh.annotations.Level;
19+
import org.openjdk.jmh.annotations.Measurement;
20+
import org.openjdk.jmh.annotations.Mode;
21+
import org.openjdk.jmh.annotations.OutputTimeUnit;
22+
import org.openjdk.jmh.annotations.Param;
23+
import org.openjdk.jmh.annotations.Scope;
24+
import org.openjdk.jmh.annotations.Setup;
25+
import org.openjdk.jmh.annotations.State;
26+
import org.openjdk.jmh.annotations.Warmup;
27+
28+
import java.util.concurrent.ThreadLocalRandom;
29+
import java.util.concurrent.TimeUnit;
30+
31+
@BenchmarkMode(Mode.Throughput)
32+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
33+
@State(Scope.Benchmark)
34+
@Warmup(iterations = 3, time = 1)
35+
@Measurement(iterations = 5, time = 1)
36+
@Fork(value = 3)
37+
public class OptimizedScalarQuantizerBenchmark {
38+
static {
39+
LogConfigurator.configureESLogging(); // native access requires logging to be initialized
40+
}
41+
@Param({ "384", "702", "1024" })
42+
int dims;
43+
44+
float[] vector;
45+
float[] centroid;
46+
byte[] destination;
47+
48+
@Param({ "1", "4", "7" })
49+
byte bits;
50+
51+
OptimizedScalarQuantizer osq = new OptimizedScalarQuantizer(VectorSimilarityFunction.DOT_PRODUCT);
52+
53+
@Setup(Level.Iteration)
54+
public void init() {
55+
ThreadLocalRandom random = ThreadLocalRandom.current();
56+
// random byte arrays for binary methods
57+
destination = new byte[dims];
58+
vector = new float[dims];
59+
centroid = new float[dims];
60+
for (int i = 0; i < dims; ++i) {
61+
vector[i] = random.nextFloat();
62+
centroid[i] = random.nextFloat();
63+
}
64+
}
65+
66+
@Benchmark
67+
public byte[] scalar() {
68+
osq.scalarQuantize(vector, destination, bits, centroid);
69+
return destination;
70+
}
71+
72+
@Benchmark
73+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
74+
public byte[] vector() {
75+
osq.scalarQuantize(vector, destination, bits, centroid);
76+
return destination;
77+
}
78+
}

‎build-tools-internal/version.properties‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ jna = 5.12.1
1717
netty = 4.1.118.Final
1818
commons_lang3 = 3.9
1919
google_oauth_client = 1.34.1
20-
awsv1sdk = 1.12.746
2120
awsv2sdk = 2.30.38
2221
reactive_streams = 1.0.4
2322

‎distribution/docker/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -590,7 +590,7 @@ for (final Architecture architecture : Architecture.values()) {
590590
}
591591
if(base == DockerBase.DEFAULT) {
592592
// Add additional docker hub specific context which we use solely for publishing to docker hub.
593-
// At the moment it only differs in not labels added that we need for openshift certification
593+
// At the moment it is exactly the same as the default context.
594594
addBuildDockerContextTask(architecture, base, 'DockerHubContext', "docker-hub-build-context")
595595
}
596596
}

‎distribution/docker/src/docker/Dockerfile.default‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -139,15 +139,13 @@ LABEL org.label-schema.build-date="${build_date}" \\
139139
org.opencontainers.image.vendor="Elastic" \\
140140
org.opencontainers.image.version="${version}"
141141

142-
<% if (docker_context != 'docker-hub-build-context') { %>
143142
LABEL name="Elasticsearch" \\
144143
maintainer="[email protected]" \\
145144
vendor="Elastic" \\
146145
version="${version}" \\
147146
release="1" \\
148147
summary="Elasticsearch" \\
149148
description="You know, for search."
150-
<% } %>
151149

152150
RUN mkdir /licenses && ln LICENSE.txt /licenses/LICENSE
153151

‎distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,13 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
6161
"-Dio.netty.noUnsafe=true",
6262
"-Dio.netty.noKeySetOptimization=true",
6363
"-Dio.netty.recycler.maxCapacityPerThread=0",
64+
// temporary until we get off-heap vector stats in Lucene 10.3
65+
"--add-opens=org.apache.lucene.core/org.apache.lucene.codecs.lucene99=org.elasticsearch.server",
66+
"--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene90=org.elasticsearch.server",
67+
"--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene91=org.elasticsearch.server",
68+
"--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene92=org.elasticsearch.server",
69+
"--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene94=org.elasticsearch.server",
70+
"--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene95=org.elasticsearch.server",
6471
// log4j 2
6572
"-Dlog4j.shutdownHookEnabled=false",
6673
"-Dlog4j2.disable.jmx=true",

‎docs/changelog/125570.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 125570
2+
summary: ES|QL random sampling
3+
area: Machine Learning
4+
type: feature
5+
issues: []

‎docs/changelog/126598.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 126598
2+
summary: "ESQL: Retain aggregate when grouping"
3+
area: ES|QL
4+
type: bug
5+
issues:
6+
- 126026

‎docs/changelog/126704.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 126704
2+
summary: Add dense vector off-heap stats to Node stats and Index stats APIs
3+
area: "Vector Search"
4+
type: enhancement
5+
issues: []

‎docs/changelog/126843.yaml‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
pr: 126843
2+
summary: Upgrade `repository-s3` to AWS SDK v2
3+
area: Snapshot/Restore
4+
type: breaking
5+
issues:
6+
- 120993
7+
highlight:
8+
title: Upgrade `repository-s3` to AWS SDK v2
9+
body: >-
10+
In earlier versions of {es} the `repository-s3` plugin was based on the AWS
11+
SDK v1. AWS will withdraw support for this SDK before the end of the life
12+
of {es} {minor-version} so we have migrated this plugin to the newer AWS SDK v2.
13+
14+
The two SDKs are not quite compatible, so please check the breaking changes
15+
documentation and test the new version thoroughly before upgrading any
16+
production workloads.
17+
notable: true
18+
breaking:
19+
title: Upgrade `repository-s3` to AWS SDK v2
20+
area: Cluster and node setting
21+
details: >-
22+
In earlier versions of {es} the `repository-s3` plugin was based on the AWS
23+
SDK v1. AWS will withdraw support for this SDK before the end of the life
24+
of {es} {minor-version} so we must migrate to the newer AWS SDK v2.
25+
26+
Unfortunately there are several differences between the two AWS SDK
27+
versions which may require you to adjust your system configuration when
28+
upgrading to {es} {minor-version} or later. These differences include, but
29+
may not be limited to, the following items.
30+
31+
* AWS SDK v2 requires users to specify the region to use for signing
32+
requests, or else to run in an environment in which it can determine the
33+
correct region automatically. The older SDK would try to determine the
34+
region based on the endpoint URL as specified with the
35+
`s3.client.${CLIENT_NAME}.endpoint` setting, together with other data
36+
drawn from the operating environment, and would ultimately fall back to
37+
`us-east-1` if no better value could be found.
38+
39+
* AWS SDK v2 does not support the EC2 IMDSv1 protocol.
40+
41+
* AWS SDK v2 does not support the
42+
`com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system property.
43+
44+
* AWS SDK v2 does not permit specifying a choice between HTTP and HTTPS so
45+
the `s3.client.${CLIENT_NAME}.protocol` setting is deprecated and no longer
46+
has any effect.
47+
48+
* AWS SDK v2 does not permit control over throttling for retries, so the
49+
the `s3.client.${CLIENT_NAME}.use_throttle_retries` setting is deprecated
50+
and no longer has any effect.
51+
52+
* AWS SDK v2 requires the use of the V4 signature algorithm, so the
53+
`s3.client.${CLIENT_NAME}.signer_override` setting is deprecated and no
54+
longer has any effect.
55+
56+
* AWS SDK v2 does not support the `log-delivery-write` canned ACL.
57+
58+
* AWS SDK v2 counts 4xx responses differently in its metrics reporting.
59+
60+
* AWS SDK v2 always uses the regional STS endpoint, whereas AWS SDK v2
61+
could use either a regional endpoint or the global
62+
`https://sts.amazonaws.com` one.
63+
64+
impact: >-
65+
If you use the `repository-s3` module, test your upgrade thoroughly before
66+
upgrading any production workloads.
67+
68+
Adapt your configuration to the new SDK functionality. This includes, but
69+
may not be limited to, the following items.
70+
71+
* Specify the correct signing region using the
72+
`s3.client.${CLIENT_NAME}.region` setting on each node. {es} will try and
73+
determine the correct region based on the endpoint URL and other data
74+
drawn from the operating environment but cannot guarantee to do so
75+
correctly in all cases.
76+
77+
* If you use IMDS to determine the availability zone of a node or to obtain
78+
credentials for accessing the EC2 API, ensure that it supports the IMDSv2
79+
protocol.
80+
81+
* If applicable, discontinue use of the
82+
`com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system property.
83+
84+
* If applicable, specify that you wish to use the insecure HTTP protocol to
85+
access the S3 API by setting `s3.client.${CLIENT_NAME}.endpoint` to a URL
86+
which starts with `http://`.
87+
88+
* If applicable, discontinue use of the `log-delivery-write` canned ACL.
89+
90+
notable: true

‎docs/changelog/126884.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 126884
2+
summary: Rare terms aggregation false **positive** fix
3+
area: Aggregations
4+
type: bug
5+
issues: []

0 commit comments

Comments
 (0)