Merge branch 'main' into 2025/04/23/netty-autoread-reapply

Author: DaveCTurner

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OptimizedScalarQuantizerBenchmark.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.vector;
+
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.index.codec.vectors.OptimizedScalarQuantizer;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(value = 3)
+public class OptimizedScalarQuantizerBenchmark {
+    static {
+        LogConfigurator.configureESLogging(); // native access requires logging to be initialized
+    }
+    @Param({ "384", "702", "1024" })
+    int dims;
+
+    float[] vector;
+    float[] centroid;
+    byte[] destination;
+
+    @Param({ "1", "4", "7" })
+    byte bits;
+
+    OptimizedScalarQuantizer osq = new OptimizedScalarQuantizer(VectorSimilarityFunction.DOT_PRODUCT);
+
+    @Setup(Level.Iteration)
+    public void init() {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+        // random byte arrays for binary methods
+        destination = new byte[dims];
+        vector = new float[dims];
+        centroid = new float[dims];
+        for (int i = 0; i < dims; ++i) {
+            vector[i] = random.nextFloat();
+            centroid[i] = random.nextFloat();
+        }
+    }
+
+    @Benchmark
+    public byte[] scalar() {
+        osq.scalarQuantize(vector, destination, bits, centroid);
+        return destination;
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public byte[] vector() {
+        osq.scalarQuantize(vector, destination, bits, centroid);
+        return destination;
+    }
+}

build-tools-internal/version.properties

@@ -17,7 +17,6 @@ jna               = 5.12.1
 netty             = 4.1.118.Final
 commons_lang3     = 3.9
 google_oauth_client = 1.34.1
-awsv1sdk            = 1.12.746
 awsv2sdk            = 2.30.38
 reactive_streams    = 1.0.4
 

distribution/docker/build.gradle

@@ -590,7 +590,7 @@ for (final Architecture architecture : Architecture.values()) {
     }
     if(base == DockerBase.DEFAULT) {
       // Add additional docker hub specific context which we use solely for publishing to docker hub.
-      // At the moment it only differs in not labels added that we need for openshift certification
+      // At the moment it is exactly the same as the default context.
       addBuildDockerContextTask(architecture, base, 'DockerHubContext', "docker-hub-build-context")
     }
   }

distribution/docker/src/docker/Dockerfile.default

@@ -139,15 +139,13 @@ LABEL org.label-schema.build-date="${build_date}" \\
   org.opencontainers.image.vendor="Elastic" \\
   org.opencontainers.image.version="${version}"
 
-<% if (docker_context != 'docker-hub-build-context') { %>
 LABEL name="Elasticsearch" \\
   maintainer="[email protected]" \\
   vendor="Elastic" \\
   version="${version}" \\
   release="1" \\
   summary="Elasticsearch" \\
   description="You know, for search."
-<% } %>
 
 RUN mkdir /licenses && ln LICENSE.txt /licenses/LICENSE
 

docs/changelog/125570.yaml

@@ -0,0 +1,5 @@
+pr: 125570
+summary: ES|QL random sampling
+area: Machine Learning
+type: feature
+issues: []

docs/changelog/126843.yaml

@@ -0,0 +1,90 @@
+pr: 126843
+summary: Upgrade `repository-s3` to AWS SDK v2
+area: Snapshot/Restore
+type: breaking
+issues:
+ - 120993
+highlight:
+  title: Upgrade `repository-s3` to AWS SDK v2
+  body: >-
+    In earlier versions of {es} the `repository-s3` plugin was based on the AWS
+    SDK v1. AWS will withdraw support for this SDK before the end of the life
+    of {es} {minor-version} so we have migrated this plugin to the newer AWS SDK v2.
+
+    The two SDKs are not quite compatible, so please check the breaking changes
+    documentation and test the new version thoroughly before upgrading any
+    production workloads.
+  notable: true
+breaking:
+  title: Upgrade `repository-s3` to AWS SDK v2
+  area: Cluster and node setting
+  details: >-
+    In earlier versions of {es} the `repository-s3` plugin was based on the AWS
+    SDK v1. AWS will withdraw support for this SDK before the end of the life
+    of {es} {minor-version} so we must migrate to the newer AWS SDK v2.
+
+    Unfortunately there are several differences between the two AWS SDK
+    versions which may require you to adjust your system configuration when
+    upgrading to {es} {minor-version} or later. These differences include, but
+    may not be limited to, the following items.
+
+    * AWS SDK v2 requires users to specify the region to use for signing
+      requests, or else to run in an environment in which it can determine the
+      correct region automatically. The older SDK would try to determine the
+      region based on the endpoint URL as specified with the
+      `s3.client.${CLIENT_NAME}.endpoint` setting, together with other data
+      drawn from the operating environment, and would ultimately fall back to
+      `us-east-1` if no better value could be found.
+
+    * AWS SDK v2 does not support the EC2 IMDSv1 protocol.
+
+    * AWS SDK v2 does not support the
+      `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system property.
+
+    * AWS SDK v2 does not permit specifying a choice between HTTP and HTTPS so
+      the `s3.client.${CLIENT_NAME}.protocol` setting is deprecated and no longer
+      has any effect.
+
+    * AWS SDK v2 does not permit control over throttling for retries, so the
+      the `s3.client.${CLIENT_NAME}.use_throttle_retries` setting is deprecated
+      and no longer has any effect.
+
+    * AWS SDK v2 requires the use of the V4 signature algorithm, so the
+      `s3.client.${CLIENT_NAME}.signer_override` setting is deprecated and no
+      longer has any effect.
+
+    * AWS SDK v2 does not support the `log-delivery-write` canned ACL.
+
+    * AWS SDK v2 counts 4xx responses differently in its metrics reporting.
+
+    * AWS SDK v2 always uses the regional STS endpoint, whereas AWS SDK v2
+      could use either a regional endpoint or the global
+      `https://sts.amazonaws.com` one.
+
+  impact: >-
+    If you use the `repository-s3` module, test your upgrade thoroughly before
+    upgrading any production workloads.
+
+    Adapt your configuration to the new SDK functionality. This includes, but
+    may not be limited to, the following items.
+
+    * Specify the correct signing region using the
+      `s3.client.${CLIENT_NAME}.region` setting on each node. {es} will try and
+      determine the correct region based on the endpoint URL and other data
+      drawn from the operating environment but cannot guarantee to do so
+      correctly in all cases.
+
+    * If you use IMDS to determine the availability zone of a node or to obtain
+      credentials for accessing the EC2 API, ensure that it supports the IMDSv2
+      protocol.
+
+    * If applicable, discontinue use of the
+      `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system property.
+
+    * If applicable, specify that you wish to use the insecure HTTP protocol to
+      access the S3 API by setting `s3.client.${CLIENT_NAME}.endpoint` to a URL
+      which starts with `http://`.
+
+    * If applicable, discontinue use of the `log-delivery-write` canned ACL.
+
+  notable: true

docs/changelog/126990.yaml

@@ -0,0 +1,7 @@
+pr: 126990
+summary: "Fix: consider case sensitiveness differences in Windows/Unix-like filesystems\
+  \ for files entitlements"
+area: Infra/Core
+type: bug
+issues:
+ - 127047

docs/changelog/127118.yaml

@@ -0,0 +1,5 @@
+pr: 127118
+summary: Panama vector accelerated optimized scalar quantization
+area: Vector Search
+type: enhancement
+issues: []

docs/changelog/127267.yaml

@@ -0,0 +1,5 @@
+pr: 127267
+summary: Bypass competitive iteration in single filter bucket case
+area: "Aggregations"
+type: bug
+issues: [127262]

docs/reference/elasticsearch/index-settings/index-modules.md

@@ -49,39 +49,27 @@ $$$index-codec$$$ `index.codec`
 
 $$$index-mode-setting$$$ `index.mode`
 :   The `index.mode` setting is used to control settings applied in specific domains like ingestion of time series data or logs. Different mutually exclusive modes exist, which are used to apply settings or default values controlling indexing of documents, sorting and other parameters whose value affects indexing or query performance.
-
-```console
-PUT my-index-000001
-{
-  "settings": {
-    "index":{
-      "mode":"standard" <1>
-    }
-  }
-}
-```
-
-1. This index uses the `standard` index mode
-
-
-Index mode supports the following values:
-
-`null`
-:   Default value (same as `standard`).
-
-`standard`
-:   Standard indexing with default settings.
-
-`lookup`
-: Index that can be used for lookup joins in ES|QL. Limited to 1 shard.
-
-
-`time_series`
-:   *(data streams only)* Index mode optimized for storage of metrics. For more information, see [Time series index settings](time-series.md).
-
-`logsdb`
-:   *(data streams only)* Index mode optimized for [logs](docs-content://manage-data/data-store/data-streams/logs-data-stream.md).
-
+        
+        **Example**
+
+      ```console
+      PUT my-index-000001
+      {
+        "settings": {
+          "index":{
+            "mode":"standard" # This index uses the `standard` index mode
+          }
+        }
+      }
+      ```
+    **Supported values**
+
+    The `index.mode` setting supports the following values:
+       - `null`:   Default value (same as `standard`).
+       -  `standard`:   Standard indexing with default settings.
+       -  `lookup`: Index that can be used for [LOOKUP JOIN](/reference/query-languages/esql/esql-lookup-join.md) in ES|QL. Limited to 1 shard.
+       - `time_series`:   *(data streams only)* Index mode optimized for storage of metrics. For more information, see [Time series index settings](time-series.md).
+       - `logsdb`: *(data streams only)* Index mode optimized for [logs](docs-content://manage-data/data-store/data-streams/logs-data-stream.md).
 
 $$$routing-partition-size$$$ `index.routing_partition_size`
 :   The number of shards a custom routing value can go to. Defaults to 1 and can only be set at index creation time. This value must be less than the `index.number_of_routing_shards` unless the `index.number_of_routing_shards` value is also 1. for more details about how this setting is used, refer to [](/reference/elasticsearch/mapping-reference/mapping-routing-field.md#routing-index-partition).