Merge branch 'main' into ES-10929

Author: tlrx

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OptimizedScalarQuantizerBenchmark.java

@@ -43,7 +43,8 @@ public class OptimizedScalarQuantizerBenchmark {
 
     float[] vector;
     float[] centroid;
-    byte[] destination;
+    byte[] legacyDestination;
+    int[] destination;
 
     @Param({ "1", "4", "7" })
     byte bits;
@@ -54,7 +55,8 @@ public class OptimizedScalarQuantizerBenchmark {
     public void init() {
         ThreadLocalRandom random = ThreadLocalRandom.current();
         // random byte arrays for binary methods
-        destination = new byte[dims];
+        legacyDestination = new byte[dims];
+        destination = new int[dims];
         vector = new float[dims];
         centroid = new float[dims];
         for (int i = 0; i < dims; ++i) {
@@ -65,13 +67,20 @@ public void init() {
 
     @Benchmark
     public byte[] scalar() {
-        osq.scalarQuantize(vector, destination, bits, centroid);
-        return destination;
+        osq.legacyScalarQuantize(vector, legacyDestination, bits, centroid);
+        return legacyDestination;
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public byte[] legacyVector() {
+        osq.legacyScalarQuantize(vector, legacyDestination, bits, centroid);
+        return legacyDestination;
     }
 
     @Benchmark
     @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
-    public byte[] vector() {
+    public int[] vector() {
         osq.scalarQuantize(vector, destination, bits, centroid);
         return destination;
     }

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java

@@ -175,6 +175,8 @@ private void addNameAndDescriptionToPom(Project project, NamedDomainObjectSet<Ma
     private static void configureWithShadowPlugin(Project project, MavenPublication publication) {
         var shadow = project.getExtensions().getByType(ShadowExtension.class);
         shadow.component(publication);
+        publication.artifact(project.getTasks().named("javadocJar"));
+        publication.artifact(project.getTasks().named("sourcesJar"));
     }
 
     private static void addScmInfo(XmlProvider xml, GitInfo gitInfo) {

docs/changelog/127223.yaml

@@ -0,0 +1,5 @@
+pr: 127223
+summary: Wrap ES KNN queries with PatienceKNN query
+area: Vector Search
+type: feature
+issues: []

docs/changelog/128866.yaml

@@ -0,0 +1,6 @@
+pr: 128866
+summary: Add `age_in_millis` to ILM Explain Response
+area: ILM+SLM
+type: enhancement
+issues:
+ - 103659

docs/changelog/129725.yaml

@@ -0,0 +1,5 @@
+pr: 129725
+summary: Throw a 400 when sorting for all types of range fields
+area: Search
+type: bug
+issues: []

docs/reference/elasticsearch/configuration-reference/data-stream-lifecycle-settings.md

@@ -3,8 +3,7 @@ navigation_title: "Data stream lifecycle settings"
 mapped_pages:
   - https://www.elastic.co/guide/en/elasticsearch/reference/current/data-stream-lifecycle-settings.html
 applies_to:
-  deployment:
-    self:
+  stack: all
 ---
 
 # Data stream lifecycle settings in {{es}} [data-stream-lifecycle-settings]
@@ -58,21 +57,24 @@ $$$data-streams-lifecycle-signalling-error-retry-interval$$$
 
 ## Index level settings [_index_level_settings]
 
+:::{include} ../index-settings/_snippets/serverless-availability.md
+:::
+
 The following index-level settings are typically configured on the backing indices of a data stream.
 
 $$$index-lifecycle-prefer-ilm$$$
 
 `index.lifecycle.prefer_ilm`
-:   ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting), boolean) This setting determines which feature is managing the backing index of a data stream if, and only if, the backing index has an [{{ilm}}](docs-content://manage-data/lifecycle/index-lifecycle-management.md) ({{ilm-init}}) policy and the data stream has also a built-in lifecycle. When `true` this index is managed by {{ilm-init}}, when `false` the backing index is managed by the data stream lifecycle. Defaults to `true`.
+:   ([Dynamic](../index-settings/index.md#index-modules-settings-description), boolean) This setting determines which feature is managing the backing index of a data stream if, and only if, the backing index has an [{{ilm}}](docs-content://manage-data/lifecycle/index-lifecycle-management.md) ({{ilm-init}}) policy and the data stream has also a built-in lifecycle. When `true` this index is managed by {{ilm-init}}, when `false` the backing index is managed by the data stream lifecycle. Defaults to `true`.
 
 $$$index-data-stream-lifecycle-origination-date$$$
 
-`index.lifecycle.origination_date`
-:   ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting), long) If specified, this is the timestamp used to calculate the backing index generation age after this backing index has been [rolled over](docs-content://manage-data/lifecycle/index-lifecycle-management/rollover.md). The generation age is used to determine data retention, consequently, you can use this setting if you create a backing index that contains older data and want to ensure that the retention period or other parts of the lifecycle will be applied based on the data’s original timestamp and not the timestamp they got indexed. Specified as a Unix epoch value in milliseconds.
+`index.lifecycle.origination_date` {applies_to}`serverless: all`
+:   ([Dynamic](../index-settings/index.md#index-modules-settings-description), long) If specified, this is the timestamp used to calculate the backing index generation age after this backing index has been [rolled over](docs-content://manage-data/lifecycle/index-lifecycle-management/rollover.md). The generation age is used to determine data retention, consequently, you can use this setting if you create a backing index that contains older data and want to ensure that the retention period or other parts of the lifecycle will be applied based on the data’s original timestamp and not the timestamp they got indexed. Specified as a Unix epoch value in milliseconds.
 
 ## Reindex settings [reindex-data-stream-settings]
 
-You can use the following settings to control the behavior of the reindex data stream API:
+You can use the following cluster-level settings to control the behavior of the reindex data stream API:
 
 `migrate.max_concurrent_indices_reindexed_per_data_stream` ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The number of backing indices within a given data stream which will be reindexed concurrently. Defaults to `1`.
 

docs/reference/elasticsearch/configuration-reference/index-lifecycle-management-settings.md

@@ -3,8 +3,7 @@ navigation_title: "{{ilm-cap}} settings"
 mapped_pages:
   - https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-settings.html
 applies_to:
-  deployment:
-    self:
+  stack: all
 ---
 
 # {{ilm-cap}} settings in {{es}} [ilm-settings]
@@ -37,34 +36,37 @@ $$$indices-lifecycle-rollover-only-if-has-documents$$$
 
 ## Index level settings [_index_level_settings_2]
 
+:::{include} ../index-settings/_snippets/serverless-availability.md
+:::
+
 These index-level {{ilm-init}} settings are typically configured through index templates. For more information, see [Create a lifecycle policy](docs-content://manage-data/lifecycle/index-lifecycle-management/tutorial-automate-rollover.md#ilm-gs-create-policy).
 
 `index.lifecycle.indexing_complete`
-:   ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting), Boolean) Indicates whether or not the index has been rolled over. Automatically set to `true` when {{ilm-init}} completes the rollover action. You can explicitly set it to [skip rollover](docs-content://manage-data/lifecycle/index-lifecycle-management/skip-rollover.md). Defaults to `false`.
+:   ([Dynamic](../index-settings/index.md#index-modules-settings-description), Boolean) Indicates whether or not the index has been rolled over. Automatically set to `true` when {{ilm-init}} completes the rollover action. You can explicitly set it to [skip rollover](docs-content://manage-data/lifecycle/index-lifecycle-management/skip-rollover.md). Defaults to `false`.
 
 $$$index-lifecycle-name$$$
 
 `index.lifecycle.name`
-:   ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting), string) The name of the policy to use to manage the index. For information about how {{es}} applies policy changes, see [Policy updates](docs-content://manage-data/lifecycle/index-lifecycle-management/policy-updates.md). If you are restoring an index from snapshot that was previously managed by {{ilm}}, you can override this setting to null during the restore operation to disable further management of the index. See also [Index level settings](#index-lifecycle-rollover-alias).
+:   ([Dynamic](../index-settings/index.md#index-modules-settings-description), string) The name of the policy to use to manage the index. For information about how {{es}} applies policy changes, see [Policy updates](docs-content://manage-data/lifecycle/index-lifecycle-management/policy-updates.md). If you are restoring an index from snapshot that was previously managed by {{ilm}}, you can override this setting to null during the restore operation to disable further management of the index. See also [Index level settings](#index-lifecycle-rollover-alias).
 
 $$$index-lifecycle-origination-date$$$
 
-`index.lifecycle.origination_date`
-:   ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting), long) If specified, this is the timestamp used to calculate the index age for its phase transitions. Use this setting if you create a new index that contains old data and want to use the original creation date to calculate the index age. Specified as a Unix epoch value in milliseconds.
+`index.lifecycle.origination_date` {applies_to}`serverless: all`
+:   ([Dynamic](../index-settings/index.md#index-modules-settings-description), long) If specified, this is the timestamp used to calculate the index age for its phase transitions. Use this setting if you create a new index that contains old data and want to use the original creation date to calculate the index age. Specified as a Unix epoch value in milliseconds.
 
 $$$index-lifecycle-parse-origination-date$$$
 
 `index.lifecycle.parse_origination_date`
-:   ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting), Boolean) Set to `true` to parse the origination date from the index name. This origination date is used to calculate the index age for its phase transitions. The index name must match the pattern `^.*-{{date_format}}-\\d+`, where the `date_format` is `yyyy.MM.dd` and the trailing digits are optional. An index that was rolled over would normally match the full format, for example `logs-2016.10.31-000002`). If the index name doesn’t match the pattern, index creation fails.
+:   ([Dynamic](../index-settings/index.md#index-modules-settings-description), Boolean) Set to `true` to parse the origination date from the index name. This origination date is used to calculate the index age for its phase transitions. The index name must match the pattern `^.*-{{date_format}}-\\d+`, where the `date_format` is `yyyy.MM.dd` and the trailing digits are optional. An index that was rolled over would normally match the full format, for example `logs-2016.10.31-000002`). If the index name doesn’t match the pattern, index creation fails.
 
 $$$index-lifecycle-step-wait-time-threshold$$$
 
 `index.lifecycle.step.wait_time_threshold`
-:   ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting), [time value](/reference/elasticsearch/rest-apis/api-conventions.md#time-units)) Time to wait for the cluster to resolve allocation issues during an {{ilm-init}} [`shrink`](/reference/elasticsearch/index-lifecycle-actions/ilm-shrink.md) action. Must be greater than `1h` (1 hour). Defaults to `12h` (12 hours). See [Shard allocation for shrink](/reference/elasticsearch/index-lifecycle-actions/ilm-shrink.md#ilm-shrink-shard-allocation).
+:   ([Dynamic](../index-settings/index.md#index-modules-settings-description), [time value](/reference/elasticsearch/rest-apis/api-conventions.md#time-units)) Time to wait for the cluster to resolve allocation issues during an {{ilm-init}} [`shrink`](/reference/elasticsearch/index-lifecycle-actions/ilm-shrink.md) action. Must be greater than `1h` (1 hour). Defaults to `12h` (12 hours). See [Shard allocation for shrink](/reference/elasticsearch/index-lifecycle-actions/ilm-shrink.md#ilm-shrink-shard-allocation).
 
 $$$index-lifecycle-rollover-alias$$$
 
 `index.lifecycle.rollover_alias`
-:   ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting), string) The index alias to update when the index rolls over. Specify when using a policy that contains a rollover action. When the index rolls over, the alias is updated to reflect that the index is no longer the write index. For more information about rolling indices, see [Rollover](docs-content://manage-data/lifecycle/index-lifecycle-management/rollover.md). If you are restoring an index from snapshot that was previously managed by {{ilm}}, you can override this setting to null during the restore operation to disable further management of future indices. See also [Index level settings](#index-lifecycle-name).
+:   ([Dynamic](../index-settings/index.md#index-modules-settings-description), string) The index alias to update when the index rolls over. Specify when using a policy that contains a rollover action. When the index rolls over, the alias is updated to reflect that the index is no longer the write index. For more information about rolling indices, see [Rollover](docs-content://manage-data/lifecycle/index-lifecycle-management/rollover.md). If you are restoring an index from snapshot that was previously managed by {{ilm}}, you can override this setting to null during the restore operation to disable further management of future indices. See also [Index level settings](#index-lifecycle-name).
 
 

docs/reference/elasticsearch/configuration-reference/index.md

@@ -5,6 +5,7 @@ applies_to:
     ess:
     ece:
     self:
+    eck:
   serverless:
 ---
 

docs/reference/elasticsearch/configuration-reference/thread-pool-settings.md

@@ -59,8 +59,11 @@ $$$search-throttled$$$`search_throttled`
 `flush`
 :   For [flush](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-flush) and [translog](/reference/elasticsearch/index-settings/translog.md) `fsync` operations. Thread pool type is `scaling` with a keep-alive of `5m` and a default maximum size of `min(5, (`[`# of allocated processors`](#node.processors)`) / 2)`.
 
+`merge`
+:   For [merge](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules-merge.html) operations of all the shards on the node. Thread pool type is `scaling` with a keep-alive of `5m` and a default maximum size of [`# of allocated processors`](#node.processors).
+
 `force_merge`
-:   For [force merge](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-forcemerge) operations. Thread pool type is `fixed` with a size of `max(1, (`[`# of allocated processors`](#node.processors)`) / 8)` and an unbounded queue size.
+:   For waiting on blocking [force merge](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-forcemerge) operations. Thread pool type is `fixed` with a size of `max(1, (`[`# of allocated processors`](#node.processors)`) / 8)` and an unbounded queue size.
 
 `management`
 :   For cluster management. Thread pool type is `scaling` with a keep-alive of `5m` and a default maximum size of `5`.

docs/reference/elasticsearch/index-settings/_snippets/serverless-availability.md

@@ -0,0 +1,3 @@
+::::{admonition} Settings supported in Serverless
+{{serverless-full}} projects [restrict the available Elasticsearch settings](docs-content://deploy-manage/deploy/elastic-cloud/differences-from-other-elasticsearch-offerings.md#elasticsearch-differences-serverless-settings-availability) to a supported subset, identified with a `Serverless` badge next to the setting name. For a complete list of available index settings, refer to the [Serverless index settings list](/reference/elasticsearch/index-settings/serverless.md).
+::::