Merge remote-tracking branch 'origin/main' into feature/esql-knn-function-minimal-support

Author: carlosdelest

docs/changelog/128736.yaml

@@ -0,0 +1,5 @@
+pr: 128736
+summary: Add `index.lifecycle.skip` index-scoped setting to instruct ILM to skip processing specific indices
+area: ILM+SLM
+type: enhancement
+issues: []

docs/changelog/128848.yaml

@@ -0,0 +1,5 @@
+pr: 128848
+summary: Add `bucketedSort` based on int
+area: Search
+type: enhancement
+issues: []

docs/reference/elasticsearch/index-settings/index.md

@@ -34,3 +34,5 @@ Settings are available for the following modules:
   Configure the backing indices in a time series data stream (TSDS).
 * [Translog](translog.md)
   Control the transaction log and background flush operations.
+
+There are also index settings associated with [text analysis](docs-content://manage-data/data-store/text-analysis.md), which define analyzers, tokenizers, token filters, and character filters.

muted-tests.yml

@@ -552,6 +552,21 @@ tests:
 - class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT
   method: test {p0=esql/190_lookup_join/alias-pattern-single}
   issue: https://github.com/elastic/elasticsearch/issues/128855
+- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT
+  method: test {p0=esql/191_lookup_join_on_datastreams/data streams supported in LOOKUP JOIN}
+  issue: https://github.com/elastic/elasticsearch/issues/128856
+- class: org.elasticsearch.upgrades.IndexSortUpgradeIT
+  method: testIndexSortForNumericTypes {upgradedNodes=3}
+  issue: https://github.com/elastic/elasticsearch/issues/128861
+- class: org.elasticsearch.upgrades.IndexSortUpgradeIT
+  method: testIndexSortForNumericTypes {upgradedNodes=1}
+  issue: https://github.com/elastic/elasticsearch/issues/128862
+- class: org.elasticsearch.upgrades.IndexSortUpgradeIT
+  method: testIndexSortForNumericTypes {upgradedNodes=2}
+  issue: https://github.com/elastic/elasticsearch/issues/128863
+- class: org.elasticsearch.packaging.test.DockerTests
+  method: test150MachineDependentHeap
+  issue: https://github.com/elastic/elasticsearch/issues/128120
 
 # Examples:
 #

server/src/main/java/module-info.java

@@ -8,6 +8,7 @@
  */
 
 import org.elasticsearch.plugins.internal.RestExtension;
+import org.elasticsearch.reservedstate.ReservedStateHandlerProvider;
 
 /** The Elasticsearch Server Module. */
 module org.elasticsearch.server {
@@ -411,7 +412,7 @@
             org.elasticsearch.index.shard.ShardToolCliProvider;
 
     uses org.elasticsearch.reservedstate.service.FileSettingsServiceProvider;
-    uses org.elasticsearch.reservedstate.ReservedClusterStateHandlerProvider;
+    uses ReservedStateHandlerProvider;
     uses org.elasticsearch.jdk.ModuleQualifiedExportsService;
     uses org.elasticsearch.node.internal.TerminationHandlerProvider;
     uses org.elasticsearch.internal.VersionExtension;

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -279,6 +279,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ESQL_REGEX_MATCH_WITH_CASE_INSENSITIVITY = def(9_086_0_00);
     public static final TransportVersion IDP_CUSTOM_SAML_ATTRIBUTES = def(9_087_0_00);
     public static final TransportVersion JOIN_ON_ALIASES = def(9_088_0_00);
+    public static final TransportVersion ILM_ADD_SKIP_SETTING = def(9_089_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/action/ActionModule.java

@@ -211,9 +211,7 @@
 import org.elasticsearch.action.termvectors.TransportTermVectorsAction;
 import org.elasticsearch.action.update.TransportUpdateAction;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.project.ProjectIdResolver;
 import org.elasticsearch.cluster.routing.RerouteService;
@@ -254,6 +252,7 @@
 import org.elasticsearch.repositories.VerifyNodeRepositoryAction;
 import org.elasticsearch.repositories.VerifyNodeRepositoryCoordinationAction;
 import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
+import org.elasticsearch.reservedstate.ReservedProjectStateHandler;
 import org.elasticsearch.reservedstate.service.ReservedClusterStateService;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
@@ -474,8 +473,8 @@ public ActionModule(
         TelemetryProvider telemetryProvider,
         ClusterService clusterService,
         RerouteService rerouteService,
-        List<ReservedClusterStateHandler<ClusterState, ?>> reservedClusterStateHandlers,
-        List<ReservedClusterStateHandler<ProjectMetadata, ?>> reservedProjectStateHandlers,
+        List<ReservedClusterStateHandler<?>> reservedClusterStateHandlers,
+        List<ReservedProjectStateHandler<?>> reservedProjectStateHandlers,
         RestExtension restExtension,
         IncrementalBulkService bulkService,
         ProjectIdResolver projectIdResolver

server/src/main/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryAction.java

@@ -34,7 +34,7 @@
  * It is used by the ReservedClusterStateService to add/update or remove snapshot repositories. Typical usage
  * for this action is in the context of file based settings.
  */
-public class ReservedRepositoryAction implements ReservedClusterStateHandler<ClusterState, List<PutRepositoryRequest>> {
+public class ReservedRepositoryAction implements ReservedClusterStateHandler<List<PutRepositoryRequest>> {
     public static final String NAME = "snapshot_repositories";
 
     private final RepositoriesService repositoriesService;
@@ -67,8 +67,7 @@ public Collection<PutRepositoryRequest> prepare(Object input) {
     }
 
     @Override
-    public TransformState<ClusterState> transform(List<PutRepositoryRequest> source, TransformState<ClusterState> prevState)
-        throws Exception {
+    public TransformState transform(List<PutRepositoryRequest> source, TransformState prevState) throws Exception {
         var requests = prepare(source);
 
         ClusterState state = prevState.state();
@@ -88,7 +87,7 @@ public TransformState<ClusterState> transform(List<PutRepositoryRequest> source,
             state = task.execute(state);
         }
 
-        return new TransformState<>(state, entities);
+        return new TransformState(state, entities);
 
     }
 

server/src/main/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateAction.java

@@ -12,13 +12,16 @@
 import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction;
 import org.elasticsearch.action.admin.indices.template.put.TransportPutComponentTemplateAction;
 import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
+import org.elasticsearch.reservedstate.ReservedProjectStateHandler;
 import org.elasticsearch.reservedstate.TransformState;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentParserConfiguration;
@@ -45,7 +48,7 @@
  */
 public class ReservedComposableIndexTemplateAction
     implements
-        ReservedClusterStateHandler<ProjectMetadata, ReservedComposableIndexTemplateAction.ComponentsAndComposables> {
+        ReservedProjectStateHandler<ReservedComposableIndexTemplateAction.ComponentsAndComposables> {
     public static final String NAME = "index_templates";
     public static final String COMPONENTS = "component_templates";
     private static final String COMPONENT_PREFIX = "component_template:";
@@ -133,10 +136,10 @@ private ComponentsAndComposables prepare(ComponentsAndComposables componentsAndC
     }
 
     @Override
-    public TransformState<ProjectMetadata> transform(ComponentsAndComposables source, TransformState<ProjectMetadata> prevState)
-        throws Exception {
+    public TransformState transform(ProjectId projectId, ComponentsAndComposables source, TransformState prevState) throws Exception {
         var requests = prepare(source);
-        ProjectMetadata project = prevState.state();
+        ClusterState clusterState = prevState.state();
+        ProjectMetadata project = clusterState.getMetadata().getProject(projectId);
 
         // We transform in the following order:
         // 1. create or update component templates (composable templates depend on them)
@@ -192,7 +195,10 @@ public TransformState<ProjectMetadata> transform(ComponentsAndComposables source
             project = MetadataIndexTemplateService.innerRemoveComponentTemplate(project, componentNames);
         }
 
-        return new TransformState<>(project, Sets.union(componentEntities, composableEntities));
+        return new TransformState(
+            ClusterState.builder(clusterState).putProjectMetadata(project).build(),
+            Sets.union(componentEntities, composableEntities)
+        );
     }
 
     @Override

server/src/main/java/org/elasticsearch/action/ingest/ReservedPipelineAction.java

@@ -10,11 +10,13 @@
 package org.elasticsearch.action.ingest;
 
 import org.elasticsearch.ElasticsearchGenerationException;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.ingest.IngestMetadata;
 import org.elasticsearch.ingest.IngestService;
-import org.elasticsearch.reservedstate.ReservedClusterStateHandler;
+import org.elasticsearch.reservedstate.ReservedProjectStateHandler;
 import org.elasticsearch.reservedstate.TransformState;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
@@ -36,7 +38,7 @@
  * It is used by the ReservedClusterStateService to add/update or remove ingest pipelines. Typical usage
  * for this action is in the context of file based state.
  */
-public class ReservedPipelineAction implements ReservedClusterStateHandler<ProjectMetadata, List<PutPipelineRequest>> {
+public class ReservedPipelineAction implements ReservedProjectStateHandler<List<PutPipelineRequest>> {
     public static final String NAME = "ingest_pipelines";
 
     /**
@@ -78,21 +80,21 @@ private static ProjectMetadata wrapIngestTaskExecute(IngestService.PipelineClust
     }
 
     @Override
-    public TransformState<ProjectMetadata> transform(List<PutPipelineRequest> source, TransformState<ProjectMetadata> prevState)
-        throws Exception {
+    public TransformState transform(ProjectId projectId, List<PutPipelineRequest> source, TransformState prevState) throws Exception {
         var requests = prepare(source);
 
-        ProjectMetadata state = prevState.state();
+        ClusterState clusterState = prevState.state();
+        ProjectMetadata projectMetadata = clusterState.metadata().getProject(projectId);
 
         for (var request : requests) {
-            var nopUpdate = IngestService.isNoOpPipelineUpdate(state, request);
+            var nopUpdate = IngestService.isNoOpPipelineUpdate(projectMetadata, request);
 
             if (nopUpdate) {
                 continue;
             }
 
-            var task = new IngestService.PutPipelineClusterStateUpdateTask(state.id(), request);
-            state = wrapIngestTaskExecute(task, state);
+            var task = new IngestService.PutPipelineClusterStateUpdateTask(projectMetadata.id(), request);
+            projectMetadata = wrapIngestTaskExecute(task, projectMetadata);
         }
 
         Set<String> entities = requests.stream().map(PutPipelineRequest::getId).collect(Collectors.toSet());
@@ -102,18 +104,18 @@ public TransformState<ProjectMetadata> transform(List<PutPipelineRequest> source
 
         for (var pipelineToDelete : toDelete) {
             var task = new IngestService.DeletePipelineClusterStateUpdateTask(
-                state.id(),
+                projectMetadata.id(),
                 null,
                 new DeletePipelineRequest(
                     RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT,
                     RESERVED_CLUSTER_STATE_HANDLER_IGNORED_TIMEOUT,
                     pipelineToDelete
                 )
             );
-            state = wrapIngestTaskExecute(task, state);
+            projectMetadata = wrapIngestTaskExecute(task, projectMetadata);
         }
 
-        return new TransformState<>(state, entities);
+        return new TransformState(ClusterState.builder(clusterState).putProjectMetadata(projectMetadata).build(), entities);
     }
 
     @Override