Merge branch 'main' into cluster_info_for_write_decider

Author: nicktindall

docs/reference/query-languages/esql/_snippets/commands/layout/completion.md

@@ -46,9 +46,39 @@ including:
 **Requirements**
 
 To use this command, you must deploy your LLM model in Elasticsearch as
-an [≈inference endpoint](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) with the
+an [inference endpoint](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) with the
 task type `completion`.
 
+#### Handling timeouts
+
+`COMPLETION` commands may time out when processing large datasets or complex prompts. The default timeout is 10 minutes, but you can increase this limit if necessary.
+
+How you increase the timeout depends on your deployment type:
+
+::::{tab-set}
+:::{tab-item} {{ech}}
+* You can adjust {{es}} settings in the [Elastic Cloud Console](docs-content://deploy-manage/deploy/elastic-cloud/edit-stack-settings.md)
+* You can also adjust the `search.default_search_timeout` cluster setting using [Kibana's Advanced settings](kibana://reference/advanced-settings.md#kibana-search-settings)
+:::
+
+:::{tab-item} Self-managed
+* You can configure at the cluster level by setting `search.default_search_timeout` in `elasticsearch.yml` or updating via [Cluster Settings API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-cluster-put-settings)
+* You can also adjust the `search:timeout` setting using [Kibana's Advanced settings](kibana://reference/advanced-settings.md#kibana-search-settings)
+* Alternatively, you can add timeout parameters to individual queries
+:::
+
+:::{tab-item} {{serverless-full}}
+* Requires a manual override from Elastic Support because you cannot modify timeout settings directly
+:::
+::::
+
+If you don't want to increase the timeout limit, try the following:
+
+* Reduce data volume with `LIMIT` or more selective filters before the `COMPLETION` command
+* Split complex operations into multiple simpler queries 
+* Configure your HTTP client's response timeout (Refer to [HTTP client configuration](/reference/elasticsearch/configuration-reference/networking-settings.md#_http_client_configuration))
+
+
 **Examples**
 
 Use the default column name (results stored in `completion` column):

muted-tests.yml

@@ -608,6 +608,12 @@ tests:
 - class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
   method: testDenseVectorMappingUpdate {initialType=int4_flat updateType=hnsw}
   issue: https://github.com/elastic/elasticsearch/issues/132150
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=int8_flat updateType=bbq_flat}
+  issue: https://github.com/elastic/elasticsearch/issues/132151
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: "testDenseVectorMappingUpdate {initialType=bbq_hnsw updateType=bbq_disk #2}"
+  issue: https://github.com/elastic/elasticsearch/issues/132152
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/cluster/project/ProjectDeletedListener.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.cluster.project;
+
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.metadata.ProjectId;
+import org.elasticsearch.cluster.service.ClusterService;
+
+import java.util.function.Consumer;
+
+/**
+ * Utility class to make it easy to run a block of code whenever a project is deleted (e.g. to cleanup cache entries)
+ */
+public class ProjectDeletedListener {
+
+    private final Consumer<ProjectId> consumer;
+
+    public ProjectDeletedListener(Consumer<ProjectId> consumer) {
+        this.consumer = consumer;
+    }
+
+    public void attach(ClusterService clusterService) {
+        clusterService.addListener(event -> {
+            final ClusterChangedEvent.ProjectsDelta delta = event.projectDelta();
+            delta.removed().forEach(consumer);
+        });
+    }
+
+}

server/src/test/java/org/elasticsearch/cluster/project/ProjectDeletedListenerTests.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.cluster.project;
+
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectId;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
+import org.elasticsearch.cluster.routing.GlobalRoutingTableTestHelper;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
+import org.elasticsearch.test.ClusterServiceUtils;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ProjectDeletedListenerTests extends ESTestCase {
+
+    public void testInvocation() {
+        final List<ProjectId> existingProjects = randomList(5, 15, ESTestCase::randomUniqueProjectId);
+
+        try (ClusterService clusterService = ClusterServiceUtils.createClusterService(new DeterministicTaskQueue().getThreadPool())) {
+            final ClusterState.Builder csBuilder = ClusterState.builder(ClusterName.DEFAULT);
+            existingProjects.forEach(p -> csBuilder.putProjectMetadata(ProjectMetadata.builder(p).build()));
+            final ClusterState cs0 = csBuilder.build();
+
+            ClusterServiceUtils.setState(clusterService, cs0);
+
+            final Set<ProjectId> notifiedProjects = new HashSet<>();
+            var pdl = new ProjectDeletedListener(notifiedProjects::add);
+            pdl.attach(clusterService);
+
+            final Set<ProjectId> projectsToDelete = Set.copyOf(
+                randomSubsetOf(randomIntBetween(1, existingProjects.size() / 2), existingProjects)
+            );
+            final List<ProjectId> projectsToCreate = randomList(0, 3, ESTestCase::randomUniqueProjectId);
+
+            final var mdBuilder = Metadata.builder(cs0.metadata());
+            projectsToDelete.forEach(mdBuilder::removeProject);
+            projectsToCreate.forEach(p -> mdBuilder.put(ProjectMetadata.builder(p).build()));
+            var md = mdBuilder.build();
+            var cs1 = ClusterState.builder(cs0)
+                .metadata(md)
+                .routingTable(GlobalRoutingTableTestHelper.buildRoutingTable(md, RoutingTable.Builder::addAsNew))
+                .build();
+            ClusterServiceUtils.setState(clusterService, cs1);
+
+            assertThat(notifiedProjects, equalTo(projectsToDelete));
+        }
+    }
+
+}

test/framework/src/main/java/org/elasticsearch/cluster/project/TestProjectResolvers.java

@@ -19,13 +19,14 @@
 
 import java.util.Collection;
 import java.util.Objects;
+import java.util.function.Supplier;
 
 /**
  * An implementation of {@link ProjectResolver} that handles multiple projects for testing purposes. Not usable in production
  */
 public final class TestProjectResolvers {
 
-    public static final ProjectResolver DEFAULT_PROJECT_ONLY = singleProject(Metadata.DEFAULT_PROJECT_ID, true);
+    public static final ProjectResolver DEFAULT_PROJECT_ONLY = singleProject(() -> Metadata.DEFAULT_PROJECT_ID, true);
 
     /**
      * @return a ProjectResolver that must only be used in a cluster context. It throws in single project related methods.
@@ -131,6 +132,14 @@ public static ProjectResolver alwaysThrow() {
      * The ProjectResolver can work with cluster state containing multiple projects and its supportsMultipleProjects returns true.
      */
     public static ProjectResolver singleProject(ProjectId projectId) {
+        return singleProject(() -> projectId, false);
+    }
+
+    /**
+     * This method returns a ProjectResolver that gives back the specified project-id when its getProjectId method is called.
+     * The ProjectResolver can work with cluster state containing multiple projects and its supportsMultipleProjects returns true.
+     */
+    public static ProjectResolver singleProject(Supplier<ProjectId> projectId) {
         return singleProject(projectId, false);
     }
 
@@ -140,11 +149,11 @@ public static ProjectResolver singleProject(ProjectId projectId) {
      * In addition, the ProjectResolvers returns false for supportsMultipleProjects.
      */
     public static ProjectResolver singleProjectOnly(ProjectId projectId) {
-        return singleProject(projectId, true);
+        return singleProject(() -> projectId, true);
     }
 
-    private static ProjectResolver singleProject(ProjectId projectId, boolean only) {
-        Objects.requireNonNull(projectId);
+    private static ProjectResolver singleProject(Supplier<ProjectId> projectIdSupplier, boolean only) {
+        Objects.requireNonNull(projectIdSupplier);
         return new ProjectResolver() {
 
             @Override
@@ -157,7 +166,7 @@ public ProjectMetadata getProjectMetadata(Metadata metadata) {
 
             @Override
             public ProjectId getProjectId() {
-                return projectId;
+                return projectIdSupplier.get();
             }
 
             @Override
@@ -170,6 +179,7 @@ public Collection<ProjectId> getProjectIds(ClusterState clusterState) {
 
             @Override
             public <E extends Exception> void executeOnProject(ProjectId otherProjectId, CheckedRunnable<E> body) throws E {
+                final ProjectId projectId = projectIdSupplier.get();
                 if (projectId.equals(otherProjectId)) {
                     body.run();
                 } else {