Merge branch 'main' into increase_ccs_yaml_suite_timeouts

Author: benchaplin

docs/changelog/131723.yaml

@@ -0,0 +1,5 @@
+pr: 131723
+summary: Tests for FORK's evaluation of field names used in `field_caps` resolve calls
+area: Search
+type: bug
+issues: []

docs/changelog/132646.yaml

@@ -0,0 +1,5 @@
+pr: 132646
+summary: Update EIS sparse and dense embedding max batch size to 16
+area: Machine Learning
+type: bug
+issues: []

docs/release-notes/known-issues.md

@@ -1,17 +1,23 @@
 ---
 navigation_title: "Known issues"
 mapped_pages:
-  - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-connectors-known-issues.html
+  - https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html
 ---
 
 # Elasticsearch known issues [elasticsearch-known-issues]
+
 Known issues are significant defects or limitations that may impact your implementation. These issues are actively being worked on and will be addressed in a future release. Review the Elasticsearch known issues to help you make informed decisions, such as upgrading to a new version.
 
 ## 9.1.0 [elasticsearch-9.1.0-known-issues]
-* An error in the configuration of vector indices with type `bbq_hnsw` may lead to significant search performance degradation on 9.1.0. To mitigate this, set the `-Dvector.rescoring.directio=false` JVM option on all search nodes, then restart the nodes.
-  This option can be removed in 9.1.1.
+
+* The `-Dvector.rescoring.directio` JVM option is enabled (set to `true`) by default. When used with `bbq_hnsw` type vector indices, this can cause significant search performance degradation; particularly when enough memory is available to hold all vector data. In some cases, kNN search latency can increase by as much as 10x. To mitigate this, set the JVM option `-Dvector.rescoring.directio=false` on all search nodes and restart them. This option can be removed in 9.1.1.
+
+**How do I know if my index vector type is `bbq_hnsw`?**
+
+* Prior to 9.1, the vector type had to be explicitly set to `bbq_hnsw`. Starting with 9.1, `bbq_hnsw` is the default vector type for dense vectors with more than 384 dimensions in new indices, unless another type is specified.
 
 ## 9.0.3 [elasticsearch-9.0.3-known-issues]
+
 * A bug in the merge scheduler in Elasticsearch 9.0.3 may prevent shards from closing when there isn’t enough disk space to complete a merge. As a result, operations such as closing or relocating an index may hang until sufficient disk space becomes available.
 To mitigate this issue, the disk space checker is disabled by default in 9.0.3 by setting `indices.merge.disk.check_interval` to `0` seconds. Manually enabling this setting is not recommended.
 

muted-tests.yml

@@ -438,9 +438,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.ccq.MultiClustersIT
   method: testLookupJoinAliasesSkipOld
   issue: https://github.com/elastic/elasticsearch/issues/131697
-- class: org.elasticsearch.indices.cluster.RemoteSearchForceConnectTimeoutIT
-  method: testTimeoutSetting
-  issue: https://github.com/elastic/elasticsearch/issues/131656
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test151MachineDependentHeapWithSizeOverride
   issue: https://github.com/elastic/elasticsearch/issues/123437
@@ -453,9 +450,6 @@ tests:
 - class: org.elasticsearch.index.engine.MergeWithLowDiskSpaceIT
   method: testRelocationWhileForceMerging
   issue: https://github.com/elastic/elasticsearch/issues/131789
-- class: org.elasticsearch.indices.cluster.FieldCapsForceConnectTimeoutIT
-  method: testTimeoutSetting
-  issue: https://github.com/elastic/elasticsearch/issues/132179
 - class: org.elasticsearch.test.rest.yaml.MDPYamlTestSuiteIT
   method: test {yaml=mdp/10_basic/Index using shared data path}
   issue: https://github.com/elastic/elasticsearch/issues/132223

rest-api-spec/src/main/resources/rest-api-spec/api/transform.set_upgrade_mode.json

@@ -0,0 +1,33 @@
+{
+  "transform.set_upgrade_mode":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/transform-set-upgrade-mode.html",
+      "description":"Sets a cluster wide upgrade_mode setting that prepares transform indices for an upgrade."
+    },
+    "stability":"stable",
+    "visibility":"public",
+    "headers":{
+      "accept": [ "application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_transform/set_upgrade_mode",
+          "methods":[
+            "POST"
+          ]
+        }
+      ]
+    },
+    "params":{
+      "enabled":{
+        "type":"boolean",
+        "description":"Whether to enable upgrade_mode Transform setting or not. Defaults to false."
+      },
+      "timeout":{
+        "type":"time",
+        "description":"Controls the time to wait before action times out. Defaults to 30 seconds"
+      }
+    }
+  }
+}

server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/FieldCapsForceConnectTimeoutIT.java

@@ -73,21 +73,23 @@ protected Map<String, Boolean> skipUnavailableForRemoteClusters() {
 
     public void testTimeoutSetting() {
         var latch = new CountDownLatch(1);
-        for (String nodeName : cluster(LOCAL_CLUSTER).getNodeNames()) {
-            MockTransportService mts = (MockTransportService) cluster(LOCAL_CLUSTER).getInstance(TransportService.class, nodeName);
-
-            mts.addConnectBehavior(
-                cluster(LINKED_CLUSTER_1).getInstance(TransportService.class, (String) null),
-                ((transport, discoveryNode, profile, listener) -> {
+        for (String localNodeName : cluster(LOCAL_CLUSTER).getNodeNames()) {
+            MockTransportService localMts = (MockTransportService) cluster(LOCAL_CLUSTER).getInstance(
+                TransportService.class,
+                localNodeName
+            );
+            for (String remoteNodeName : cluster(LINKED_CLUSTER_1).getNodeNames()) {
+                TransportService remoteTs = cluster(LINKED_CLUSTER_1).getInstance(TransportService.class, remoteNodeName);
+                localMts.addConnectBehavior(remoteTs, ((transport, discoveryNode, profile, listener) -> {
                     try {
                         latch.await();
                     } catch (InterruptedException e) {
                         throw new AssertionError(e);
                     }
 
                     transport.openConnection(discoveryNode, profile, listener);
-                })
-            );
+                }));
+            }
         }
 
         // Add some dummy data to prove we are communicating fine with the remote.

server/src/internalClusterTest/java/org/elasticsearch/indices/cluster/RemoteSearchForceConnectTimeoutIT.java

@@ -70,21 +70,23 @@ protected Map<String, Boolean> skipUnavailableForRemoteClusters() {
 
     public void testTimeoutSetting() {
         var latch = new CountDownLatch(1);
-        for (String nodeName : cluster(LOCAL_CLUSTER).getNodeNames()) {
-            MockTransportService mts = (MockTransportService) cluster(LOCAL_CLUSTER).getInstance(TransportService.class, nodeName);
-
-            mts.addConnectBehavior(
-                cluster(REMOTE_CLUSTER_1).getInstance(TransportService.class, (String) null),
-                ((transport, discoveryNode, profile, listener) -> {
+        for (String localNodeName : cluster(LOCAL_CLUSTER).getNodeNames()) {
+            MockTransportService localMts = (MockTransportService) cluster(LOCAL_CLUSTER).getInstance(
+                TransportService.class,
+                localNodeName
+            );
+            for (String remoteNodeName : cluster(REMOTE_CLUSTER_1).getNodeNames()) {
+                TransportService remoteTs = cluster(REMOTE_CLUSTER_1).getInstance(TransportService.class, remoteNodeName);
+                localMts.addConnectBehavior(remoteTs, ((transport, discoveryNode, profile, listener) -> {
                     try {
                         latch.await();
                     } catch (InterruptedException e) {
                         throw new AssertionError(e);
                     }
 
                     transport.openConnection(discoveryNode, profile, listener);
-                })
-            );
+                }));
+            }
         }
 
         // Add some dummy data to prove we are communicating fine with the remote.

server/src/main/java/org/elasticsearch/action/admin/cluster/remote/TransportRemoteInfoAction.java

@@ -13,7 +13,6 @@
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.cluster.node.DiscoveryNodeRole;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
@@ -35,15 +34,7 @@ public TransportRemoteInfoAction(TransportService transportService, ActionFilter
 
     @Override
     protected void doExecute(Task task, RemoteInfoRequest remoteInfoRequest, ActionListener<RemoteInfoResponse> listener) {
-        if (remoteClusterService.isEnabled() == false) {
-            throw new IllegalArgumentException(
-                "node ["
-                    + remoteClusterService.getLocalNode().getName()
-                    + "] does not have the ["
-                    + DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE.roleName()
-                    + "] role"
-            );
-        }
+        remoteClusterService.ensureClientIsEnabled();
         listener.onResponse(new RemoteInfoResponse(remoteClusterService.getRemoteConnectionInfos().collect(toList())));
     }
 }

server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java

@@ -49,6 +49,10 @@ protected RemoteClusterAware(Settings settings) {
         this.isRemoteClusterClientEnabled = DiscoveryNode.isRemoteClusterClient(settings);
     }
 
+    protected String getNodeName() {
+        return nodeName;
+    }
+
     /**
      * Returns remote clusters that are enabled in these settings
      */

server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java

@@ -147,13 +147,11 @@ public final class RemoteClusterService extends RemoteClusterAware
 
     public static final String REMOTE_CLUSTER_HANDSHAKE_ACTION_NAME = "cluster:internal/remote_cluster/handshake";
 
-    private final boolean enabled;
+    private final boolean isRemoteClusterClient;
+    private final boolean isSearchNode;
+    private final boolean isStateless;
     private final boolean remoteClusterServerEnabled;
 
-    public boolean isEnabled() {
-        return enabled;
-    }
-
     public boolean isRemoteClusterServerEnabled() {
         return remoteClusterServerEnabled;
     }
@@ -166,7 +164,9 @@ public boolean isRemoteClusterServerEnabled() {
     @FixForMultiProject(description = "Inject the ProjectResolver instance.")
     RemoteClusterService(Settings settings, TransportService transportService) {
         super(settings);
-        this.enabled = DiscoveryNode.isRemoteClusterClient(settings);
+        this.isRemoteClusterClient = DiscoveryNode.isRemoteClusterClient(settings);
+        this.isSearchNode = DiscoveryNode.hasRole(settings, DiscoveryNodeRole.SEARCH_ROLE);
+        this.isStateless = DiscoveryNode.isStateless(settings);
         this.remoteClusterServerEnabled = REMOTE_CLUSTER_SERVER_ENABLED.get(settings);
         this.transportService = transportService;
         this.projectResolver = DefaultProjectResolver.INSTANCE;
@@ -179,10 +179,6 @@ public boolean isRemoteClusterServerEnabled() {
         }
     }
 
-    public DiscoveryNode getLocalNode() {
-        return transportService.getLocalNode();
-    }
-
     /**
      * Group indices by cluster alias mapped to OriginalIndices for that cluster.
      * @param remoteClusterNames Set of configured remote cluster names.
@@ -335,11 +331,7 @@ public void maybeEnsureConnectedAndGetConnection(
     }
 
     public RemoteClusterConnection getRemoteClusterConnection(String cluster) {
-        if (enabled == false) {
-            throw new IllegalArgumentException(
-                "this node does not have the " + DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE.roleName() + " role"
-            );
-        }
+        ensureClientIsEnabled();
         @FixForMultiProject(description = "Verify all callers will have the proper context set for resolving the origin project ID.")
         RemoteClusterConnection connection = getConnectionsMapForCurrentProject().get(cluster);
         if (connection == null) {
@@ -595,11 +587,7 @@ public RemoteClusterServerInfo info() {
      * function on success.
      */
     public void collectNodes(Set<String> clusters, ActionListener<BiFunction<String, String, DiscoveryNode>> listener) {
-        if (enabled == false) {
-            throw new IllegalArgumentException(
-                "this node does not have the " + DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE.roleName() + " role"
-            );
-        }
+        ensureClientIsEnabled();
         @FixForMultiProject(description = "Analyze usages and determine if the project ID must be provided.")
         final var projectConnectionsMap = getConnectionsMapForCurrentProject();
         final var connectionsMap = new HashMap<String, RemoteClusterConnection>();
@@ -662,11 +650,7 @@ public RemoteClusterClient getRemoteClusterClient(
         Executor responseExecutor,
         DisconnectedStrategy disconnectedStrategy
     ) {
-        if (transportService.getRemoteClusterService().isEnabled() == false) {
-            throw new IllegalArgumentException(
-                "this node does not have the " + DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE.roleName() + " role"
-            );
-        }
+        ensureClientIsEnabled();
         if (transportService.getRemoteClusterService().getRegisteredRemoteClusterNames().contains(clusterAlias) == false) {
             throw new NoSuchRemoteClusterException(clusterAlias);
         }
@@ -677,6 +661,34 @@ public RemoteClusterClient getRemoteClusterClient(
         });
     }
 
+    /**
+     * Verifies this node is configured to support linked project client operations.
+     * @throws IllegalArgumentException If this node is not configured to support client operations.
+     */
+    public void ensureClientIsEnabled() {
+        if (isRemoteClusterClient) {
+            return;
+        }
+        if (isStateless == false) {
+            throw new IllegalArgumentException(
+                "node [" + getNodeName() + "] does not have the [" + DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE.roleName() + "] role"
+            );
+        }
+        // For stateless the remote cluster client is enabled by default for search nodes,
+        // REMOTE_CLUSTER_CLIENT_ROLE is not explicitly required.
+        if (isSearchNode == false) {
+            throw new IllegalArgumentException(
+                "node ["
+                    + getNodeName()
+                    + "] must have the ["
+                    + DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE.roleName()
+                    + "] role or the ["
+                    + DiscoveryNodeRole.SEARCH_ROLE.roleName()
+                    + "] role in stateless environments to use linked project client features"
+            );
+        }
+    }
+
     static void registerRemoteClusterHandshakeRequestHandler(TransportService transportService) {
         transportService.registerRequestHandler(
             REMOTE_CLUSTER_HANDSHAKE_ACTION_NAME,