Merge remote-tracking branch 'es/main' into synthetic_source_encode_arrays

Author: martijnvg

docs/changelog/122653.yaml

@@ -0,0 +1,6 @@
+pr: 122653
+summary: Knn vector rescoring to sort score docs
+area: Vector Search
+type: bug
+issues:
+ - 119711

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -38,7 +38,8 @@ public record BootstrapArgs(
         Function<Class<?>, String> pluginResolver,
         Path[] dataDirs,
         Path configDir,
-        Path tempDir
+        Path tempDir,
+        Path logsDir
     ) {
         public BootstrapArgs {
             requireNonNull(pluginPolicies);
@@ -64,22 +65,24 @@ public static BootstrapArgs bootstrapArgs() {
      *
      * @param pluginPolicies a map holding policies for plugins (and modules), by plugin (or module) name.
      * @param pluginResolver a functor to map a Java Class to the plugin it belongs to (the plugin name).
-     * @param dataDirs data directories for Elasticsearch
-     * @param configDir the config directory for Elasticsearch
-     * @param tempDir the temp directory for Elasticsearch
+     * @param dataDirs       data directories for Elasticsearch
+     * @param configDir      the config directory for Elasticsearch
+     * @param tempDir        the temp directory for Elasticsearch
+     * @param logsDir        the log directory for Elasticsearch
      */
     public static void bootstrap(
         Map<String, Policy> pluginPolicies,
         Function<Class<?>, String> pluginResolver,
         Path[] dataDirs,
         Path configDir,
-        Path tempDir
+        Path tempDir,
+        Path logsDir
     ) {
         logger.debug("Loading entitlement agent");
         if (EntitlementBootstrap.bootstrapArgs != null) {
             throw new IllegalStateException("plugin data is already set");
         }
-        EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(pluginPolicies, pluginResolver, dataDirs, configDir, tempDir);
+        EntitlementBootstrap.bootstrapArgs = new BootstrapArgs(pluginPolicies, pluginResolver, dataDirs, configDir, tempDir, logsDir);
         exportInitializationToAgent();
         loadAgent(findAgentJar());
         selfTest();

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -129,6 +129,7 @@ private static PolicyManager createPolicyManager() {
         EntitlementBootstrap.BootstrapArgs bootstrapArgs = EntitlementBootstrap.bootstrapArgs();
         Map<String, Policy> pluginPolicies = bootstrapArgs.pluginPolicies();
         var pathLookup = new PathLookup(bootstrapArgs.configDir(), bootstrapArgs.dataDirs(), bootstrapArgs.tempDir());
+        Path logsDir = EntitlementBootstrap.bootstrapArgs().logsDir();
 
         // TODO(ES-10031): Decide what goes in the elasticsearch default policy and extend it
         var serverPolicy = new Policy(
@@ -147,7 +148,10 @@ private static PolicyManager createPolicyManager() {
                         new LoadNativeLibrariesEntitlement(),
                         new ManageThreadsEntitlement(),
                         new FilesEntitlement(
-                            List.of(FilesEntitlement.FileData.ofPath(EntitlementBootstrap.bootstrapArgs().tempDir(), READ_WRITE))
+                            List.of(
+                                FilesEntitlement.FileData.ofPath(EntitlementBootstrap.bootstrapArgs().tempDir(), READ_WRITE),
+                                FilesEntitlement.FileData.ofPath(EntitlementBootstrap.bootstrapArgs().logsDir(), READ_WRITE)
+                            )
                         )
                     )
                 ),

modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4WriteThrottlingHandlerTests.java

@@ -109,6 +109,7 @@ public void testThrottleLargeCompositeMessage() {
             Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE * fullSizeChunks + extraChunkSize
         );
         int splitOffset = randomIntBetween(0, messageBytes.length);
+        int lastChunkSizeOfTheFirstSplit = splitOffset % Netty4WriteThrottlingHandler.MAX_BYTES_PER_WRITE;
         final BytesReference message = CompositeBytesReference.of(
             new BytesArray(messageBytes, 0, splitOffset),
             new BytesArray(messageBytes, splitOffset, messageBytes.length - splitOffset)
@@ -120,7 +121,9 @@ public void testThrottleLargeCompositeMessage() {
         assertFalse(promise.isDone());
         embeddedChannel.flush();
         assertTrue(promise.isDone());
-        assertThat(seen, hasSize(oneOf(fullSizeChunks, fullSizeChunks + 1)));
+        // If the extra chunk size is greater than the last chunk size for the first half of the split, it means we will need to send
+        // (extraChunkSize - lastChunkSizeOfTheFirstSplit) bytes as the very last chunk of the entire message.
+        assertThat(seen, hasSize(oneOf(fullSizeChunks, fullSizeChunks + 1 + (extraChunkSize > lastChunkSizeOfTheFirstSplit ? 1 : 0))));
         assertTrue(capturingHandler.didWriteAfterThrottled);
         assertBufferEquals(Unpooled.compositeBuffer().addComponents(true, seen), message);
     }

muted-tests.yml

@@ -150,9 +150,6 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/117740
 - class: org.elasticsearch.xpack.security.authc.ldap.MultiGroupMappingIT
   issue: https://github.com/elastic/elasticsearch/issues/119599
-- class: org.elasticsearch.search.profile.dfs.DfsProfilerIT
-  method: testProfileDfs
-  issue: https://github.com/elastic/elasticsearch/issues/119711
 - class: org.elasticsearch.multi_cluster.MultiClusterYamlTestSuiteIT
   issue: https://github.com/elastic/elasticsearch/issues/119983
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
@@ -216,41 +213,14 @@ tests:
 - class: org.elasticsearch.action.search.SearchProgressActionListenerIT
   method: testSearchProgressWithQuery
   issue: https://github.com/elastic/elasticsearch/issues/120994
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
-  method: testSuggestProfilesWithName
-  issue: https://github.com/elastic/elasticsearch/issues/121022
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
-  method: testProfileAPIsWhenIndexNotCreated
-  issue: https://github.com/elastic/elasticsearch/issues/121096
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
-  method: testGetProfiles
-  issue: https://github.com/elastic/elasticsearch/issues/121101
-- class: org.elasticsearch.xpack.security.authc.service.ServiceAccountSingleNodeTests
-  method: testAuthenticateWithServiceFileToken
-  issue: https://github.com/elastic/elasticsearch/issues/120988
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
-  method: testUpdateProfileData
-  issue: https://github.com/elastic/elasticsearch/issues/121108
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
   method: test {p0=nodes.stats/11_indices_metrics/indices mappings exact count test for indices level}
   issue: https://github.com/elastic/elasticsearch/issues/120950
-- class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests
-  method: testActivateProfileForJWT
-  issue: https://github.com/elastic/elasticsearch/issues/120983
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
-  method: testProfileIndexAutoCreation
-  issue: https://github.com/elastic/elasticsearch/issues/120987
 - class: org.elasticsearch.xpack.security.FileSettingsRoleMappingsRestartIT
   method: testFileSettingsReprocessedOnRestartWithoutVersionChange
   issue: https://github.com/elastic/elasticsearch/issues/120964
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
-  method: testGetUsersWithProfileUidWhenProfileIndexDoesNotExists
-  issue: https://github.com/elastic/elasticsearch/issues/121179
 - class: org.elasticsearch.xpack.ml.integration.PyTorchModelIT
   issue: https://github.com/elastic/elasticsearch/issues/121165
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
-  method: testSetEnabled
-  issue: https://github.com/elastic/elasticsearch/issues/121183
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/*}
   issue: https://github.com/elastic/elasticsearch/issues/120816
@@ -289,29 +259,8 @@ tests:
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/snapshot-restore/apis/get-snapshot-api/line_751}
   issue: https://github.com/elastic/elasticsearch/issues/121345
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
-  method: testHasPrivileges
-  issue: https://github.com/elastic/elasticsearch/issues/121346
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
-  method: testActivateProfile
-  issue: https://github.com/elastic/elasticsearch/issues/121151
 - class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
   issue: https://github.com/elastic/elasticsearch/issues/121407
-- class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests
-  method: testClientSecretRotation
-  issue: https://github.com/elastic/elasticsearch/issues/120985
-- class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests
-  method: testGrantApiKeyForJWT
-  issue: https://github.com/elastic/elasticsearch/issues/121039
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
-  method: testGetUsersWithProfileUid
-  issue: https://github.com/elastic/elasticsearch/issues/121483
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
-  method: testSuggestProfilesWithHint
-  issue: https://github.com/elastic/elasticsearch/issues/121116
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
-  method: testSuggestProfileWithData
-  issue: https://github.com/elastic/elasticsearch/issues/121258
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/cat/health/cat-health-no-timestamp-example}
   issue: https://github.com/elastic/elasticsearch/issues/121867
@@ -395,6 +344,68 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/122566
 - class: org.elasticsearch.entitlement.qa.EntitlementsDeniedNonModularIT
   issue: https://github.com/elastic/elasticsearch/issues/122569
+- class: org.elasticsearch.entitlement.qa.EntitlementsAllowedIT
+  issue: https://github.com/elastic/elasticsearch/issues/122680
+- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
+  method: test {yaml=reference/snapshot-restore/apis/get-snapshot-api/line_408}
+  issue: https://github.com/elastic/elasticsearch/issues/122681
+- class: org.elasticsearch.xpack.autoscaling.storage.ReactiveStorageIT
+  method: testScaleWhileShrinking
+  issue: https://github.com/elastic/elasticsearch/issues/122119
+- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
+  method: testIndexUpgrade {p0=[9.1.0, 8.19.0, 8.19.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/122688
+- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
+  method: testRestoreIndex {p0=[9.1.0, 9.1.0, 8.19.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/122689
+- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
+  method: testClosedIndexUpgrade {p0=[9.1.0, 8.19.0, 8.19.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/122690
+- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
+  method: testRestoreIndex {p0=[9.1.0, 8.19.0, 8.19.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/122691
+- class: org.elasticsearch.xpack.searchablesnapshots.FrozenSearchableSnapshotsIntegTests
+  method: testCreateAndRestorePartialSearchableSnapshot
+  issue: https://github.com/elastic/elasticsearch/issues/122693
+- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
+  method: testClosedIndexUpgrade {p0=[9.1.0, 9.1.0, 8.19.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/122694
+- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
+  method: testClosedIndexUpgrade {p0=[9.1.0, 9.1.0, 9.1.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/122695
+- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
+  method: testIndexUpgrade {p0=[9.1.0, 9.1.0, 8.19.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/122696
+- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
+  method: testIndexUpgrade {p0=[9.1.0, 9.1.0, 9.1.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/122697
+- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
+  method: testRestoreIndex {p0=[9.1.0, 9.1.0, 9.1.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/122698
+- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
+  method: testSearchableSnapshotUpgrade {p0=[9.1.0, 8.19.0, 8.19.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/122700
+- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
+  method: testSearchableSnapshotUpgrade {p0=[9.1.0, 9.1.0, 8.19.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/122701
+- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
+  method: testMountSearchableSnapshot {p0=[9.1.0, 8.19.0, 8.19.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/122702
+- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
+  method: testMountSearchableSnapshot {p0=[9.1.0, 9.1.0, 8.19.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/122703
+- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
+  method: testSearchableSnapshotUpgrade {p0=[9.1.0, 9.1.0, 9.1.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/122704
+- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT
+  method: testMountSearchableSnapshot {p0=[9.1.0, 9.1.0, 9.1.0]}
+  issue: https://github.com/elastic/elasticsearch/issues/122705
+- class: org.elasticsearch.search.basic.SearchWithRandomDisconnectsIT
+  method: testSearchWithRandomDisconnects
+  issue: https://github.com/elastic/elasticsearch/issues/122707
+- class: org.elasticsearch.indices.recovery.IndexRecoveryIT
+  method: testSourceThrottling
+  issue: https://github.com/elastic/elasticsearch/issues/122712
 
 # Examples:
 #

server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateNodeRemovalIT.java

@@ -179,28 +179,35 @@ public void testNodeRemovalFromRedClusterWithTimeout() throws Exception {
         // make it red!
         internalCluster().stopNode(node1);
         ensureRed(indexName);
+        CountDownLatch stallPrevalidateShardPathActionLatch = new CountDownLatch(1);
         MockTransportService.getInstance(node2)
             .addRequestHandlingBehavior(TransportPrevalidateShardPathAction.ACTION_NAME + "[n]", (handler, request, channel, task) -> {
                 logger.info("drop the check shards request");
+                safeAwait(stallPrevalidateShardPathActionLatch);
+                handler.messageReceived(request, channel, task);
             });
-        PrevalidateNodeRemovalRequest req = PrevalidateNodeRemovalRequest.builder()
-            .setNames(node2)
-            .build(TEST_REQUEST_TIMEOUT)
-            .masterNodeTimeout(TimeValue.timeValueSeconds(1))
-            .timeout(TimeValue.timeValueSeconds(1));
-        PrevalidateNodeRemovalResponse resp = client().execute(PrevalidateNodeRemovalAction.INSTANCE, req).get();
-        assertFalse("prevalidation result should return false", resp.getPrevalidation().isSafe());
-        String node2Id = getNodeId(node2);
-        assertThat(
-            resp.getPrevalidation().message(),
-            equalTo("cannot prevalidate removal of nodes with the following IDs: [" + node2Id + "]")
-        );
-        assertThat(resp.getPrevalidation().nodes().size(), equalTo(1));
-        NodesRemovalPrevalidation.NodeResult nodeResult = resp.getPrevalidation().nodes().get(0);
-        assertThat(nodeResult.name(), equalTo(node2));
-        assertFalse(nodeResult.result().isSafe());
-        assertThat(nodeResult.result().message(), startsWith("failed contacting the node"));
-        assertThat(nodeResult.result().reason(), equalTo(NodesRemovalPrevalidation.Reason.UNABLE_TO_VERIFY));
+        try {
+            PrevalidateNodeRemovalRequest req = PrevalidateNodeRemovalRequest.builder()
+                .setNames(node2)
+                .build(TEST_REQUEST_TIMEOUT)
+                .masterNodeTimeout(TimeValue.timeValueSeconds(1))
+                .timeout(TimeValue.timeValueSeconds(1));
+            PrevalidateNodeRemovalResponse resp = client().execute(PrevalidateNodeRemovalAction.INSTANCE, req).get();
+            assertFalse("prevalidation result should return false", resp.getPrevalidation().isSafe());
+            String node2Id = getNodeId(node2);
+            assertThat(
+                resp.getPrevalidation().message(),
+                equalTo("cannot prevalidate removal of nodes with the following IDs: [" + node2Id + "]")
+            );
+            assertThat(resp.getPrevalidation().nodes().size(), equalTo(1));
+            NodesRemovalPrevalidation.NodeResult nodeResult = resp.getPrevalidation().nodes().get(0);
+            assertThat(nodeResult.name(), equalTo(node2));
+            assertFalse(nodeResult.result().isSafe());
+            assertThat(nodeResult.result().message(), startsWith("failed contacting the node"));
+            assertThat(nodeResult.result().reason(), equalTo(NodesRemovalPrevalidation.Reason.UNABLE_TO_VERIFY));
+        } finally {
+            stallPrevalidateShardPathActionLatch.countDown();
+        }
     }
 
     private void ensureRed(String indexName) throws Exception {

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -176,6 +176,8 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_INFERENCE_IBM_WATSONX_RERANK_ADDED = def(8_840_0_00);
     public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED_BACKPORT_8_X = def(8_840_0_01);
     public static final TransportVersion REMOVE_ALL_APPLICABLE_SELECTOR_BACKPORT_8_X = def(8_840_0_02);
+    public static final TransportVersion ESQL_RETRY_ON_SHARD_LEVEL_FAILURE_BACKPORT_8_19 = def(8_840_0_03);
+    public static final TransportVersion ESQL_SUPPORT_PARTIAL_RESULTS_BACKPORT_8_19 = def(8_840_0_04);
     public static final TransportVersion ELASTICSEARCH_9_0 = def(9_000_0_00);
     public static final TransportVersion REMOVE_SNAPSHOT_FAILURES_90 = def(9_000_0_01);
     public static final TransportVersion TRANSPORT_STATS_HANDLING_TIME_REQUIRED_90 = def(9_000_0_02);

server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java

@@ -387,7 +387,7 @@ public CanMatchNodeRequest.Shard buildShardLevelRequest(SearchShardIterator shar
     }
 
     public void start() {
-        if (shardsIts.size() == 0) {
+        if (shardsIts.isEmpty()) {
             finishPhase();
             return;
         }

server/src/main/java/org/elasticsearch/action/search/SearchShardIterator.java

@@ -11,7 +11,6 @@
 
 import org.elasticsearch.action.OriginalIndices;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.common.util.PlainIterator;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
@@ -113,15 +112,6 @@ SearchShardTarget nextOrNull() {
         return null;
     }
 
-    /**
-     * Return the number of shards remaining in this {@link ShardsIterator}
-     *
-     * @return number of shard remaining
-     */
-    int remaining() {
-        return targetNodesIterator.remaining();
-    }
-
     /**
      * Returns a non-null value if this request should use a specific search context instead of the latest one.
      */