Merge branch '9.3' into backport/9.3/pr-142386

Author: luigidellaquila

muted-tests.yml

@@ -171,9 +171,6 @@ tests:
 - class: org.elasticsearch.xpack.security.authc.AuthenticationServiceTests
   method: testInvalidToken
   issue: https://github.com/elastic/elasticsearch/issues/133328
-- class: org.elasticsearch.xpack.search.CrossClusterAsyncSearchIT
-  method: testCancelViaExpirationOnRemoteResultsWithMinimizeRoundtrips
-  issue: https://github.com/elastic/elasticsearch/issues/127302
 - class: org.elasticsearch.xpack.ml.integration.InferenceIT
   method: testInferClassificationModel
   issue: https://github.com/elastic/elasticsearch/issues/133448
@@ -213,9 +210,6 @@ tests:
 - class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
   method: test {p0=search.vectors/200_dense_vector_docvalue_fields/Enable docvalue_fields parameter for dense_vector fields}
   issue: https://github.com/elastic/elasticsearch/issues/136443
-- class: org.elasticsearch.xpack.search.CrossClusterAsyncSearchIT
-  method: testRemoteClusterOnlyCCSWithFailuresOnAllShards
-  issue: https://github.com/elastic/elasticsearch/issues/136894
 - class: org.elasticsearch.xpack.inference.DefaultEndPointsIT
   method: testMultipleInferencesTriggeringDownloadAndDeploy
   issue: https://github.com/elastic/elasticsearch/issues/117208

x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CrossClusterAsyncSearchIT.java

@@ -10,6 +10,7 @@
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse;
@@ -1946,14 +1947,19 @@ private Map<String, Object> setupTwoClusters() {
                 .setSettings(Settings.builder().put(remoteSettings.build()))
                 .setMapping("@timestamp", "type=date", "f", "type=text")
         );
-        assertFalse(
-            client(REMOTE_CLUSTER).admin()
-                .cluster()
-                .prepareHealth(TEST_REQUEST_TIMEOUT, remoteIndex)
-                .setWaitForYellowStatus()
-                .setTimeout(TimeValue.timeValueSeconds(10))
-                .get()
-                .isTimedOut()
+        // Wait for yellow and no initializing shards so all assigned shards are STARTED before tests run searches.
+        ClusterHealthResponse healthResponse = client(REMOTE_CLUSTER).admin()
+            .cluster()
+            .prepareHealth(TEST_REQUEST_TIMEOUT, remoteIndex)
+            .setWaitForYellowStatus()
+            .setWaitForNoInitializingShards(true)
+            .setTimeout(TimeValue.timeValueSeconds(10))
+            .get();
+        assertFalse("remote index health check timed out", healthResponse.isTimedOut());
+        assertEquals(
+            "remote index should have no initializing shards, had: " + healthResponse.getInitializingShards(),
+            0,
+            healthResponse.getInitializingShards()
         );
         indexDocs(client(REMOTE_CLUSTER), remoteIndex);