Optimize RcsCcsCommonYamlTestSuiteIT and unmute tests

muted-tests.yml

@@ -356,9 +356,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.inference.bulk.BulkInferenceExecutorTests
   method: testSuccessfulExecution
   issue: https://github.com/elastic/elasticsearch/issues/130306
-- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
-  method: test {p0=msearch/20_typed_keys/Multisearch test with typed_keys parameter for sampler and significant terms}
-  issue: https://github.com/elastic/elasticsearch/issues/130472
 - class: org.elasticsearch.gradle.LoggedExecFuncTest
   method: failed tasks output logged to console when spooling true
   issue: https://github.com/elastic/elasticsearch/issues/119509
@@ -380,9 +377,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.ccq.MultiClustersIT
   method: testLookupJoinAliases
   issue: https://github.com/elastic/elasticsearch/issues/131166
-- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
-  method: test {p0=field_caps/40_time_series/Get simple time series field caps}
-  issue: https://github.com/elastic/elasticsearch/issues/131225
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test090SecurityCliPackaging
   issue: https://github.com/elastic/elasticsearch/issues/131107
@@ -398,9 +392,6 @@ tests:
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test171AdditionalCliOptionsAreForwarded
   issue: https://github.com/elastic/elasticsearch/issues/120925
-- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
-  method: test {p0=search/110_field_collapsing/field collapsing, inner_hits and maxConcurrentGroupRequests}
-  issue: https://github.com/elastic/elasticsearch/issues/131348
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/delete_expired_data/Test delete expired data with body parameters}
   issue: https://github.com/elastic/elasticsearch/issues/131364
@@ -422,9 +413,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
   method: testLookupExplosionNoFetch
   issue: https://github.com/elastic/elasticsearch/issues/128720
-- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
-  method: test {p0=vector-tile/20_aggregations/stats agg}
-  issue: https://github.com/elastic/elasticsearch/issues/131484
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test050BasicApiTests
   issue: https://github.com/elastic/elasticsearch/issues/120911
@@ -443,9 +431,6 @@ tests:
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test010Install
   issue: https://github.com/elastic/elasticsearch/issues/131376
-- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
-  method: test {p0=search/40_indices_boost/Indices boost with alias}
-  issue: https://github.com/elastic/elasticsearch/issues/131598
 - class: org.elasticsearch.compute.lucene.read.SortedSetOrdinalsBuilderTests
   method: testReader
   issue: https://github.com/elastic/elasticsearch/issues/131573
@@ -458,6 +443,9 @@ tests:
 - class: org.elasticsearch.xpack.restart.FullClusterRestartIT
   method: testWatcherWithApiKey {cluster=UPGRADED}
   issue: https://github.com/elastic/elasticsearch/issues/131964
+- class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS1EnrichUnavailableRemotesIT
+  method: testEsqlEnrichWithSkipUnavailable
+  issue: https://github.com/elastic/elasticsearch/issues/132078
 - class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
   method: test {p0=search/600_flattened_ignore_above/flattened ignore_above multi-value field}
   issue: https://github.com/elastic/elasticsearch/issues/131967

qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java

@@ -47,6 +47,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -75,6 +76,11 @@ public class RcsCcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
     // the remote cluster is the one we write index operations etc... to
     private static final String REMOTE_CLUSTER_NAME = "remote_cluster";
     private static final AtomicReference<Map<String, Object>> API_KEY_MAP_REF = new AtomicReference<>();
+    private static final AtomicBoolean isRemoteConfigured = new AtomicBoolean(false);
+    private static final AtomicBoolean isCombinedComputed = new AtomicBoolean(false);
+    private static final AtomicReference<TestFeatureService> combinedTestFeatureServiceRef = new AtomicReference<>();
+    private static final AtomicReference<Set<String>> combinedOsSetRef = new AtomicReference<>();
+    private static final AtomicReference<Set<String>> combinedNodeVersionsRef = new AtomicReference<>();
 
     private static LocalClusterConfigProvider commonClusterConfig = cluster -> cluster.module("x-pack-async-search")
         .module("aggregations")
@@ -217,7 +223,7 @@ public void initSearchClient() throws IOException {
                         basicAuthHeaderValue("remote_search_user", new SecureString("x-pack-test-password".toCharArray()))
                     )
                     .build(),
-                clusterHosts.toArray(new HttpHost[clusterHosts.size()])
+                clusterHosts.toArray(new HttpHost[0])
             );
             adminSearchClient = buildClient(
                 Settings.builder()
@@ -226,13 +232,18 @@ public void initSearchClient() throws IOException {
                         basicAuthHeaderValue("test_admin", new SecureString("x-pack-test-password".toCharArray()))
                     )
                     .build(),
-                clusterHosts.toArray(new HttpHost[clusterHosts.size()])
+                clusterHosts.toArray(new HttpHost[0])
             );
 
             searchYamlTestClient = new TestCandidateAwareClient(getRestSpec(), searchClient, hosts, this::getClientBuilderWithSniffedHosts);
+        }
+
+        assert searchClient != null;
+        assert adminSearchClient != null;
+        assert clusterHosts != null;
 
+        if (isRemoteConfigured.compareAndSet(false, true)) {
             configureRemoteCluster();
-            // check that we have an established CCS connection
             Request request = new Request("GET", "_remote/info");
             Response response = adminSearchClient.performRequest(request);
             assertOK(response);
@@ -241,11 +252,6 @@ public void initSearchClient() throws IOException {
             assertEquals("::es_redacted::", responseObject.evaluate(REMOTE_CLUSTER_NAME + ".cluster_credentials"));
             logger.info("Established connection to remote cluster [" + REMOTE_CLUSTER_NAME + "]");
         }
-
-        assert searchClient != null;
-        assert adminSearchClient != null;
-        assert clusterHosts != null;
-
         searchYamlTestClient.setTestCandidate(getTestCandidate());
     }
 
@@ -288,42 +294,47 @@ protected ClientYamlTestExecutionContext createRestTestExecutionContext(
             // Ensure the test specific initialization is run by calling it explicitly (@Before annotations on base-derived class may
             // be called in a different order)
             initSearchClient();
-            // Reconcile and provide unified features, os, version(s), based on both clientYamlTestClient and searchYamlTestClient
-            var searchOs = readOsFromNodesInfo(adminSearchClient);
-            var searchNodeVersions = readVersionsFromNodesInfo(adminSearchClient);
-            var semanticNodeVersions = searchNodeVersions.stream()
-                .map(ESRestTestCase::parseLegacyVersion)
-                .flatMap(Optional::stream)
-                .collect(Collectors.toSet());
-            final TestFeatureService searchTestFeatureService = createTestFeatureService(
-                getClusterStateFeatures(adminSearchClient),
-                semanticNodeVersions
-            );
-            final TestFeatureService combinedTestFeatureService = (featureId, any) -> {
-                boolean adminFeature = testFeatureService.clusterHasFeature(featureId, any);
-                boolean searchFeature = searchTestFeatureService.clusterHasFeature(featureId, any);
-                return any ? adminFeature || searchFeature : adminFeature && searchFeature;
-            };
 
-            final Set<String> combinedOsSet = Stream.concat(osSet.stream(), Stream.of(searchOs)).collect(Collectors.toSet());
-            final Set<String> combinedNodeVersions = Stream.concat(nodesVersions.stream(), searchNodeVersions.stream())
-                .collect(Collectors.toSet());
+            // Compute & cache combined features/OS/versions
+            if (isCombinedComputed.compareAndSet(false, true)) {
+                var searchOs = readOsFromNodesInfo(adminSearchClient);
+                var searchNodeVersions = readVersionsFromNodesInfo(adminSearchClient);
+
+                var semanticNodeVersions = searchNodeVersions.stream()
+                    .map(ESRestTestCase::parseLegacyVersion)
+                    .flatMap(Optional::stream)
+                    .collect(Collectors.toSet());
+
+                final TestFeatureService searchTestFeatureService = createTestFeatureService(
+                    getClusterStateFeatures(adminSearchClient),
+                    semanticNodeVersions
+                );
+
+                final TestFeatureService combinedTestFeatureService = (featureId, any) -> {
+                    boolean adminFeature = testFeatureService.clusterHasFeature(featureId, any);
+                    boolean searchFeature = searchTestFeatureService.clusterHasFeature(featureId, any);
+                    return any ? (adminFeature || searchFeature) : (adminFeature && searchFeature);
+                };
+                final Set<String> combinedOsSet = Stream.concat(osSet.stream(), Stream.of(searchOs)).collect(Collectors.toSet());
+                final Set<String> combinedNodeVersions = Stream.concat(nodesVersions.stream(), searchNodeVersions.stream())
+                    .collect(Collectors.toSet());
+
+                combinedTestFeatureServiceRef.set(combinedTestFeatureService);
+                combinedOsSetRef.set(combinedOsSet);
+                combinedNodeVersionsRef.set(combinedNodeVersions);
+            }
 
             return new ClientYamlTestExecutionContext(
                 clientYamlTestCandidate,
                 clientYamlTestClient,
                 randomizeContentType(),
-                combinedNodeVersions,
-                combinedTestFeatureService,
-                combinedOsSet
+                combinedNodeVersionsRef.get(),
+                combinedTestFeatureServiceRef.get(),
+                combinedOsSetRef.get()
             ) {
                 // depending on the API called, we either return the client running against the "write" or the "search" cluster here
                 protected ClientYamlTestClient clientYamlTestClient(String apiName) {
-                    if (CCS_APIS.contains(apiName)) {
-                        return searchYamlTestClient;
-                    } else {
-                        return super.clientYamlTestClient(apiName);
-                    }
+                    return CCS_APIS.contains(apiName) ? searchYamlTestClient : super.clientYamlTestClient(apiName);
                 }
             };
         } catch (IOException e) {