Split the CCS YAML rest tests into a multiple tests (elastic#139107)

Running full test set in a single IT file can lead to suite timeouts on 
slow machines. This moves the search APIs tests into a different IT.

Author: davidkyle

qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CcsCommonYamlTestSuiteIT.java

@@ -66,6 +66,11 @@
  * defined in CCS_APIS against the "search" cluster, while all other operations like indexing are performed
  * using the client running against the "write" cluster.
  *
+ * Running all the YAML tests in a single test suite can lead to the suite timing out.
+ * To avoid timeouts subsets of the tests are executed in specific test suites according
+ * to the logic in {@link TestSuiteApiCheck}. To further split the tests add another suite
+ * by subclassing this class then add an entry to {@link TestSuiteApiCheck} mapping the API
+ * name(s) to the new class.
  */
 @TimeoutSuite(millis = 20 * TimeUnits.MINUTE) // to account for slow as hell VMs
 public class CcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
@@ -297,6 +302,17 @@ public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
 
+    @Override
+    public void test() throws IOException {
+        boolean shouldBeExecutedByThisSuite = TestSuiteApiCheck.shouldExecuteTest(this, getTestCandidate().getApi());
+        assumeTrue(
+            "Skipping test as the API [" + getTestCandidate().getApi() + "] is not covered by this suite",
+            shouldBeExecutedByThisSuite
+        );
+
+        super.test();
+    }
+
     @Override
     protected ClientYamlTestExecutionContext createRestTestExecutionContext(
         ClientYamlTestCandidate clientYamlTestCandidate,

qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/CssSearchYamlTestSuiteIT.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.test.rest.yaml;
+
+import java.io.IOException;
+
+/**
+ * Runs the search and msearch API tests only. See {@link TestSuiteApiCheck}
+ */
+public class CssSearchYamlTestSuiteIT extends CcsCommonYamlTestSuiteIT {
+    public CssSearchYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) throws IOException {
+        super(testCandidate);
+    }
+}

qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsCommonYamlTestSuiteIT.java

@@ -62,6 +62,11 @@
  * defined in CCS_APIS against the "search" cluster, while all other operations like indexing are performed
  * using the client running against the "write" cluster.
  *
+ * Running all the YAML tests in a single test suite can lead to the suite timing out.
+ * To avoid timeouts subsets of the tests are executed in specific test suites according
+ * to the logic in {@link TestSuiteApiCheck}. To further split the tests add another suite
+ * by subclassing this class then add an entry to {@link TestSuiteApiCheck} mapping the API
+ * name(s) to the new class.
  */
 @TimeoutSuite(millis = 25 * TimeUnits.MINUTE) // to account for slow as hell VMs
 public class RcsCcsCommonYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
@@ -278,6 +283,17 @@ public static Iterable<Object[]> parameters() throws Exception {
         return createParameters();
     }
 
+    @Override
+    public void test() throws IOException {
+        boolean shouldBeExecutedByThisSuite = TestSuiteApiCheck.shouldExecuteTest(this, getTestCandidate().getApi());
+        assumeTrue(
+            "Skipping test as the API [" + getTestCandidate().getApi() + "] is not covered by this suite",
+            shouldBeExecutedByThisSuite
+        );
+
+        super.test();
+    }
+
     @Override
     protected ClientYamlTestExecutionContext createRestTestExecutionContext(
         ClientYamlTestCandidate clientYamlTestCandidate,

qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/RcsCcsSearchYamlTestSuiteIT.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.test.rest.yaml;
+
+import java.io.IOException;
+
+public class RcsCcsSearchYamlTestSuiteIT extends RcsCcsCommonYamlTestSuiteIT {
+    public RcsCcsSearchYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) throws IOException {
+        super(testCandidate);
+    }
+}

qa/ccs-common-rest/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/TestSuiteApiCheck.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.test.rest.yaml;
+
+/**
+ * Contains the logic for which test suites should execute which tests.
+ * YAML tests are split between multiple suites to avoid the test suite timing out.
+ *
+ * IMPORTANT: This is a temporary solution to reduce the number of test failures
+ * due to suite timeouts for this large test set which suffers the problem more
+ * than most. It is not a pattern that should be copied or applied elsewhere.
+ * A longer term solution would be the test runner automatically splitting the
+ * tests.
+ */
+class TestSuiteApiCheck {
+
+    /**
+     * Returns true if the test suite should run the tests for the given API.
+     * @param testSuite a concrete subclass of ESClientYamlSuiteTestCase
+     * @param apiName The API name as described in the rest spec e.g. `search`
+     * @return True if the test
+     */
+    public static boolean shouldExecuteTest(ESClientYamlSuiteTestCase testSuite, String apiName) {
+        return switch (testSuite) {
+            case CssSearchYamlTestSuiteIT cssSearch -> isSearchApi(apiName);
+            case RcsCcsSearchYamlTestSuiteIT rssSearch -> isSearchApi(apiName);
+            default -> true;
+        };
+    }
+
+    private static boolean isSearchApi(String apiName) {
+        return apiName.startsWith("search") || apiName.startsWith("msearch");
+    }
+
+    private TestSuiteApiCheck() {}
+}