Skip spec tests when test cluster broken (#135862) (#135986)

If the test clusters are broken, we should stop running spec tests; 
otherwise we will see a lot of failures, making triage and investigating
more difficult.

Author: dnhatn

x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java

@@ -53,6 +53,7 @@ protected boolean supportsSourceFieldMapping() {
 
     @Before
     public void configureChunks() throws IOException {
+        assumeTrue("test clusters were broken", testClustersOk);
         boolean smallChunks = randomBoolean();
         Request request = new Request("PUT", "/_cluster/settings");
         XContentBuilder builder = JsonXContent.contentBuilder().startObject().startObject("persistent");

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java

@@ -122,9 +122,11 @@ protected EsqlSpecTestCase(
     }
 
     private static boolean dataLoaded = false;
+    protected static boolean testClustersOk = true;
 
     @Before
     public void setup() throws IOException {
+        assumeTrue("test clusters were broken", testClustersOk);
         boolean supportsLookup = supportsIndexModeLookup();
         boolean supportsSourceMapping = supportsSourceFieldMapping();
         boolean supportsInferenceTestService = supportsInferenceTestService();
@@ -140,6 +142,9 @@ public void setup() throws IOException {
 
     @AfterClass
     public static void wipeTestData() throws IOException {
+        if (testClustersOk == false) {
+            return;
+        }
         try {
             dataLoaded = false;
             adminClient().performRequest(new Request("DELETE", "/*"));
@@ -162,11 +167,25 @@ public final void test() throws Throwable {
             shouldSkipTest(testName);
             doTest();
         } catch (Exception e) {
+            ensureTestClustersAreOk(e);
             throw reworkException(e);
         }
     }
 
+    protected void ensureTestClustersAreOk(Exception failure) {
+        try {
+            ensureHealth(client(), "", (request) -> {
+                request.addParameter("wait_for_status", "yellow");
+                request.addParameter("level", "shards");
+            });
+        } catch (Exception inner) {
+            testClustersOk = false;
+            failure.addSuppressed(inner);
+        }
+    }
+
     protected void shouldSkipTest(String testName) throws IOException {
+        assumeTrue("test clusters were broken", testClustersOk);
         if (requiresInferenceEndpoint()) {
             assumeTrue("Inference test service needs to be supported", supportsInferenceTestService());
         }
@@ -374,7 +393,9 @@ protected boolean preserveClusterUponCompletion() {
 
     @After
     public void assertRequestBreakerEmptyAfterTests() throws Exception {
-        assertRequestBreakerEmpty();
+        if (testClustersOk) {
+            assertRequestBreakerEmpty();
+        }
     }
 
     public static void assertRequestBreakerEmpty() throws Exception {