After a HeapAttack failure, skip remaining tests (#142708)

Adds a utility to `ESTestCase` to allow skipping remaining tests after a
single failure. Then uses it in the `HeapAttackIT`.

This is good because `HeapAttackIT` failures *typically* poison the
cluster under test, causing subsequent tests to fail. This poisons our
failure reports and confuses our muting robot. And confuses anyone
reviewing the failure. Skipping the remaining tests should make them
easier to read for the robots and humans.

Author: nik9000

test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackTestCase.java

@@ -54,10 +54,18 @@ public abstract class HeapAttackTestCase extends ESRestTestCase {
     @ClassRule
     public static ElasticsearchCluster cluster = Clusters.buildCluster();
 
-    static volatile boolean SUITE_ABORTED = false;
-
     protected static final int MAX_ATTEMPTS = 5;
 
+    @Override
+    protected boolean shouldFailureSkipRemainingTests() {
+        /*
+         * Failures will frequently poison the cluster being tested, causing the next
+         * test to fail because the cluster has OOMed or exploded in some fun way. So
+         * we skip them.
+         */
+        return true;
+    }
+
     protected interface TryCircuitBreaking {
         Map<String, Object> attempt(int attempt) throws IOException;
     }
@@ -67,11 +75,6 @@ protected String getTestRestCluster() {
         return cluster.getHttpAddresses();
     }
 
-    @Before
-    public void skipOnAborted() {
-        assumeFalse("skip on aborted", SUITE_ABORTED);
-    }
-
     protected void assertCircuitBreaks(TryCircuitBreaking tryBreaking) throws IOException {
         assertCircuitBreaks(
             tryBreaking,
@@ -125,7 +128,6 @@ public void onFailure(Exception e) {
 
                 @Override
                 protected void doRun() throws Exception {
-                    SUITE_ABORTED = true;
                     TimeValue elapsed = TimeValue.timeValueNanos(System.nanoTime() - startedTimeInNanos);
                     logger.info("--> test {} triggering OOM after {}", getTestName(), elapsed);
                     Request triggerOOM = new Request("POST", "/_trigger_out_of_memory");
@@ -301,7 +303,7 @@ protected static void assertWriteResponse(Response response) throws IOException
     @Before
     @After
     public void assertRequestBreakerEmpty() throws Exception {
-        if (SUITE_ABORTED) {
+        if (previousFailureSkipsRemaining()) {
             return;
         }
         assertBusy(() -> {

test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java

@@ -166,6 +166,8 @@
 import org.junit.internal.AssumptionViolatedException;
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -233,6 +235,7 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.startsWith;
+import static org.junit.Assume.assumeFalse;
 
 /**
  * Base testcase for randomized unit testing with Elasticsearch
@@ -3182,4 +3185,44 @@ public static BytesRef embedInRandomBytes(BytesRef bytesRef) {
 
         return new BytesRef(newBytesArray, offset, bytesRef.length);
     }
+
+    private static boolean previousFailureSkipsRemaining;
+    @Rule
+    public final TestWatcher previousFailureSkipsRemainingRule = new TestWatcher() {
+        @Override
+        protected void failed(Throwable e, Description description) {
+            previousFailureSkipsRemaining = shouldFailureSkipRemainingTests();
+        }
+    };
+
+    @Before
+    public final void checkPreviousFailureSkipsRemaining() {
+        assumeFalse("previous failures broke system under test", previousFailureSkipsRemaining);
+    }
+
+    /**
+     * Should a failure cause subsequent tests to be skipped?
+     */
+    protected boolean shouldFailureSkipRemainingTests() {
+        return false;
+    }
+
+    /**
+     * Have previous failures forced us to skip the test?
+     * <p>
+     *     This should only be used in rare cases where the system being tested
+     *     is typically poisoned by test failures. ESQL's HeapAttack tests are
+     *     like this. As are packaging tests.
+     * </p>
+     * <p>
+     *     If you find yourself reaching for this, ask yourself if it's the right
+     *     tool three times before actually picking it up. If you are writing a
+     *     unit test without dependencies this is almost certainly not the right
+     *     tool.
+     * </p>
+     */
+    protected boolean previousFailureSkipsRemaining() {
+        return previousFailureSkipsRemaining;
+    }
+
 }

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -641,7 +641,7 @@ protected HttpHost buildHttpHost(String host, int port) {
      */
     @After
     public final void cleanUpCluster() throws Exception {
-        if (preserveClusterUponCompletion() == false) {
+        if (previousFailureSkipsRemaining() == false && preserveClusterUponCompletion() == false) {
             ensureNoInitializingShards();
             wipeCluster();
             waitForClusterStateUpdatesToFinish();