[CI] DocsClientYamlTestSuiteIT test {yaml=reference/watcher/example-watches/example-watch-clusterstatus/line_137} failing - (#115809) (#117354)

* Ignore system index access errors in YAML test index cleanup method

* Remove test mute

* Swap the logic back as it was right the first time

* Resolve conflict with latest merge

* Move warning handler into it's own method to reduce nesting

Author: lukewhiting

muted-tests.yml

@@ -100,9 +100,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_start_stop/Verify start transform reuses destination index}
   issue: https://github.com/elastic/elasticsearch/issues/115808
-- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
-  method: test {yaml=reference/watcher/example-watches/example-watch-clusterstatus/line_137}
-  issue: https://github.com/elastic/elasticsearch/issues/115809
 - class: org.elasticsearch.search.StressSearchServiceReaperIT
   method: testStressReaper
   issue: https://github.com/elastic/elasticsearch/issues/115816

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -26,6 +26,8 @@
 import org.apache.http.ssl.SSLContextBuilder;
 import org.apache.http.ssl.SSLContexts;
 import org.apache.http.util.EntityUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Build;
 import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
@@ -158,6 +160,8 @@ public abstract class ESRestTestCase extends ESTestCase {
 
     private static final Pattern SEMANTIC_VERSION_PATTERN = Pattern.compile("^(\\d+\\.\\d+\\.\\d+)\\D?.*");
 
+    private static final Logger SUITE_LOGGER = LogManager.getLogger(ESRestTestCase.class);
+
     /**
      * Convert the entity from a {@link Response} into a map of maps.
      * Consumes the underlying HttpEntity, releasing any resources it may be holding.
@@ -1111,7 +1115,14 @@ protected static void wipeAllIndices(boolean preserveSecurityIndices) throws IOE
             }
             final Request deleteRequest = new Request("DELETE", Strings.collectionToCommaDelimitedString(indexPatterns));
             deleteRequest.addParameter("expand_wildcards", "open,closed,hidden");
-            deleteRequest.setOptions(deleteRequest.getOptions().toBuilder().setWarningsHandler(ignoreAsyncSearchWarning()).build());
+
+            // If system index warning, ignore but log
+            // See: https://github.com/elastic/elasticsearch/issues/117099
+            // and: https://github.com/elastic/elasticsearch/issues/115809
+            deleteRequest.setOptions(
+                RequestOptions.DEFAULT.toBuilder().setWarningsHandler(ESRestTestCase::ignoreSystemIndexAccessWarnings)
+            );
+
             final Response response = adminClient().performRequest(deleteRequest);
             try (InputStream is = response.getEntity().getContent()) {
                 assertTrue((boolean) XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true).get("acknowledged"));
@@ -1124,28 +1135,16 @@ protected static void wipeAllIndices(boolean preserveSecurityIndices) throws IOE
         }
     }
 
-    // Make warnings handler that ignores the .async-search warning since .async-search may randomly appear when async requests are slow
-    // See: https://github.com/elastic/elasticsearch/issues/117099
-    protected static WarningsHandler ignoreAsyncSearchWarning() {
-        return new WarningsHandler() {
-            @Override
-            public boolean warningsShouldFailRequest(List<String> warnings) {
-                if (warnings.isEmpty()) {
-                    return false;
-                }
-                return warnings.equals(
-                    List.of(
-                        "this request accesses system indices: [.async-search], "
-                            + "but in a future major version, direct access to system indices will be prevented by default"
-                    )
-                ) == false;
+    private static boolean ignoreSystemIndexAccessWarnings(List<String> warnings) {
+        for (String warning : warnings) {
+            if (warning.startsWith("this request accesses system indices:")) {
+                SUITE_LOGGER.warn("Ignoring system index access warning during test cleanup: {}", warning);
+            } else {
+                return true;
             }
+        }
 
-            @Override
-            public String toString() {
-                return "ignore .async-search warning";
-            }
-        };
+        return false;
     }
 
     protected static void wipeDataStreams() throws IOException {