Wait for search tasks to disappear before test finishes

Author: benchaplin

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -1369,16 +1369,13 @@ private static void wipeClusterSettings() throws IOException {
 
         if (mustClear.get()) {
             request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(warnings -> {
-                for (String warning : warnings) {
-                    if (warning.contains("xpack.monitoring")) {
-                        SUITE_LOGGER.warn("Ignoring xpack monitoring warning during test cleanup: {}", warning);
-                    } else if (warning.contains("name begins with a dot")) {
-                        SUITE_LOGGER.warn("Ignoring dot prefix warning during test cleanup: {}", warning);
-                    } else {
-                        return true;
-                    }
+                if (warnings.isEmpty()) {
+                    return false;
+                } else if (warnings.size() > 1) {
+                    return true;
+                } else {
+                    return warnings.get(0).contains("xpack.monitoring") == false;
                 }
-                return false;
             }));
             cleanupClient().performRequest(request);
         }

x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRestIT.java

@@ -155,6 +155,7 @@ public void testTaskCancellation() throws Exception {
         final String indexName = "index_fulfilling";
         final String roleName = "taskCancellationRoleName";
         final String userName = "taskCancellationUsername";
+        String asyncSearchOpaqueId = "async-search-opaque-id-" + randomUUID();
         try {
             // create some index on the fulfilling cluster, to be searched from the querying cluster
             {
@@ -206,13 +207,12 @@ public void testTaskCancellation() throws Exception {
                         {
                           "name": "*:*",
                           "error_type": "exception",
-                          "stall_time_seconds": 60
+                          "stall_time_seconds": 10
                         }
                       ]
                     }
                   }
                 }""");
-            String asyncSearchOpaqueId = "async-search-opaque-id-" + randomUUID();
             submitAsyncSearchRequest.setOptions(
                 RequestOptions.DEFAULT.toBuilder()
                     .addHeader("Authorization", headerFromRandomAuthMethod(userName, PASS))
@@ -308,6 +308,27 @@ public void testTaskCancellation() throws Exception {
             assertOK(adminClient().performRequest(new Request("DELETE", "/_security/user/" + userName)));
             assertOK(adminClient().performRequest(new Request("DELETE", "/_security/role/" + roleName)));
             assertOK(performRequestAgainstFulfillingCluster(new Request("DELETE", indexName)));
+            // wait for search related tasks to finish on the query cluster
+            assertBusy(() -> {
+                try {
+                    Response queryingClusterTasks = adminClient().performRequest(new Request("GET", "/_tasks"));
+                    assertOK(queryingClusterTasks);
+                    Map<String, Object> responseMap = XContentHelper.convertToMap(
+                        JsonXContent.jsonXContent,
+                        EntityUtils.toString(queryingClusterTasks.getEntity()),
+                        false
+                    );
+                    AtomicBoolean searchTasksFound = new AtomicBoolean(false);
+                    selectTasksWithOpaqueId(responseMap, asyncSearchOpaqueId, task -> {
+                        if (task.get("action") instanceof String action && action.contains("indices:data/read/search")) {
+                            searchTasksFound.set(true);
+                        }
+                    });
+                    assertFalse("Expected no search tasks to be running", searchTasksFound.get());
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            });
         }
     }