Fix testRoleMigration race condition (elastic#138742)

Author: jfreden

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/SecurityIndexRolesMetadataMigrationIT.java

@@ -63,8 +63,9 @@ public void testRoleMigration() throws Exception {
             assertMigratedDocInSecurityIndex(mixed1TestRole, "meta", "test");
             assertMigratedDocInSecurityIndex(mixed2TestRole, "meta", "test");
             assertMigratedDocInSecurityIndex(upgradedTestRole, "meta", "test");
-            // queries all roles by metadata
-            assertAllRoles(client(), "mixed1-test-role", "mixed2-test-role", "old-test-role", "upgraded-test-role");
+            // query all roles by metadata - use assertBusy to handle the case where the node handling the query is not yet aware of the
+            // successful migration
+            assertBusy(() -> assertAllRoles(client(), "mixed1-test-role", "mixed2-test-role", "old-test-role", "upgraded-test-role"));
         }
     }
 
@@ -184,7 +185,13 @@ private void assertAllRoles(RestClient client, String... roleNames) throws IOExc
             {"query":{"bool":{"must":[{"exists":{"field":"metadata.meta"}}]}},"sort":["name"]}""";
         Request request = new Request(randomFrom("POST", "GET"), "/_security/_query/role");
         request.setJsonEntity(metadataQuery);
-        Response response = client.performRequest(request);
+        Response response = null;
+        try {
+            response = client.performRequest(request);
+        } catch (ResponseException e) {
+            fail(e);
+        }
+        assertNotNull(response);
         assertOK(response);
         Map<String, Object> responseMap = responseAsMap(response);
         assertThat(responseMap.get("total"), is(roleNames.length));