Add some utilties to run search queries in parallel in ITs (#115590) (#117330)

We have loads of tests that assert the same thing about a number of
different queries. This introduces some tooling to run some of these
spots in parallel.
I only changed a couple of examples in the tests for now, but in general
this could be used to save thousands of lines of test code and more
importantly, get some coverage on parallel query execution which is
covered very little today.

Author: original-brownbear

plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java

@@ -25,6 +25,7 @@
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponses;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
@@ -136,14 +137,11 @@ public void testWildCardWithFieldsWhenDisabled() throws Exception {
         assertAcked(prepareCreate("test").setMapping("_size", "enabled=false"));
         final String source = "{\"f\":\"" + randomAlphaOfLengthBetween(1, 100) + "\"}";
         indexRandom(true, prepareIndex("test").setId("1").setSource(source, XContentType.JSON));
-        assertResponse(
+        assertResponses(
+            response -> assertNull(response.getHits().getHits()[0].getFields().get("_size")),
             prepareSearch("test").addFetchField("_size"),
-            response -> assertNull(response.getHits().getHits()[0].getFields().get("_size"))
-        );
-
-        assertResponse(
             prepareSearch("test").addFetchField("*"),
-            response -> assertNull(response.getHits().getHits()[0].getFields().get("_size"))
+            prepareSearch("test").addStoredField("*")
         );
 
         assertResponse(
@@ -156,19 +154,11 @@ public void testWildCardWithFieldsWhenNotProvided() throws Exception {
         assertAcked(prepareCreate("test"));
         final String source = "{\"f\":\"" + randomAlphaOfLengthBetween(1, 100) + "\"}";
         indexRandom(true, prepareIndex("test").setId("1").setSource(source, XContentType.JSON));
-        assertResponse(
+        assertResponses(
+            response -> assertNull(response.getHits().getHits()[0].getFields().get("_size")),
             prepareSearch("test").addFetchField("_size"),
-            response -> assertNull(response.getHits().getHits()[0].getFields().get("_size"))
-        );
-
-        assertResponse(
             prepareSearch("test").addFetchField("*"),
-            response -> assertNull(response.getHits().getHits()[0].getFields().get("_size"))
-        );
-
-        assertResponse(
-            prepareSearch("test").addStoredField("*"),
-            response -> assertNull(response.getHits().getHits()[0].getFields().get("_size"))
+            prepareSearch("test").addStoredField("*")
         );
     }
 }

server/src/internalClusterTest/java/org/elasticsearch/search/query/MultiMatchQueryIT.java

@@ -57,6 +57,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponses;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHitsWithoutFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSecondHit;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
@@ -772,26 +773,21 @@ public void testCrossFieldMode() throws ExecutionException, InterruptedException
         );
         // counter example
         assertHitCount(
+            0L,
             prepareSearch("test").setQuery(
                 randomizeType(
                     multiMatchQuery("captain america marvel hero", "first_name", "last_name", "category").type(
                         randomBoolean() ? MultiMatchQueryBuilder.Type.CROSS_FIELDS : MultiMatchQueryBuilder.DEFAULT_TYPE
                     ).operator(Operator.AND)
                 )
             ),
-            0L
-        );
-
-        // counter example
-        assertHitCount(
             prepareSearch("test").setQuery(
                 randomizeType(
                     multiMatchQuery("captain america marvel hero", "first_name", "last_name", "category").type(
                         randomBoolean() ? MultiMatchQueryBuilder.Type.CROSS_FIELDS : MultiMatchQueryBuilder.DEFAULT_TYPE
                     ).operator(Operator.AND)
                 )
-            ),
-            0L
+            )
         );
 
         // test if boosts work
@@ -828,40 +824,21 @@ public void testCrossFieldMode() throws ExecutionException, InterruptedException
             }
         );
         // Test group based on numeric fields
-        assertResponse(
+        assertResponses(response -> {
+            assertHitCount(response, 1L);
+            assertFirstHit(response, hasId("theone"));
+        },
             prepareSearch("test").setQuery(randomizeType(multiMatchQuery("15", "skill").type(MultiMatchQueryBuilder.Type.CROSS_FIELDS))),
-            response -> {
-                assertHitCount(response, 1L);
-                assertFirstHit(response, hasId("theone"));
-            }
-        );
-        assertResponse(
             prepareSearch("test").setQuery(
                 randomizeType(multiMatchQuery("15", "skill", "first_name").type(MultiMatchQueryBuilder.Type.CROSS_FIELDS))
             ),
-            response -> {
-                assertHitCount(response, 1L);
-                assertFirstHit(response, hasId("theone"));
-            }
-        );
-        // Two numeric fields together caused trouble at one point!
-        assertResponse(
+            // Two numeric fields together caused trouble at one point!
             prepareSearch("test").setQuery(
                 randomizeType(multiMatchQuery("15", "int-field", "skill").type(MultiMatchQueryBuilder.Type.CROSS_FIELDS))
             ),
-            response -> {
-                assertHitCount(response, 1L);
-                assertFirstHit(response, hasId("theone"));
-            }
-        );
-        assertResponse(
             prepareSearch("test").setQuery(
                 randomizeType(multiMatchQuery("15", "int-field", "first_name", "skill").type(MultiMatchQueryBuilder.Type.CROSS_FIELDS))
-            ),
-            response -> {
-                assertHitCount(response, 1L);
-                assertFirstHit(response, hasId("theone"));
-            }
+            )
         );
         assertResponse(
             prepareSearch("test").setQuery(