Tighten asserts in FullClusterRestartSystemIndexCompatibilityIT and provide more context for failures (elastic#134754)

Author: lkts

qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartSystemIndexCompatibilityIT.java

@@ -67,7 +67,7 @@ public void testAsyncSearchIndexMigration() throws Exception {
             String asyncId = searchAsyncAndStoreId(asyncSearchRequest, "n-2_id");
             ensureGreen(asyncSearchIndex);
 
-            assertAsyncSearchHitCount(asyncId, numDocs);
+            assertBusy(() -> assertAsyncSearchHitCount(asyncId, numDocs));
             assertBusy(() -> assertDocCountNoWarnings(client(), asyncSearchIndex, 1));
             assertThat(indexVersion(asyncSearchIndex, true), equalTo(VERSION_MINUS_2));
             return;
@@ -103,7 +103,7 @@ public void testAsyncSearchIndexMigration() throws Exception {
 
             // perform new async search and check its readable
             String asyncId = searchAsyncAndStoreId(asyncSearchRequest, "n-1_id");
-            assertAsyncSearchHitCount(asyncId, numDocs);
+            assertBusy(() -> assertAsyncSearchHitCount(asyncId, numDocs));
             assertBusy(() -> assertDocCountNoWarnings(client(), asyncSearchIndex, 2));
 
             // in order to move to current version we need write block for n-2 index
@@ -117,7 +117,7 @@ public void testAsyncSearchIndexMigration() throws Exception {
 
             // check system index is still writeable
             String asyncId = searchAsyncAndStoreId(asyncSearchRequest, "n_id");
-            assertAsyncSearchHitCount(asyncId, numDocs);
+            assertBusy(() -> assertAsyncSearchHitCount(asyncId, numDocs));
             assertBusy(() -> assertDocCountNoWarnings(client(), asyncSearchIndex, 3));
         }
 
@@ -134,7 +134,13 @@ private static String searchAsyncAndStoreId(Request asyncSearchRequest, String a
     private static void assertAsyncSearchHitCount(String asyncId, int numDocs) throws IOException {
         var asyncGet = new Request("GET", "/_async_search/" + asyncId);
         ObjectPath resp = ObjectPath.createFromResponse(client().performRequest(asyncGet));
-        assertEquals(Integer.valueOf(numDocs), resp.evaluate("response.hits.total.value"));
+        assertFalse("Async search is not complete", resp.evaluate("is_running"));
+        assertFalse("Async search has partial results", resp.evaluate("is_partial"));
+        assertEquals(
+            "Unexpected number of hits in search response. Response: \n" + resp.evaluate("response"),
+            Integer.valueOf(numDocs),
+            resp.evaluate("response.hits.total.value")
+        );
     }
 
     /**