Fix AllocationFailuresResetOnShutdownIT#testResetAllocationFailuresOnNodeShutdown (#121378)

The ITs assumes that `ensureRed` means we've exhausted all the attempts for allocation, which doesn't hold always. 

Closes #121129

Author: pxsalehi

muted-tests.yml

@@ -268,9 +268,6 @@ tests:
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
   method: test {p0=nodes.stats/11_indices_metrics/indices mappings exact count test for indices level}
   issue: https://github.com/elastic/elasticsearch/issues/120950
-- class: org.elasticsearch.xpack.shutdown.AllocationFailuresResetOnShutdownIT
-  method: testResetAllocationFailuresOnNodeShutdown
-  issue: https://github.com/elastic/elasticsearch/issues/121129
 - class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests
   method: testActivateProfileForJWT
   issue: https://github.com/elastic/elasticsearch/issues/120983

x-pack/plugin/shutdown/src/internalClusterTest/java/org/elasticsearch/xpack/shutdown/AllocationFailuresResetOnShutdownIT.java

@@ -175,18 +175,18 @@ public void beforeIndexShardCreated(ShardRouting routing, Settings indexSettings
 
         prepareCreate("index1", indexSettings(1, 0)).execute();
 
-        // await all relocation attempts are exhausted
+        // await all allocation attempts are exhausted
         var maxAttempts = MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY.get(Settings.EMPTY);
         ensureRed("index1");
-        {
+        assertBusy(() -> {
             var state = safeGet(clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).execute()).getState();
             var index = state.getRoutingTable().index("index1");
             assertNotNull(index);
             var shard = index.shard(0).primaryShard();
             assertNotNull(shard);
             assertNotNull(shard.unassignedInfo());
             assertThat(maxAttempts, equalTo(shard.unassignedInfo().failedAllocations()));
-        }
+        });
 
         failAllocation.set(false);
 
@@ -240,13 +240,13 @@ public void beforeIndexShardCreated(ShardRouting routing, Settings indexSettings
         // await all allocation attempts are exhausted
         var maxAttempts = MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY.get(Settings.EMPTY);
         ensureRed("index1");
-        {
+        assertBusy(() -> {
             var state = safeGet(clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).execute()).getState();
             var shard = state.getRoutingTable().index("index1").shard(0).primaryShard();
             assertNotNull(shard);
             assertNotNull(shard.unassignedInfo());
             assertThat(maxAttempts, equalTo(shard.unassignedInfo().failedAllocations()));
-        }
+        });
 
         failAllocation.set(false);