Fix some ILM tests for searchable snapshots

These tests are very prone to the "quiet test cluster syndrome" and
sometimes need a little push by means of some artificial cluster state
updates. To hopefully be done with these test failures once and for all
(famous last words), I've updated all tests in this class to trigger
cluster state updates where necessary.

This is basically a follow-up of #108162.

Fixes #125683
Fixes #125789

Author: nielsbauman

muted-tests.yml

@@ -368,9 +368,6 @@ tests:
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test010Install
   issue: https://github.com/elastic/elasticsearch/issues/125680
-- class: org.elasticsearch.xpack.ilm.actions.SearchableSnapshotActionIT
-  method: testSearchableSnapshotsInHotPhasePinnedToHotNodes
-  issue: https://github.com/elastic/elasticsearch/issues/125683
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
   method: test {p0=cat.allocation/10_basic/Node forecasts}
   issue: https://github.com/elastic/elasticsearch/issues/125711

x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java

@@ -106,13 +106,7 @@ public void testSearchableSnapshotAction() throws Exception {
         assertThat(backingIndices.size(), equalTo(2));
         String backingIndexName = backingIndices.getFirst();
         String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;
-        assertTrue(waitUntil(() -> {
-            try {
-                return indexExists(restoredIndexName);
-            } catch (IOException e) {
-                return false;
-            }
-        }, 30, TimeUnit.SECONDS));
+        waitForIndexToExist(restoredIndexName, 30);
 
         assertBusy(() -> {
             triggerStateChange();
@@ -146,6 +140,7 @@ public void testSearchableSnapshotForceMergesIndexToOneSegment() throws Exceptio
         updateIndexSettings(dataStream, Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policy));
         assertTrue(waitUntil(() -> {
             try {
+                triggerStateChange();
                 Integer numberOfSegments = getNumberOfPrimarySegments(client(), backingIndexName);
                 logger.info("index {} has {} segments", backingIndexName, numberOfSegments);
                 // this is a loose assertion here as forcemerge is best effort
@@ -166,13 +161,7 @@ public void testSearchableSnapshotForceMergesIndexToOneSegment() throws Exceptio
         }, 60, TimeUnit.SECONDS));
 
         String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;
-        assertTrue(waitUntil(() -> {
-            try {
-                return indexExists(restoredIndexName);
-            } catch (IOException e) {
-                return false;
-            }
-        }, 60, TimeUnit.SECONDS));
+        waitForIndexToExist(restoredIndexName, 60);
 
         assertBusy(() -> {
             triggerStateChange();
@@ -221,6 +210,7 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception {
 
         assertTrue("the snapshot we generate in the cold phase should be deleted by the delete phase", waitUntil(() -> {
             try {
+                triggerStateChange();
                 Request getSnapshotsRequest = new Request("GET", "_snapshot/" + snapshotRepo + "/_all");
                 Response getSnapshotsResponse = client().performRequest(getSnapshotsRequest);
 
@@ -306,13 +296,7 @@ public void testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped() throws
 
         var backingIndices = getDataStreamBackingIndexNames(dataStream);
         String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndices.get(0);
-        assertTrue(waitUntil(() -> {
-            try {
-                return indexExists(restoredIndexName);
-            } catch (IOException e) {
-                return false;
-            }
-        }, 30, TimeUnit.SECONDS));
+        waitForIndexToExist(restoredIndexName, 30);
 
         assertBusy(() -> {
             triggerStateChange();
@@ -385,13 +369,7 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex
 
         String backingIndexName = getDataStreamBackingIndexNames(dataStream).getFirst();
         String searchableSnapMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;
-        assertTrue(waitUntil(() -> {
-            try {
-                return indexExists(searchableSnapMountedIndexName);
-            } catch (IOException e) {
-                return false;
-            }
-        }, 30, TimeUnit.SECONDS));
+        waitForIndexToExist(searchableSnapMountedIndexName, 30);
 
         assertBusy(() -> {
             triggerStateChange();
@@ -442,6 +420,7 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex
 
         // the restored index is now managed by the now updated ILM policy and needs to go through the warm and cold phase
         assertBusy(() -> {
+            triggerStateChange();
             Step.StepKey stepKeyForIndex;
             try {
                 stepKeyForIndex = getStepKeyForIndex(client(), searchableSnapMountedIndexName);
@@ -493,10 +472,7 @@ public void testIdenticalSearchableSnapshotActionIsNoop() throws Exception {
 
         final String searchableSnapMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index;
 
-        assertBusy(() -> {
-            logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName);
-            assertTrue(indexExists(searchableSnapMountedIndexName));
-        }, 30, TimeUnit.SECONDS);
+        waitForIndexToExist(searchableSnapMountedIndexName, 30);
 
         assertBusy(() -> {
             triggerStateChange();
@@ -555,10 +531,7 @@ public void testConvertingSearchableSnapshotFromFullToPartial() throws Exception
         final String searchableSnapMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX
             + SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index;
 
-        assertBusy(() -> {
-            logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName);
-            assertTrue(indexExists(searchableSnapMountedIndexName));
-        }, 30, TimeUnit.SECONDS);
+        waitForIndexToExist(searchableSnapMountedIndexName, 30);
 
         assertBusy(() -> {
             triggerStateChange();
@@ -638,10 +611,7 @@ public void testResumingSearchableSnapshotFromFullToPartial() throws Exception {
 
         final String fullMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index;
 
-        assertBusy(() -> {
-            logger.info("--> waiting for [{}] to exist...", fullMountedIndexName);
-            assertTrue(indexExists(fullMountedIndexName));
-        }, 30, TimeUnit.SECONDS);
+        waitForIndexToExist(fullMountedIndexName, 30);
 
         assertBusy(() -> {
             triggerStateChange();
@@ -659,10 +629,7 @@ public void testResumingSearchableSnapshotFromFullToPartial() throws Exception {
         // add cold-frozen
         updateIndexSettings(index, Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policyFrozen));
         String partiallyMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX + fullMountedIndexName;
-        assertBusy(() -> {
-            logger.info("--> waiting for [{}] to exist...", partiallyMountedIndexName);
-            assertTrue(indexExists(partiallyMountedIndexName));
-        }, 30, TimeUnit.SECONDS);
+        waitForIndexToExist(partiallyMountedIndexName, 30);
 
         assertBusy(() -> {
             triggerStateChange();
@@ -693,6 +660,7 @@ public void testResumingSearchableSnapshotFromFullToPartial() throws Exception {
         );
         assertBusy(() -> {
             logger.info("--> waiting for [{}] to be deleted...", partiallyMountedIndexName);
+            triggerStateChange();
             assertThat(indexExists(partiallyMountedIndexName), is(false));
             Request getSnaps = new Request("GET", "/_snapshot/" + snapshotRepo + "/_all");
             Map<String, Object> responseMap = responseAsMap(client().performRequest(getSnaps));
@@ -749,10 +717,7 @@ public void testResumingSearchableSnapshotFromPartialToFull() throws Exception {
         final String fullMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index;
         final String partialMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX + fullMountedIndexName;
 
-        assertBusy(() -> {
-            logger.info("--> waiting for [{}] to exist...", partialMountedIndexName);
-            assertTrue(indexExists(partialMountedIndexName));
-        }, 30, TimeUnit.SECONDS);
+        waitForIndexToExist(partialMountedIndexName, 30);
 
         assertBusy(() -> {
             triggerStateChange();
@@ -770,10 +735,7 @@ public void testResumingSearchableSnapshotFromPartialToFull() throws Exception {
         // add a policy that will only include the fully mounted searchable snapshot
         updateIndexSettings(index, Settings.builder().put(LifecycleSettings.LIFECYCLE_NAME, policyCold));
         String restoredPartiallyMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + partialMountedIndexName;
-        assertBusy(() -> {
-            logger.info("--> waiting for [{}] to exist...", restoredPartiallyMountedIndexName);
-            assertTrue(indexExists(restoredPartiallyMountedIndexName));
-        }, 30, TimeUnit.SECONDS);
+        waitForIndexToExist(restoredPartiallyMountedIndexName, 30);
 
         assertBusy(() -> {
             triggerStateChange();
@@ -800,6 +762,7 @@ public void testResumingSearchableSnapshotFromPartialToFull() throws Exception {
         );
         assertBusy(() -> {
             logger.info("--> waiting for [{}] to be deleted...", restoredPartiallyMountedIndexName);
+            triggerStateChange();
             assertThat(indexExists(restoredPartiallyMountedIndexName), is(false));
             Request getSnaps = new Request("GET", "/_snapshot/" + snapshotRepo + "/_all");
             Map<String, Object> responseMap = responseAsMap(client().performRequest(getSnaps));
@@ -882,15 +845,11 @@ public void testSearchableSnapshotsInHotPhasePinnedToHotNodes() throws Exception
         indexDocument(client(), dataStream, true);
 
         final String restoredIndex = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + firstGenIndex;
+        waitForIndexToExist(restoredIndex, 30);
         assertBusy(() -> {
-            logger.info("--> waiting for [{}] to exist...", restoredIndex);
-            assertTrue(indexExists(restoredIndex));
+            triggerStateChange();
+            assertThat(getStepKeyForIndex(client(), restoredIndex), is(PhaseCompleteStep.finalStep("hot").getKey()));
         }, 30, TimeUnit.SECONDS);
-        assertBusy(
-            () -> assertThat(getStepKeyForIndex(client(), restoredIndex), is(PhaseCompleteStep.finalStep("hot").getKey())),
-            30,
-            TimeUnit.SECONDS
-        );
 
         Map<String, Object> hotIndexSettings = getIndexSettingsAsMap(restoredIndex);
         // searchable snapshots mounted in the hot phase should be pinned to hot nodes
@@ -928,13 +887,7 @@ public void testSearchableSnapshotInvokesAsyncActionOnNewIndex() throws Exceptio
         assertThat(backingIndices.size(), equalTo(2));
         String backingIndexName = backingIndices.getFirst();
         String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;
-        assertTrue(waitUntil(() -> {
-            try {
-                return indexExists(restoredIndexName);
-            } catch (IOException e) {
-                return false;
-            }
-        }, 30, TimeUnit.SECONDS));
+        waitForIndexToExist(restoredIndexName, 30);
 
         assertBusy(() -> {
             triggerStateChange();
@@ -975,10 +928,7 @@ public void testSearchableSnapshotTotalShardsPerNode() throws Exception {
         // wait for snapshot successfully mounted and ILM execution completed
         final String searchableSnapMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX
             + SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index;
-        assertBusy(() -> {
-            logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName);
-            assertTrue(indexExists(searchableSnapMountedIndexName));
-        }, 30, TimeUnit.SECONDS);
+        waitForIndexToExist(searchableSnapMountedIndexName, 30);
         assertBusy(() -> {
             triggerStateChange();
             Step.StepKey stepKeyForIndex = getStepKeyForIndex(client(), searchableSnapMountedIndexName);
@@ -1034,13 +984,7 @@ public void testSearchableSnapshotReplicateFor() throws Exception {
         assertThat(backingIndices.size(), equalTo(2));
         String backingIndexName = backingIndices.getFirst();
         String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;
-        assertTrue(waitUntil(() -> {
-            try {
-                return indexExists(restoredIndexName);
-            } catch (IOException e) {
-                return false;
-            }
-        }, 30, TimeUnit.SECONDS));
+        waitForIndexToExist(restoredIndexName, 30);
 
         // check that the index is in the expected step and has the expected step_info.message
         assertBusy(() -> {
@@ -1097,6 +1041,14 @@ public void testSearchableSnapshotReplicateFor() throws Exception {
         }
     }
 
+    private void waitForIndexToExist(String indexName, int timeoutInSeconds) throws Exception {
+        assertBusy(() -> {
+            logger.info("--> waiting for [{}] to exist...", indexName);
+            triggerStateChange();
+            assertTrue(indexExists(indexName));
+        }, timeoutInSeconds, TimeUnit.SECONDS);
+    }
+
     /**
      * Cause a bit of cluster activity using an empty reroute call in case the `wait-for-index-colour` ILM step missed the
      * notification that partial-index is now GREEN.