Increase timeout for searchable snapshots in ILM tests (#137514)

As of #133954, we clone indices before performing the force-merge step in the `searchable_snapshot` action. On slow CI servers, 10 seconds for the index to go through the whole `searchable_snapshot` action isn't enough, so we bump the timeout to 20 seconds.

I looked at the logs of a few test failures, and ILM was clearly still progressing when the test timed out. I didn't identify any particular step that was taking extraordinarily long; there were always just a few steps that took a bit longer. I would love to make these tests faster rather than bumping the timeout, but the `searchable_snapshot` action is simply one of the largest ILM actions and ILM itself isn't particularly fast.

That being said, if a timeout of 20 seconds proves to be insufficient (i.e. test failures come back), I do think it's worth having a look at reducing the runtime of the tests somehow first before we increase the timeout further.

Closes #137149
Closes #137151
Closes #137152
Closes #137153
Closes #137156
Closes #137166
Closes #137167
Closes #137192

Author: nielsbauman

muted-tests.yml

@@ -492,9 +492,6 @@ tests:
 - class: org.elasticsearch.readiness.ReadinessClusterIT
   method: testReadinessDuringRestartsNormalOrder
   issue: https://github.com/elastic/elasticsearch/issues/136955
-- class: org.elasticsearch.xpack.ilm.TimeSeriesDataStreamsIT
-  method: testSearchableSnapshotAction
-  issue: https://github.com/elastic/elasticsearch/issues/137167
 - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
   method: test {yaml=indices.validate_query/20_query_string/validate_query with query_string parameters}
   issue: https://github.com/elastic/elasticsearch/issues/137391

x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java

@@ -184,8 +184,8 @@ public void testSearchableSnapshotAction() throws Exception {
         // Manual rollover the original index such that it's not the write index in the data stream anymore
         rolloverMaxOneDocCondition(client(), dataStream);
 
-        awaitIndexExists(restoredIndexName);
-        awaitIndexDoesNotExist(backingIndexName, TimeValue.timeValueSeconds(60));
+        awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20));
+        awaitIndexDoesNotExist(backingIndexName);
         assertBusy(
             () -> assertThat(explainIndex(client(), restoredIndexName).get("step"), is(PhaseCompleteStep.NAME)),
             30,

x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java

@@ -111,7 +111,7 @@ public void testSearchableSnapshotAction() throws Exception {
         assertThat(backingIndices.size(), equalTo(2));
         String backingIndexName = backingIndices.getFirst();
         String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;
-        awaitIndexExists(restoredIndexName);
+        awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20));
 
         TimeSeriesRestDriver.awaitStepKey(client(), restoredIndexName, null, null, PhaseCompleteStep.NAME);
         // Wait for the original index to be deleted, to ensure ILM has finished
@@ -233,7 +233,7 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception {
         Map<String, LifecycleAction> coldActions = Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo));
         Map<String, Phase> phases = new HashMap<>();
         phases.put("cold", new Phase("cold", TimeValue.ZERO, coldActions));
-        phases.put("delete", new Phase("delete", TimeValue.timeValueMillis(10000), Map.of(DeleteAction.NAME, WITH_SNAPSHOT_DELETE)));
+        phases.put("delete", new Phase("delete", TimeValue.ZERO, Map.of(DeleteAction.NAME, WITH_SNAPSHOT_DELETE)));
         LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, phases);
         // PUT policy
         XContentBuilder builder = jsonBuilder();
@@ -261,7 +261,7 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception {
         String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;
 
         // let's wait for ILM to finish
-        awaitIndexDoesNotExist(backingIndexName);
+        awaitIndexDoesNotExist(backingIndexName, TimeValue.timeValueSeconds(20));
         awaitIndexDoesNotExist(restoredIndexName);
 
         List<Map<String, Object>> snapshots = getSnapshots();
@@ -347,7 +347,7 @@ public void testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped() throws
 
         String backingIndexName = backingIndices.getFirst();
         String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;
-        awaitIndexExists(restoredIndexName);
+        awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20));
         TimeSeriesRestDriver.awaitStepKey(client(), restoredIndexName, "hot", null, PhaseCompleteStep.NAME);
         // Wait for the original index to be deleted, to ensure ILM has finished
         awaitIndexDoesNotExist(backingIndexName);
@@ -417,7 +417,7 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex
 
         String backingIndexName = backingIndices.getFirst();
         String searchableSnapMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;
-        awaitIndexExists(searchableSnapMountedIndexName);
+        awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20));
         TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "hot", null, PhaseCompleteStep.NAME);
         // Wait for the original index to be deleted, to ensure ILM has finished
         awaitIndexDoesNotExist(backingIndexName);
@@ -459,7 +459,7 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex
         restoreSnapshot.setJsonEntity("{\"indices\": \"" + dataStream + "\", \"include_global_state\": false}");
         assertOK(client().performRequest(restoreSnapshot));
 
-        assertThat(indexExists(searchableSnapMountedIndexName), is(true));
+        awaitIndexExists(searchableSnapMountedIndexName);
         ensureGreen(searchableSnapMountedIndexName);
 
         // the restored index is now managed by the now updated ILM policy and needs to go through the warm and cold phase
@@ -523,7 +523,7 @@ public void testIdenticalSearchableSnapshotActionIsNoop() throws Exception {
         final String searchableSnapMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index;
 
         logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName);
-        awaitIndexExists(searchableSnapMountedIndexName);
+        awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20));
         TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "cold", null, PhaseCompleteStep.NAME);
         // Wait for the original index to be deleted, to ensure ILM has finished
         awaitIndexDoesNotExist(index);
@@ -569,7 +569,7 @@ public void testConvertingSearchableSnapshotFromFullToPartial() throws Exception
             + SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index;
 
         logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName);
-        awaitIndexExists(searchableSnapMountedIndexName);
+        awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20));
         TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "frozen", null, PhaseCompleteStep.NAME);
         // Wait for the original index to be deleted, to ensure ILM has finished
         awaitIndexDoesNotExist(index);
@@ -635,7 +635,7 @@ public void testResumingSearchableSnapshotFromFullToPartial() throws Exception {
 
         final String fullMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index;
         logger.info("--> waiting for [{}] to exist...", fullMountedIndexName);
-        awaitIndexExists(fullMountedIndexName);
+        awaitIndexExists(fullMountedIndexName, TimeValue.timeValueSeconds(20));
         TimeSeriesRestDriver.awaitStepKey(client(), fullMountedIndexName, "cold", null, PhaseCompleteStep.NAME);
         // Wait for the original index to be deleted, to ensure ILM has finished
         awaitIndexDoesNotExist(index);
@@ -651,7 +651,7 @@ public void testResumingSearchableSnapshotFromFullToPartial() throws Exception {
 
         String partiallyMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX + fullMountedIndexName;
         logger.info("--> waiting for [{}] to exist...", partiallyMountedIndexName);
-        awaitIndexExists(partiallyMountedIndexName);
+        awaitIndexExists(partiallyMountedIndexName, TimeValue.timeValueSeconds(20));
         TimeSeriesRestDriver.awaitStepKey(client(), partiallyMountedIndexName, "frozen", null, PhaseCompleteStep.NAME);
 
         // Ensure the searchable snapshot is not deleted when the index was deleted because it was not created by this
@@ -727,7 +727,7 @@ public void testResumingSearchableSnapshotFromPartialToFull() throws Exception {
         final String fullMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index;
         final String partialMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX + fullMountedIndexName;
         logger.info("--> waiting for [{}] to exist...", partialMountedIndexName);
-        awaitIndexExists(partialMountedIndexName);
+        awaitIndexExists(partialMountedIndexName, TimeValue.timeValueSeconds(20));
         TimeSeriesRestDriver.awaitStepKey(client(), partialMountedIndexName, "frozen", null, PhaseCompleteStep.NAME);
         // Wait for the original index to be deleted, to ensure ILM has finished
         awaitIndexDoesNotExist(index);
@@ -743,7 +743,7 @@ public void testResumingSearchableSnapshotFromPartialToFull() throws Exception {
 
         String restoredPartiallyMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + partialMountedIndexName;
         logger.info("--> waiting for [{}] to exist...", restoredPartiallyMountedIndexName);
-        awaitIndexExists(restoredPartiallyMountedIndexName);
+        awaitIndexExists(restoredPartiallyMountedIndexName, TimeValue.timeValueSeconds(20));
         TimeSeriesRestDriver.awaitStepKey(client(), restoredPartiallyMountedIndexName, "cold", null, PhaseCompleteStep.NAME);
 
         // Ensure the searchable snapshot is not deleted when the index was deleted because it was not created by this
@@ -848,7 +848,7 @@ public void testSearchableSnapshotsInHotPhasePinnedToHotNodes() throws Exception
 
         final String restoredIndex = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + firstGenIndex;
         logger.info("--> waiting for [{}] to exist...", restoredIndex);
-        awaitIndexExists(restoredIndex);
+        awaitIndexExists(restoredIndex, TimeValue.timeValueSeconds(20));
         TimeSeriesRestDriver.awaitStepKey(client(), restoredIndex, "hot", PhaseCompleteStep.NAME, PhaseCompleteStep.NAME);
 
         Map<String, Object> hotIndexSettings = getIndexSettingsAsMap(restoredIndex);
@@ -887,7 +887,7 @@ public void testSearchableSnapshotInvokesAsyncActionOnNewIndex() throws Exceptio
         assertThat(backingIndices.size(), equalTo(2));
         String backingIndexName = backingIndices.getFirst();
         String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;
-        awaitIndexExists(restoredIndexName);
+        awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20));
 
         TimeSeriesRestDriver.awaitStepKey(client(), restoredIndexName, null, null, PhaseCompleteStep.NAME);
         // Wait for the original index to be deleted, to ensure ILM has finished
@@ -931,7 +931,7 @@ public void testSearchableSnapshotTotalShardsPerNode() throws Exception {
         final String searchableSnapMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX
             + SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index;
         logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName);
-        awaitIndexExists(searchableSnapMountedIndexName);
+        awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20));
         TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "frozen", null, PhaseCompleteStep.NAME);
         // Wait for the original index to be deleted, to ensure ILM has finished
         awaitIndexDoesNotExist(index);
@@ -984,7 +984,7 @@ public void testSearchableSnapshotReplicateFor() throws Exception {
         assertThat(backingIndices.size(), equalTo(2));
         String backingIndexName = backingIndices.getFirst();
         String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;
-        awaitIndexExists(restoredIndexName);
+        awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20));
 
         // check that the index is in the expected step and has the expected step_info.message
         assertBusy(() -> {
@@ -1133,7 +1133,7 @@ private void assertForceMergedSnapshotDone(String phase, String backingIndexName
             ? SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX
             : SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX;
         final String restoredIndexName = prefix + backingIndexName;
-        awaitIndexExists(restoredIndexName);
+        awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20));
 
         assertBusy(() -> assertThat(explainIndex(client(), restoredIndexName).get("step"), is(PhaseCompleteStep.NAME)));
         // Wait for the original index to be deleted, to ensure ILM has finished