From bd0ea0a8efbb5cdc83b5798b56a2f062ba29d9ec Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Mon, 3 Nov 2025 15:35:20 +0100 Subject: [PATCH] Increase timeout for searchable snapshots in ILM tests (#137514) As of #133954, we clone indices before performing the force-merge step in the `searchable_snapshot` action. On slow CI servers, 10 seconds for the index to go through the whole `searchable_snapshot` action isn't enough, so we bump the timeout to 20 seconds. I looked at the logs of a few test failures, and ILM was clearly still progressing when the test timed out. I didn't identify any particular step that was taking extraordinarily long; there were always just a few steps that took a bit longer. I would love to make these tests faster rather than bumping the timeout, but the `searchable_snapshot` action is simply one of the largest ILM actions and ILM itself isn't particularly fast. That being said, if a timeout of 20 seconds proves to be insufficient (i.e. test failures come back), I do think it's worth having a look at reducing the runtime of the tests somehow first before we increase the timeout further. Closes #137149 Closes #137151 Closes #137152 Closes #137153 Closes #137156 Closes #137166 Closes #137167 Closes #137192 (cherry picked from commit 60b89a88c570133555eacd745b06b71b732fd04a) # Conflicts: # muted-tests.yml --- .../xpack/ilm/TimeSeriesDataStreamsIT.java | 4 +-- .../actions/SearchableSnapshotActionIT.java | 34 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java index 366fcc2f821df..ffd2b2b681824 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java @@ -184,8 +184,8 @@ public void testSearchableSnapshotAction() throws Exception { // Manual rollover the original index such that it's not the write index in the data stream anymore rolloverMaxOneDocCondition(client(), dataStream); - awaitIndexExists(restoredIndexName); - awaitIndexDoesNotExist(backingIndexName, TimeValue.timeValueSeconds(60)); + awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20)); + awaitIndexDoesNotExist(backingIndexName); assertBusy( () -> assertThat(explainIndex(client(), restoredIndexName).get("step"), is(PhaseCompleteStep.NAME)), 30, diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java index 735f64821045d..ce70f73d7c28f 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java @@ -111,7 +111,7 @@ public void testSearchableSnapshotAction() throws Exception { assertThat(backingIndices.size(), equalTo(2)); String backingIndexName = backingIndices.getFirst(); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; - awaitIndexExists(restoredIndexName); + awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), restoredIndexName, null, null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished @@ -233,7 +233,7 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception { Map coldActions = Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo)); Map phases = new HashMap<>(); phases.put("cold", new Phase("cold", TimeValue.ZERO, coldActions)); - phases.put("delete", new Phase("delete", TimeValue.timeValueMillis(10000), Map.of(DeleteAction.NAME, WITH_SNAPSHOT_DELETE))); + phases.put("delete", new Phase("delete", TimeValue.ZERO, Map.of(DeleteAction.NAME, WITH_SNAPSHOT_DELETE))); LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, phases); // PUT policy XContentBuilder builder = jsonBuilder(); @@ -261,7 +261,7 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception { String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; // let's wait for ILM to finish - awaitIndexDoesNotExist(backingIndexName); + awaitIndexDoesNotExist(backingIndexName, TimeValue.timeValueSeconds(20)); awaitIndexDoesNotExist(restoredIndexName); List> snapshots = getSnapshots(); @@ -347,7 +347,7 @@ public void testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped() throws String backingIndexName = backingIndices.getFirst(); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; - awaitIndexExists(restoredIndexName); + awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), restoredIndexName, "hot", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(backingIndexName); @@ -417,7 +417,7 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex String backingIndexName = backingIndices.getFirst(); String searchableSnapMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; - awaitIndexExists(searchableSnapMountedIndexName); + awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "hot", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(backingIndexName); @@ -459,7 +459,7 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex restoreSnapshot.setJsonEntity("{\"indices\": \"" + dataStream + "\", \"include_global_state\": false}"); assertOK(client().performRequest(restoreSnapshot)); - assertThat(indexExists(searchableSnapMountedIndexName), is(true)); + awaitIndexExists(searchableSnapMountedIndexName); ensureGreen(searchableSnapMountedIndexName); // the restored index is now managed by the now updated ILM policy and needs to go through the warm and cold phase @@ -523,7 +523,7 @@ public void testIdenticalSearchableSnapshotActionIsNoop() throws Exception { final String searchableSnapMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index; logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName); - awaitIndexExists(searchableSnapMountedIndexName); + awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "cold", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(index); @@ -569,7 +569,7 @@ public void testConvertingSearchableSnapshotFromFullToPartial() throws Exception + SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index; logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName); - awaitIndexExists(searchableSnapMountedIndexName); + awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "frozen", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(index); @@ -635,7 +635,7 @@ public void testResumingSearchableSnapshotFromFullToPartial() throws Exception { final String fullMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index; logger.info("--> waiting for [{}] to exist...", fullMountedIndexName); - awaitIndexExists(fullMountedIndexName); + awaitIndexExists(fullMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), fullMountedIndexName, "cold", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(index); @@ -651,7 +651,7 @@ public void testResumingSearchableSnapshotFromFullToPartial() throws Exception { String partiallyMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX + fullMountedIndexName; logger.info("--> waiting for [{}] to exist...", partiallyMountedIndexName); - awaitIndexExists(partiallyMountedIndexName); + awaitIndexExists(partiallyMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), partiallyMountedIndexName, "frozen", null, PhaseCompleteStep.NAME); // Ensure the searchable snapshot is not deleted when the index was deleted because it was not created by this @@ -727,7 +727,7 @@ public void testResumingSearchableSnapshotFromPartialToFull() throws Exception { final String fullMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index; final String partialMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX + fullMountedIndexName; logger.info("--> waiting for [{}] to exist...", partialMountedIndexName); - awaitIndexExists(partialMountedIndexName); + awaitIndexExists(partialMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), partialMountedIndexName, "frozen", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(index); @@ -743,7 +743,7 @@ public void testResumingSearchableSnapshotFromPartialToFull() throws Exception { String restoredPartiallyMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + partialMountedIndexName; logger.info("--> waiting for [{}] to exist...", restoredPartiallyMountedIndexName); - awaitIndexExists(restoredPartiallyMountedIndexName); + awaitIndexExists(restoredPartiallyMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), restoredPartiallyMountedIndexName, "cold", null, PhaseCompleteStep.NAME); // Ensure the searchable snapshot is not deleted when the index was deleted because it was not created by this @@ -848,7 +848,7 @@ public void testSearchableSnapshotsInHotPhasePinnedToHotNodes() throws Exception final String restoredIndex = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + firstGenIndex; logger.info("--> waiting for [{}] to exist...", restoredIndex); - awaitIndexExists(restoredIndex); + awaitIndexExists(restoredIndex, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), restoredIndex, "hot", PhaseCompleteStep.NAME, PhaseCompleteStep.NAME); Map hotIndexSettings = getIndexSettingsAsMap(restoredIndex); @@ -887,7 +887,7 @@ public void testSearchableSnapshotInvokesAsyncActionOnNewIndex() throws Exceptio assertThat(backingIndices.size(), equalTo(2)); String backingIndexName = backingIndices.getFirst(); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; - awaitIndexExists(restoredIndexName); + awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), restoredIndexName, null, null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished @@ -931,7 +931,7 @@ public void testSearchableSnapshotTotalShardsPerNode() throws Exception { final String searchableSnapMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX + SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index; logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName); - awaitIndexExists(searchableSnapMountedIndexName); + awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20)); TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "frozen", null, PhaseCompleteStep.NAME); // Wait for the original index to be deleted, to ensure ILM has finished awaitIndexDoesNotExist(index); @@ -984,7 +984,7 @@ public void testSearchableSnapshotReplicateFor() throws Exception { assertThat(backingIndices.size(), equalTo(2)); String backingIndexName = backingIndices.getFirst(); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; - awaitIndexExists(restoredIndexName); + awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20)); // check that the index is in the expected step and has the expected step_info.message assertBusy(() -> { @@ -1133,7 +1133,7 @@ private void assertForceMergedSnapshotDone(String phase, String backingIndexName ? SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX : SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX; final String restoredIndexName = prefix + backingIndexName; - awaitIndexExists(restoredIndexName); + awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20)); assertBusy(() -> assertThat(explainIndex(client(), restoredIndexName).get("step"), is(PhaseCompleteStep.NAME))); // Wait for the original index to be deleted, to ensure ILM has finished