Skip to content

Increase timeout for searchable snapshots in ILM tests #137514

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nielsbauman merged 3 commits into from
Nov 3, 2025
Merged

Increase timeout for searchable snapshots in ILM tests #137514

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
90e1774
Increase timeout for searchable snapshots in ILM tests
nielsbauman Nov 3, 2025
db77ab4
Merge branch 'main' into fix-searchable-snapshot-tests
nielsbauman Nov 3, 2025
38a2ec4
Unmute test
nielsbauman Nov 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions muted-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -492,9 +492,6 @@ tests:
- class: org.elasticsearch.readiness.ReadinessClusterIT
method: testReadinessDuringRestartsNormalOrder
issue: https://github.com/elastic/elasticsearch/issues/136955
- class: org.elasticsearch.xpack.ilm.TimeSeriesDataStreamsIT
method: testSearchableSnapshotAction
issue: https://github.com/elastic/elasticsearch/issues/137167
- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
method: test {yaml=indices.validate_query/20_query_string/validate_query with query_string parameters}
issue: https://github.com/elastic/elasticsearch/issues/137391
Expand Down
4 changes: 2 additions & 2 deletions ...multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,8 @@ public void testSearchableSnapshotAction() throws Exception {
// Manual rollover the original index such that it's not the write index in the data stream anymore
rolloverMaxOneDocCondition(client(), dataStream);

awaitIndexExists(restoredIndexName);
awaitIndexDoesNotExist(backingIndexName, TimeValue.timeValueSeconds(60));
awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20));
awaitIndexDoesNotExist(backingIndexName);
assertBusy(
() -> assertThat(explainIndex(client(), restoredIndexName).get("step"), is(PhaseCompleteStep.NAME)),
30,
Expand Down
34 changes: 17 additions & 17 deletions ...src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ public void testSearchableSnapshotAction() throws Exception {
assertThat(backingIndices.size(), equalTo(2));
String backingIndexName = backingIndices.getFirst();
String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;
awaitIndexExists(restoredIndexName);
awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20));

TimeSeriesRestDriver.awaitStepKey(client(), restoredIndexName, null, null, PhaseCompleteStep.NAME);
// Wait for the original index to be deleted, to ensure ILM has finished
Expand Down Expand Up @@ -233,7 +233,7 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception {
Map<String, LifecycleAction> coldActions = Map.of(SearchableSnapshotAction.NAME, new SearchableSnapshotAction(snapshotRepo));
Map<String, Phase> phases = new HashMap<>();
phases.put("cold", new Phase("cold", TimeValue.ZERO, coldActions));
phases.put("delete", new Phase("delete", TimeValue.timeValueMillis(10000), Map.of(DeleteAction.NAME, WITH_SNAPSHOT_DELETE)));
phases.put("delete", new Phase("delete", TimeValue.ZERO, Map.of(DeleteAction.NAME, WITH_SNAPSHOT_DELETE)));
Copy link
Contributor Author

@nielsbauman nielsbauman Nov 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

By the way, FTR, I changed this value from 10s to 0s because there is no point in waiting 10 seconds before we delete the searchable snapshotted index; we can just delete it immediately without compromising the flakiness or value of this test.

LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, phases);
// PUT policy
XContentBuilder builder = jsonBuilder();
Expand Down Expand Up @@ -261,7 +261,7 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception {
String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;

// let's wait for ILM to finish
awaitIndexDoesNotExist(backingIndexName);
awaitIndexDoesNotExist(backingIndexName, TimeValue.timeValueSeconds(20));
awaitIndexDoesNotExist(restoredIndexName);

List<Map<String, Object>> snapshots = getSnapshots();
Expand Down Expand Up @@ -347,7 +347,7 @@ public void testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped() throws

String backingIndexName = backingIndices.getFirst();
String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;
awaitIndexExists(restoredIndexName);
awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20));
TimeSeriesRestDriver.awaitStepKey(client(), restoredIndexName, "hot", null, PhaseCompleteStep.NAME);
// Wait for the original index to be deleted, to ensure ILM has finished
awaitIndexDoesNotExist(backingIndexName);
Expand Down Expand Up @@ -417,7 +417,7 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex

String backingIndexName = backingIndices.getFirst();
String searchableSnapMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;
awaitIndexExists(searchableSnapMountedIndexName);
awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20));
TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "hot", null, PhaseCompleteStep.NAME);
// Wait for the original index to be deleted, to ensure ILM has finished
awaitIndexDoesNotExist(backingIndexName);
Expand Down Expand Up @@ -459,7 +459,7 @@ public void testRestoredIndexManagedByLocalPolicySkipsIllegalActions() throws Ex
restoreSnapshot.setJsonEntity("{\"indices\": \"" + dataStream + "\", \"include_global_state\": false}");
assertOK(client().performRequest(restoreSnapshot));

assertThat(indexExists(searchableSnapMountedIndexName), is(true));
awaitIndexExists(searchableSnapMountedIndexName);
Copy link
Member

@PeteGillinElastic PeteGillinElastic Nov 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just checking, this one doesn't need the extended timeout?

Copy link
Contributor Author

@nielsbauman nielsbauman Nov 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nope, this is just waiting for the index to be restored after the _restore API from a few lines before. That should definitely not take more than 10 seconds. Thanks for checking!

ensureGreen(searchableSnapMountedIndexName);

// the restored index is now managed by the now updated ILM policy and needs to go through the warm and cold phase
Expand Down Expand Up @@ -523,7 +523,7 @@ public void testIdenticalSearchableSnapshotActionIsNoop() throws Exception {
final String searchableSnapMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index;

logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName);
awaitIndexExists(searchableSnapMountedIndexName);
awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20));
TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "cold", null, PhaseCompleteStep.NAME);
// Wait for the original index to be deleted, to ensure ILM has finished
awaitIndexDoesNotExist(index);
Expand Down Expand Up @@ -569,7 +569,7 @@ public void testConvertingSearchableSnapshotFromFullToPartial() throws Exception
+ SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index;

logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName);
awaitIndexExists(searchableSnapMountedIndexName);
awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20));
TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "frozen", null, PhaseCompleteStep.NAME);
// Wait for the original index to be deleted, to ensure ILM has finished
awaitIndexDoesNotExist(index);
Expand Down Expand Up @@ -635,7 +635,7 @@ public void testResumingSearchableSnapshotFromFullToPartial() throws Exception {

final String fullMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index;
logger.info("--> waiting for [{}] to exist...", fullMountedIndexName);
awaitIndexExists(fullMountedIndexName);
awaitIndexExists(fullMountedIndexName, TimeValue.timeValueSeconds(20));
TimeSeriesRestDriver.awaitStepKey(client(), fullMountedIndexName, "cold", null, PhaseCompleteStep.NAME);
// Wait for the original index to be deleted, to ensure ILM has finished
awaitIndexDoesNotExist(index);
Expand All @@ -651,7 +651,7 @@ public void testResumingSearchableSnapshotFromFullToPartial() throws Exception {

String partiallyMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX + fullMountedIndexName;
logger.info("--> waiting for [{}] to exist...", partiallyMountedIndexName);
awaitIndexExists(partiallyMountedIndexName);
awaitIndexExists(partiallyMountedIndexName, TimeValue.timeValueSeconds(20));
TimeSeriesRestDriver.awaitStepKey(client(), partiallyMountedIndexName, "frozen", null, PhaseCompleteStep.NAME);

// Ensure the searchable snapshot is not deleted when the index was deleted because it was not created by this
Expand Down Expand Up @@ -727,7 +727,7 @@ public void testResumingSearchableSnapshotFromPartialToFull() throws Exception {
final String fullMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index;
final String partialMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX + fullMountedIndexName;
logger.info("--> waiting for [{}] to exist...", partialMountedIndexName);
awaitIndexExists(partialMountedIndexName);
awaitIndexExists(partialMountedIndexName, TimeValue.timeValueSeconds(20));
TimeSeriesRestDriver.awaitStepKey(client(), partialMountedIndexName, "frozen", null, PhaseCompleteStep.NAME);
// Wait for the original index to be deleted, to ensure ILM has finished
awaitIndexDoesNotExist(index);
Expand All @@ -743,7 +743,7 @@ public void testResumingSearchableSnapshotFromPartialToFull() throws Exception {

String restoredPartiallyMountedIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + partialMountedIndexName;
logger.info("--> waiting for [{}] to exist...", restoredPartiallyMountedIndexName);
awaitIndexExists(restoredPartiallyMountedIndexName);
awaitIndexExists(restoredPartiallyMountedIndexName, TimeValue.timeValueSeconds(20));
TimeSeriesRestDriver.awaitStepKey(client(), restoredPartiallyMountedIndexName, "cold", null, PhaseCompleteStep.NAME);

// Ensure the searchable snapshot is not deleted when the index was deleted because it was not created by this
Expand Down Expand Up @@ -848,7 +848,7 @@ public void testSearchableSnapshotsInHotPhasePinnedToHotNodes() throws Exception

final String restoredIndex = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + firstGenIndex;
logger.info("--> waiting for [{}] to exist...", restoredIndex);
awaitIndexExists(restoredIndex);
awaitIndexExists(restoredIndex, TimeValue.timeValueSeconds(20));
TimeSeriesRestDriver.awaitStepKey(client(), restoredIndex, "hot", PhaseCompleteStep.NAME, PhaseCompleteStep.NAME);

Map<String, Object> hotIndexSettings = getIndexSettingsAsMap(restoredIndex);
Expand Down Expand Up @@ -887,7 +887,7 @@ public void testSearchableSnapshotInvokesAsyncActionOnNewIndex() throws Exceptio
assertThat(backingIndices.size(), equalTo(2));
String backingIndexName = backingIndices.getFirst();
String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;
awaitIndexExists(restoredIndexName);
awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20));

TimeSeriesRestDriver.awaitStepKey(client(), restoredIndexName, null, null, PhaseCompleteStep.NAME);
// Wait for the original index to be deleted, to ensure ILM has finished
Expand Down Expand Up @@ -931,7 +931,7 @@ public void testSearchableSnapshotTotalShardsPerNode() throws Exception {
final String searchableSnapMountedIndexName = SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX
+ SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + index;
logger.info("--> waiting for [{}] to exist...", searchableSnapMountedIndexName);
awaitIndexExists(searchableSnapMountedIndexName);
awaitIndexExists(searchableSnapMountedIndexName, TimeValue.timeValueSeconds(20));
TimeSeriesRestDriver.awaitStepKey(client(), searchableSnapMountedIndexName, "frozen", null, PhaseCompleteStep.NAME);
// Wait for the original index to be deleted, to ensure ILM has finished
awaitIndexDoesNotExist(index);
Expand Down Expand Up @@ -984,7 +984,7 @@ public void testSearchableSnapshotReplicateFor() throws Exception {
assertThat(backingIndices.size(), equalTo(2));
String backingIndexName = backingIndices.getFirst();
String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;
awaitIndexExists(restoredIndexName);
awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20));

// check that the index is in the expected step and has the expected step_info.message
assertBusy(() -> {
Expand Down Expand Up @@ -1133,7 +1133,7 @@ private void assertForceMergedSnapshotDone(String phase, String backingIndexName
? SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX
: SearchableSnapshotAction.PARTIAL_RESTORED_INDEX_PREFIX;
final String restoredIndexName = prefix + backingIndexName;
awaitIndexExists(restoredIndexName);
awaitIndexExists(restoredIndexName, TimeValue.timeValueSeconds(20));

assertBusy(() -> assertThat(explainIndex(client(), restoredIndexName).get("step"), is(PhaseCompleteStep.NAME)));
// Wait for the original index to be deleted, to ensure ILM has finished
Expand Down