diff --git a/muted-tests.yml b/muted-tests.yml index 720c855e19c59..5d9c883f65196 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -324,21 +324,9 @@ tests: - class: org.elasticsearch.common.metrics.ExponentiallyWeightedMovingRateTests method: testEwmr_threadSafe issue: https://github.com/elastic/elasticsearch/issues/124692 -- class: org.elasticsearch.xpack.ilm.DataStreamAndIndexLifecycleMixingTests - method: testUpdateIndexTemplateToDataStreamLifecyclePreference - issue: https://github.com/elastic/elasticsearch/issues/124837 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT method: test {p0=search.vectors/41_knn_search_bbq_hnsw/Test knn search} issue: https://github.com/elastic/elasticsearch/issues/124848 -- class: org.elasticsearch.xpack.ilm.DataStreamAndIndexLifecycleMixingTests - method: testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm - issue: https://github.com/elastic/elasticsearch/issues/124849 -- class: org.elasticsearch.xpack.ilm.DataStreamAndIndexLifecycleMixingTests - method: testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle - issue: https://github.com/elastic/elasticsearch/issues/124850 -- class: org.elasticsearch.xpack.ilm.DataStreamAndIndexLifecycleMixingTests - method: testIndexTemplateSwapsILMForDataStreamLifecycle - issue: https://github.com/elastic/elasticsearch/issues/124886 - class: org.elasticsearch.packaging.test.BootstrapCheckTests method: test20RunWithBootstrapChecks issue: https://github.com/elastic/elasticsearch/issues/124940 @@ -360,9 +348,6 @@ tests: - class: org.elasticsearch.packaging.test.DockerTests method: test011SecurityEnabledStatus issue: https://github.com/elastic/elasticsearch/issues/124990 -- class: org.elasticsearch.xpack.ilm.DataStreamAndIndexLifecycleMixingTests - method: testGetDataStreamResponse - issue: https://github.com/elastic/elasticsearch/issues/125083 - class: org.elasticsearch.packaging.test.DockerTests method: test012SecurityCanBeDisabled issue: https://github.com/elastic/elasticsearch/issues/116636 diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java index 6ab1b0a7bb35a..9fb04a96282e4 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java @@ -58,13 +58,10 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Optional; -import java.util.function.Function; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -127,28 +124,15 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception { indexDocs(dataStreamName, 2); - assertBusy(() -> { - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request( - TEST_REQUEST_TIMEOUT, - new String[] { dataStreamName } - ); - GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) - .actionGet(); - assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); - assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo(dataStreamName)); - List backingIndices = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices(); - assertThat(backingIndices.size(), equalTo(2)); - String backingIndex = backingIndices.get(0).getName(); - assertThat(backingIndex, backingIndexEqualTo(dataStreamName, 1)); - String writeIndex = backingIndices.get(1).getName(); - assertThat(writeIndex, backingIndexEqualTo(dataStreamName, 2)); - }); + List backingIndices = waitForDataStreamBackingIndices(dataStreamName, 2); + assertThat(backingIndices.size(), equalTo(2)); + String firstGenerationIndex = backingIndices.get(0); + assertThat(firstGenerationIndex, backingIndexEqualTo(dataStreamName, 1)); + String secondGenerationIndex = backingIndices.get(1); + assertThat(secondGenerationIndex, backingIndexEqualTo(dataStreamName, 2)); // data stream was rolled over and has 2 indices managed by ILM assertBusy(() -> { - List backingIndices = getBackingIndices(dataStreamName); - String firstGenerationIndex = backingIndices.get(0); - String secondGenerationIndex = backingIndices.get(1); ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex @@ -181,27 +165,14 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception { indexDocs(dataStreamName, 2); // data stream was rolled over and has 3 indices, two managed by ILM and the write index will be unmanaged - assertBusy(() -> { - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request( - TEST_REQUEST_TIMEOUT, - new String[] { dataStreamName } - ); - GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) - .actionGet(); - assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); - assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().size(), is(3)); - - }); + backingIndices = waitForDataStreamBackingIndices(dataStreamName, 3); + String thirdGenerationIndex = backingIndices.get(2); assertBusy(() -> { - List backingIndices = getBackingIndices(dataStreamName); - String firstGenerationIndex = backingIndices.get(0); - String secondGenerationIndex = backingIndices.get(1); - String writeIndex = backingIndices.get(2); ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, - writeIndex + thirdGenerationIndex ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); @@ -216,13 +187,13 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception { assertThat(secondGenerationExplain.getStep(), is(PhaseCompleteStep.NAME)); // the write index is not managed by ILM - IndexLifecycleExplainResponse thirdGenerationExplain = explainResponse.getIndexResponses().get(writeIndex); + IndexLifecycleExplainResponse thirdGenerationExplain = explainResponse.getIndexResponses().get(thirdGenerationIndex); assertThat(thirdGenerationExplain.managedByILM(), is(false)); // the write index is not managed by DSL either ExplainDataStreamLifecycleAction.Response dataStreamLifecycleExplainResponse = client().execute( ExplainDataStreamLifecycleAction.INSTANCE, - new ExplainDataStreamLifecycleAction.Request(TEST_REQUEST_TIMEOUT, new String[] { writeIndex }) + new ExplainDataStreamLifecycleAction.Request(TEST_REQUEST_TIMEOUT, new String[] { thirdGenerationIndex }) ).actionGet(); assertThat(dataStreamLifecycleExplainResponse.getIndices().size(), is(1)); ExplainIndexDataStreamLifecycle writeIndexDataStreamLifecycleExplain = dataStreamLifecycleExplainResponse.getIndices().get(0); @@ -246,17 +217,8 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception { // data stream was rolled over and has 4 indices, 2 managed by ILM, and the latest 2 generations managed by the data stream // lifecycle - assertBusy(() -> { - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request( - TEST_REQUEST_TIMEOUT, - new String[] { dataStreamName } - ); - GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) - .actionGet(); - assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); - assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().size(), is(4)); - - }); + backingIndices = waitForDataStreamBackingIndices(dataStreamName, 4); + String fourthGenerationExplain = backingIndices.get(3); // let's migrate this data stream to use the custom data stream lifecycle client().execute( @@ -270,16 +232,11 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception { ).actionGet(); assertBusy(() -> { - List backingIndices = getBackingIndices(dataStreamName); - String firstGenerationIndex = backingIndices.get(0); - String secondGenerationIndex = backingIndices.get(1); - String thirdGenerationIndex = backingIndices.get(2); - String writeIndex = backingIndices.get(3); ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex, - writeIndex + fourthGenerationExplain ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); @@ -296,13 +253,16 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception { IndexLifecycleExplainResponse thirdGenerationExplain = explainResponse.getIndexResponses().get(thirdGenerationIndex); assertThat(thirdGenerationExplain.managedByILM(), is(false)); - IndexLifecycleExplainResponse writeIndexExplain = explainResponse.getIndexResponses().get(writeIndex); + IndexLifecycleExplainResponse writeIndexExplain = explainResponse.getIndexResponses().get(fourthGenerationExplain); assertThat(writeIndexExplain.managedByILM(), is(false)); // the third generation and the write index are managed based on the custom lifecycle ExplainDataStreamLifecycleAction.Response dataStreamLifecycleExplainResponse = client().execute( ExplainDataStreamLifecycleAction.INSTANCE, - new ExplainDataStreamLifecycleAction.Request(TEST_REQUEST_TIMEOUT, new String[] { thirdGenerationIndex, writeIndex }) + new ExplainDataStreamLifecycleAction.Request( + TEST_REQUEST_TIMEOUT, + new String[] { thirdGenerationIndex, fourthGenerationExplain } + ) ).actionGet(); assertThat(dataStreamLifecycleExplainResponse.getIndices().size(), is(2)); for (ExplainIndexDataStreamLifecycle index : dataStreamLifecycleExplainResponse.getIndices()) { @@ -336,23 +296,13 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); indexDocs(dataStreamName, 2); - assertBusy(() -> { - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request( - TEST_REQUEST_TIMEOUT, - new String[] { dataStreamName } - ); - GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) - .actionGet(); - assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); - assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo(dataStreamName)); - List backingIndices = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices(); - assertThat(backingIndices.size(), equalTo(2)); - }); + // data stream was rolled over and has 2 indices managed by ILM + List backingIndices = waitForDataStreamBackingIndices(dataStreamName, 2); + String firstGenerationIndex = backingIndices.get(0); + String secondGenerationIndex = backingIndices.get(1); + assertBusy(() -> { - List backingIndices = getBackingIndices(dataStreamName); - String firstGenerationIndex = backingIndices.get(0); - String secondGenerationIndex = backingIndices.get(1); ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex @@ -393,22 +343,10 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro indexDocs(dataStreamName, 2); // data stream was rolled over and has 3 indices, ALL managed by ILM - assertBusy(() -> { - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request( - TEST_REQUEST_TIMEOUT, - new String[] { dataStreamName } - ); - GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) - .actionGet(); - assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); - assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().size(), is(3)); + backingIndices = waitForDataStreamBackingIndices(dataStreamName, 3); + String thirdGenerationIndex = backingIndices.get(2); - }); assertBusy(() -> { - List backingIndices = getBackingIndices(dataStreamName); - String firstGenerationIndex = backingIndices.get(0); - String secondGenerationIndex = backingIndices.get(1); - String thirdGenerationIndex = backingIndices.get(2); ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, @@ -455,29 +393,15 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro indexDocs(dataStreamName, 2); // data stream was rolled over and has 4 indices, 3 managed by ILM, and the write index managed by the data stream lifecycle - assertBusy(() -> { - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request( - TEST_REQUEST_TIMEOUT, - new String[] { dataStreamName } - ); - GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) - .actionGet(); - assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); - assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().size(), is(4)); - - }); + backingIndices = waitForDataStreamBackingIndices(dataStreamName, 4); + String fourthGenerationIndex = backingIndices.get(3); assertBusy(() -> { - List backingIndices = getBackingIndices(dataStreamName); - String firstGenerationIndex = backingIndices.get(0); - String secondGenerationIndex = backingIndices.get(1); - String thirdGenerationIndex = backingIndices.get(2); - String writeIndex = backingIndices.get(3); ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex, - writeIndex + fourthGenerationIndex ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); @@ -499,12 +423,12 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro // the write index is managed by the data stream lifecycle ExplainDataStreamLifecycleAction.Response dataStreamLifecycleExplainResponse = client().execute( ExplainDataStreamLifecycleAction.INSTANCE, - new ExplainDataStreamLifecycleAction.Request(TEST_REQUEST_TIMEOUT, new String[] { writeIndex }) + new ExplainDataStreamLifecycleAction.Request(TEST_REQUEST_TIMEOUT, new String[] { fourthGenerationIndex }) ).actionGet(); assertThat(dataStreamLifecycleExplainResponse.getIndices().size(), is(1)); ExplainIndexDataStreamLifecycle dataStreamLifecycleExplain = dataStreamLifecycleExplainResponse.getIndices().get(0); assertThat(dataStreamLifecycleExplain.isManagedByLifecycle(), is(true)); - assertThat(dataStreamLifecycleExplain.getIndex(), is(writeIndex)); + assertThat(dataStreamLifecycleExplain.getIndex(), is(fourthGenerationIndex)); }); } @@ -533,24 +457,12 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce indexDocs(dataStreamName, 2); - assertBusy(() -> { - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request( - TEST_REQUEST_TIMEOUT, - new String[] { dataStreamName } - ); - GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) - .actionGet(); - assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); - assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo(dataStreamName)); - List backingIndices = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices(); - assertThat(backingIndices.size(), equalTo(2)); - }); + List backingIndices = waitForDataStreamBackingIndices(dataStreamName, 2); + String firstGenerationIndex = backingIndices.get(0); + String secondGenerationIndex = backingIndices.get(1); // data stream was rolled over and has 2 indices managed by ILM assertBusy(() -> { - List backingIndices = getBackingIndices(dataStreamName); - String firstGenerationIndex = backingIndices.get(0); - String secondGenerationIndex = backingIndices.get(1); ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex @@ -587,27 +499,14 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce indexDocs(dataStreamName, 2); - assertBusy(() -> { - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request( - TEST_REQUEST_TIMEOUT, - new String[] { dataStreamName } - ); - GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) - .actionGet(); - assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); - assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().size(), is(3)); - - }); + backingIndices = waitForDataStreamBackingIndices(dataStreamName, 3); + String thirdGenerationIndex = backingIndices.get(2); assertBusy(() -> { - List backingIndices = getBackingIndices(dataStreamName); - String firstGenerationIndex = backingIndices.get(0); - String secondGenerationIndex = backingIndices.get(1); - String writeIndex = backingIndices.get(2); ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, - writeIndex + thirdGenerationIndex ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); @@ -621,10 +520,14 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce assertThat(secondGenerationExplain.getPhase(), is("hot")); assertThat(secondGenerationExplain.getStep(), is(PhaseCompleteStep.NAME)); - // the write index is unmanaged + IndexLifecycleExplainResponse writeIndexExplain = explainResponse.getIndexResponses().get(thirdGenerationIndex); + assertThat(writeIndexExplain.managedByILM(), is(true)); + assertThat(writeIndexExplain.getPhase(), is("hot")); + assertThat(writeIndexExplain.getStep(), is(WaitForRolloverReadyStep.NAME)); + ExplainDataStreamLifecycleAction.Response dataStreamLifecycleExplainResponse = client().execute( ExplainDataStreamLifecycleAction.INSTANCE, - new ExplainDataStreamLifecycleAction.Request(TEST_REQUEST_TIMEOUT, new String[] { writeIndex }) + new ExplainDataStreamLifecycleAction.Request(TEST_REQUEST_TIMEOUT, new String[] { thirdGenerationIndex }) ).actionGet(); assertThat(dataStreamLifecycleExplainResponse.getIndices().size(), is(1)); ExplainIndexDataStreamLifecycle dataStreamLifecycleExplain = dataStreamLifecycleExplainResponse.getIndices().get(0); @@ -658,29 +561,15 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce ).actionGet(); // data stream was rolled over and has 4 indices, 2 managed by ILM, and 2 managed by the custom data stream lifecycle - assertBusy(() -> { - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request( - TEST_REQUEST_TIMEOUT, - new String[] { dataStreamName } - ); - GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) - .actionGet(); - assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); - assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().size(), is(4)); - - }); + backingIndices = waitForDataStreamBackingIndices(dataStreamName, 4); + String fourthGenerationIndex = backingIndices.get(3); assertBusy(() -> { - List backingIndices = getBackingIndices(dataStreamName); - String firstGenerationIndex = backingIndices.get(0); - String secondGenerationIndex = backingIndices.get(1); - String thirdGenerationIndex = backingIndices.get(2); - String writeIndex = backingIndices.get(3); ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex, - writeIndex + fourthGenerationIndex ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); @@ -698,13 +587,16 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce IndexLifecycleExplainResponse thirdGenerationExplain = explainResponse.getIndexResponses().get(thirdGenerationIndex); assertThat(thirdGenerationExplain.managedByILM(), is(false)); - IndexLifecycleExplainResponse writeIndexExplain = explainResponse.getIndexResponses().get(writeIndex); + IndexLifecycleExplainResponse writeIndexExplain = explainResponse.getIndexResponses().get(fourthGenerationIndex); assertThat(writeIndexExplain.managedByILM(), is(false)); // the third generation and the write index are managed by the data stream lifecycle ExplainDataStreamLifecycleAction.Response dataStreamLifecycleExplainResponse = client().execute( ExplainDataStreamLifecycleAction.INSTANCE, - new ExplainDataStreamLifecycleAction.Request(TEST_REQUEST_TIMEOUT, new String[] { thirdGenerationIndex, writeIndex }) + new ExplainDataStreamLifecycleAction.Request( + TEST_REQUEST_TIMEOUT, + new String[] { thirdGenerationIndex, fourthGenerationIndex } + ) ).actionGet(); assertThat(dataStreamLifecycleExplainResponse.getIndices().size(), is(2)); for (ExplainIndexDataStreamLifecycle index : dataStreamLifecycleExplainResponse.getIndices()) { @@ -728,16 +620,11 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce // At this point the data stream is not managed by the data stream lifecycle and all the indices fallback to ILM. assertBusy(() -> { - List backingIndices = getBackingIndices(dataStreamName); - String firstGenerationIndex = backingIndices.get(0); - String secondGenerationIndex = backingIndices.get(1); - String thirdGenerationIndex = backingIndices.get(2); - String writeIndex = backingIndices.get(3); ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex, - writeIndex + fourthGenerationIndex ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); @@ -758,7 +645,7 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce assertThat(thirdGenerationExplain.getStep(), is(PhaseCompleteStep.NAME)); // the write index is managed by the ILM as well - IndexLifecycleExplainResponse writeIndexExplain = explainResponse.getIndexResponses().get(writeIndex); + IndexLifecycleExplainResponse writeIndexExplain = explainResponse.getIndexResponses().get(fourthGenerationIndex); assertThat(writeIndexExplain.managedByILM(), is(true)); assertThat(writeIndexExplain.getPhase(), is("hot")); assertThat(writeIndexExplain.getStep(), is(WaitForRolloverReadyStep.NAME)); @@ -779,27 +666,18 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw // this will create the data stream and trigger a rollover so we will end up with a data stream with 2 backing indices indexDocs(dataStreamName, 1); - assertBusy(() -> { - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request( - TEST_REQUEST_TIMEOUT, - new String[] { dataStreamName } - ); - GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) - .actionGet(); - assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); - assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().size(), is(2)); - - }); + List backingIndices = waitForDataStreamBackingIndices(dataStreamName, 2); + String firstGenerationIndex = backingIndices.get(0); + String secondGenerationIndex = backingIndices.get(1); assertBusy(() -> { - List backingIndices = getBackingIndices(dataStreamName); - String firstGenerationIndex = backingIndices.get(0); - String writeIndex = backingIndices.get(1); - // let's check the indices are managed by the data stream lifecycle ExplainDataStreamLifecycleAction.Response dataStreamLifecycleExplainResponse = client().execute( ExplainDataStreamLifecycleAction.INSTANCE, - new ExplainDataStreamLifecycleAction.Request(TEST_REQUEST_TIMEOUT, new String[] { firstGenerationIndex, writeIndex }) + new ExplainDataStreamLifecycleAction.Request( + TEST_REQUEST_TIMEOUT, + new String[] { firstGenerationIndex, secondGenerationIndex } + ) ).actionGet(); assertThat(dataStreamLifecycleExplainResponse.getIndices().size(), is(2)); for (ExplainIndexDataStreamLifecycle index : dataStreamLifecycleExplainResponse.getIndices()) { @@ -835,23 +713,10 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw indexDocs(dataStreamName, 1); - assertBusy(() -> { - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request( - TEST_REQUEST_TIMEOUT, - new String[] { dataStreamName } - ); - GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) - .actionGet(); - assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); - assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().size(), is(3)); - - }); + backingIndices = waitForDataStreamBackingIndices(dataStreamName, 3); + String thirdGenerationIndex = backingIndices.get(2); assertBusy(() -> { - List backingIndices = getBackingIndices(dataStreamName); - String firstGenerationIndex = backingIndices.get(0); - String secondGenerationIndex = backingIndices.get(1); - String writeIndex = backingIndices.get(2); // let's check the previous indices are managed by the data stream lifecycle ExplainDataStreamLifecycleAction.Response dataStreamLifecycleExplainResponse = client().execute( @@ -866,10 +731,10 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw assertThat(index.isManagedByLifecycle(), is(true)); } - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(writeIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(thirdGenerationIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); - IndexLifecycleExplainResponse writeIndexExplain = explainResponse.getIndexResponses().get(writeIndex); + IndexLifecycleExplainResponse writeIndexExplain = explainResponse.getIndexResponses().get(thirdGenerationIndex); assertThat(writeIndexExplain.managedByILM(), is(true)); assertThat(writeIndexExplain.getPhase(), is("hot")); assertThat(writeIndexExplain.getStep(), is(WaitForRolloverReadyStep.NAME)); @@ -877,27 +742,16 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw // rollover should now happen when indexing 2 documents (as configured in ILM) indexDocs(dataStreamName, 2); - assertBusy(() -> { - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request( - TEST_REQUEST_TIMEOUT, - new String[] { dataStreamName } - ); - GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) - .actionGet(); - assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); - assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().size(), is(4)); - }); + backingIndices = waitForDataStreamBackingIndices(dataStreamName, 4); + String fourthGenerationIndex = backingIndices.get(3); // the new write index is also managed by ILM assertBusy(() -> { - List backingIndices = getBackingIndices(dataStreamName); - String thirdGenerationIndex = backingIndices.get(2); - String writeIndex = backingIndices.get(3); ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( thirdGenerationIndex, - writeIndex + fourthGenerationIndex ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); @@ -906,7 +760,7 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw assertThat(thirdGenerationExplain.getPhase(), is("hot")); assertThat(thirdGenerationExplain.getStep(), is(PhaseCompleteStep.NAME)); - IndexLifecycleExplainResponse writeIndexExplain = explainResponse.getIndexResponses().get(writeIndex); + IndexLifecycleExplainResponse writeIndexExplain = explainResponse.getIndexResponses().get(fourthGenerationIndex); assertThat(writeIndexExplain.managedByILM(), is(true)); assertThat(writeIndexExplain.getPhase(), is("hot")); assertThat(writeIndexExplain.getStep(), is(WaitForRolloverReadyStep.NAME)); @@ -939,16 +793,7 @@ public void testGetDataStreamResponse() throws Exception { indexDocs(dataStreamName, 2); // wait to rollover - assertBusy(() -> { - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request( - TEST_REQUEST_TIMEOUT, - new String[] { dataStreamName } - ); - GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) - .actionGet(); - assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); - assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().size(), is(2)); - }); + waitForDataStreamBackingIndices(dataStreamName, 2); // prefer_ilm false in the index template putComposableIndexTemplate( @@ -991,36 +836,21 @@ public void testGetDataStreamResponse() throws Exception { // the template ILM policy should still be reflected at the top level assertThat(dataStreamInfo.getIlmPolicy(), is(policy)); - List backingIndices = getBackingIndices(dataStreamName); - String firstGenerationIndex = backingIndices.get(0); - String secondGenerationIndex = backingIndices.get(1); - String writeIndex = backingIndices.get(2); - assertThat( - indices.stream().map(i -> i.getName()).toList(), - containsInAnyOrder(firstGenerationIndex, secondGenerationIndex, writeIndex) - ); - - Function> backingIndexSupplier = indexName -> indices.stream() - .filter(index -> index.getName().equals(indexName)) - .findFirst(); + Index firstGenSettings = indices.get(0); + Index secondGenSettings = indices.get(1); + Index writeIndexSettings = indices.get(2); // let's assert the policy is reported for all indices (as it's present in the index template) and the value of the // prefer_ilm setting remains true for the first 2 generations and is false for the write index (the generation after rollover) - Optional firstGenSettings = backingIndexSupplier.apply(firstGenerationIndex); - assertThat(firstGenSettings.isPresent(), is(true)); - assertThat(dataStreamInfo.getIndexSettingsValues().get(firstGenSettings.get()).preferIlm(), is(true)); - assertThat(dataStreamInfo.getIndexSettingsValues().get(firstGenSettings.get()).ilmPolicyName(), is(policy)); - assertThat(dataStreamInfo.getIndexSettingsValues().get(firstGenSettings.get()).managedBy(), is(ManagedBy.ILM)); - Optional secondGenSettings = backingIndexSupplier.apply(secondGenerationIndex); - assertThat(secondGenSettings.isPresent(), is(true)); - assertThat(dataStreamInfo.getIndexSettingsValues().get(secondGenSettings.get()).preferIlm(), is(true)); - assertThat(dataStreamInfo.getIndexSettingsValues().get(secondGenSettings.get()).ilmPolicyName(), is(policy)); - assertThat(dataStreamInfo.getIndexSettingsValues().get(secondGenSettings.get()).managedBy(), is(ManagedBy.ILM)); - Optional writeIndexSettings = backingIndexSupplier.apply(writeIndex); - assertThat(writeIndexSettings.isPresent(), is(true)); - assertThat(dataStreamInfo.getIndexSettingsValues().get(writeIndexSettings.get()).preferIlm(), is(false)); - assertThat(dataStreamInfo.getIndexSettingsValues().get(writeIndexSettings.get()).ilmPolicyName(), is(policy)); - assertThat(dataStreamInfo.getIndexSettingsValues().get(writeIndexSettings.get()).managedBy(), is(ManagedBy.LIFECYCLE)); + assertThat(dataStreamInfo.getIndexSettingsValues().get(firstGenSettings).preferIlm(), is(true)); + assertThat(dataStreamInfo.getIndexSettingsValues().get(firstGenSettings).ilmPolicyName(), is(policy)); + assertThat(dataStreamInfo.getIndexSettingsValues().get(firstGenSettings).managedBy(), is(ManagedBy.ILM)); + assertThat(dataStreamInfo.getIndexSettingsValues().get(secondGenSettings).preferIlm(), is(true)); + assertThat(dataStreamInfo.getIndexSettingsValues().get(secondGenSettings).ilmPolicyName(), is(policy)); + assertThat(dataStreamInfo.getIndexSettingsValues().get(secondGenSettings).managedBy(), is(ManagedBy.ILM)); + assertThat(dataStreamInfo.getIndexSettingsValues().get(writeIndexSettings).preferIlm(), is(false)); + assertThat(dataStreamInfo.getIndexSettingsValues().get(writeIndexSettings).ilmPolicyName(), is(policy)); + assertThat(dataStreamInfo.getIndexSettingsValues().get(writeIndexSettings).managedBy(), is(ManagedBy.LIFECYCLE)); // with the current configuratino, the next generation index will be managed by DSL assertThat(dataStreamInfo.getNextGenerationManagedBy(), is(ManagedBy.LIFECYCLE)); @@ -1104,14 +934,4 @@ static void putComposableIndexTemplate( ); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); } - - private List getBackingIndices(String dataStreamName) { - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request( - TEST_REQUEST_TIMEOUT, - new String[] { dataStreamName } - ); - GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) - .actionGet(); - return getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().stream().map(Index::getName).toList(); - } }