Fix failing test(s) in TimeSeriesDataStreamsIT

When these tests were run around midnight, the use of
`DataStream#getDefaultBackingIndexName` could result in a potential
mismatch in the generated index name and the one that the cluster
actually created. Instead, we need to obtain the backing index and
extract the desired index name from there.

Fixes #123086
Relates #123376

Author: nielsbauman

x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java

@@ -7,10 +7,10 @@
 
 package org.elasticsearch.xpack.ilm;
 
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.WarningFailureException;
-import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -81,20 +81,13 @@ public void testRolloverAction() throws Exception {
 
         indexDocument(client(), dataStream, true);
 
-        assertBusy(() -> assertTrue(indexExists(DataStream.getDefaultBackingIndexName(dataStream, 2))));
-        assertBusy(
-            () -> assertTrue(
-                Boolean.parseBoolean(
-                    (String) getIndexSettingsAsMap(DataStream.getDefaultBackingIndexName(dataStream, 2)).get("index.hidden")
-                )
-            )
-        );
-        assertBusy(
-            () -> assertThat(
-                getStepKeyForIndex(client(), DataStream.getDefaultBackingIndexName(dataStream, 1)),
-                equalTo(PhaseCompleteStep.finalStep("hot").getKey())
-            )
-        );
+        assertBusy(() -> {
+            final var backingIndices = getBackingIndices(client(), dataStream);
+            assertEquals(2, backingIndices.size());
+            final var writeIndex = backingIndices.getFirst();
+            assertTrue(Boolean.parseBoolean((String) getIndexSettingsAsMap(writeIndex).get("index.hidden")));
+            assertEquals(PhaseCompleteStep.finalStep("hot").getKey(), getStepKeyForIndex(client(), writeIndex));
+        });
     }
 
     public void testRolloverIsSkippedOnManualDataStreamRollover() throws Exception {
@@ -104,20 +97,24 @@ public void testRolloverIsSkippedOnManualDataStreamRollover() throws Exception {
 
         indexDocument(client(), dataStream, true);
 
-        String firstGenerationIndex = DataStream.getDefaultBackingIndexName(dataStream, 1);
-        assertBusy(
-            () -> assertThat(getStepKeyForIndex(client(), firstGenerationIndex).name(), equalTo(WaitForRolloverReadyStep.NAME)),
-            30,
-            TimeUnit.SECONDS
-        );
+        final var firstGenerationIndex = new SetOnce<String>();
+        assertBusy(() -> {
+            final var backingIndices = getBackingIndices(client(), dataStream);
+            assertEquals(1, backingIndices.size());
+            assertEquals(WaitForRolloverReadyStep.NAME, getStepKeyForIndex(client(), backingIndices.getFirst()).name());
+            firstGenerationIndex.set(backingIndices.getFirst());
+        }, 30, TimeUnit.SECONDS);
 
         rolloverMaxOneDocCondition(client(), dataStream);
-        assertBusy(() -> assertThat(indexExists(DataStream.getDefaultBackingIndexName(dataStream, 2)), is(true)), 30, TimeUnit.SECONDS);
+        assertBusy(() -> {
+            final var backingIndices = getBackingIndices(client(), dataStream);
+            assertEquals(2, backingIndices.size());
+        }, 30, TimeUnit.SECONDS);
 
         // even though the first index doesn't have 2 documents to fulfill the rollover condition, it should complete the rollover action
         // because it's not the write index anymore
         assertBusy(
-            () -> assertThat(getStepKeyForIndex(client(), firstGenerationIndex), equalTo(PhaseCompleteStep.finalStep("hot").getKey())),
+            () -> assertEquals(PhaseCompleteStep.finalStep("hot").getKey(), getStepKeyForIndex(client(), firstGenerationIndex.get())),
             30,
             TimeUnit.SECONDS
         );
@@ -129,7 +126,7 @@ public void testShrinkActionInPolicyWithoutHotPhase() throws Exception {
         createComposableTemplate(client(), template, dataStream + "*", getTemplate(policyName));
         indexDocument(client(), dataStream, true);
 
-        String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1);
+        String backingIndexName = getBackingIndices(client(), dataStream).getFirst();
         assertBusy(
             () -> assertThat(
                 "original index must wait in the " + CheckNotDataStreamWriteIndexStep.NAME + " until it is not the write index anymore",
@@ -143,8 +140,11 @@ public void testShrinkActionInPolicyWithoutHotPhase() throws Exception {
         // Manual rollover the original index such that it's not the write index in the data stream anymore
         rolloverMaxOneDocCondition(client(), dataStream);
         // Wait for rollover to happen
-        String rolloverIndex = DataStream.getDefaultBackingIndexName(dataStream, 2);
-        assertBusy(() -> assertTrue("the rollover action created the rollover index", indexExists(rolloverIndex)), 30, TimeUnit.SECONDS);
+        assertBusy(
+            () -> assertEquals("the rollover action created the rollover index", 2, getBackingIndices(client(), dataStream).size()),
+            30,
+            TimeUnit.SECONDS
+        );
 
         String shrunkenIndex = waitAndGetShrinkIndexName(client(), backingIndexName);
         assertBusy(() -> assertTrue(indexExists(shrunkenIndex)), 30, TimeUnit.SECONDS);
@@ -160,7 +160,7 @@ public void testSearchableSnapshotAction() throws Exception {
         createComposableTemplate(client(), template, dataStream + "*", getTemplate(policyName));
         indexDocument(client(), dataStream, true);
 
-        String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1);
+        String backingIndexName = getBackingIndices(client(), dataStream).getFirst();
         String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName;
 
         assertBusy(
@@ -191,7 +191,7 @@ public void testReadOnlyAction() throws Exception {
         createComposableTemplate(client(), template, dataStream + "*", getTemplate(policyName));
         indexDocument(client(), dataStream, true);
 
-        String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1);
+        String backingIndexName = getBackingIndices(client(), dataStream).getFirst();
         assertBusy(
             () -> assertThat(
                 "index must wait in the " + CheckNotDataStreamWriteIndexStep.NAME + " until it is not the write index anymore",
@@ -226,7 +226,7 @@ public void testFreezeAction() throws Exception {
         indexDocument(client(), dataStream, true);
 
         // The freeze action is a noop action with only noop steps and should pass through to complete the phase asap.
-        String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1);
+        String backingIndexName = getBackingIndices(client(), dataStream).getFirst();
         assertBusy(() -> {
             try {
                 assertThat(explainIndex(client(), backingIndexName).get("step"), is(PhaseCompleteStep.NAME));
@@ -248,7 +248,7 @@ public void checkForceMergeAction(String codec) throws Exception {
         createComposableTemplate(client(), template, dataStream + "*", getTemplate(policyName));
         indexDocument(client(), dataStream, true);
 
-        String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1);
+        String backingIndexName = getBackingIndices(client(), dataStream).getFirst();
         assertBusy(
             () -> assertThat(
                 "index must wait in the " + CheckNotDataStreamWriteIndexStep.NAME + " until it is not the write index anymore",