Fix IndexTemplateRegistryRolloverIT (elastic#127695)

My theory of why this test failed is that when we waited for the data
stream to be marked for rollover, that condition was met on one of the
nodes, causing us to index the documents to trigger the rollover, but
the master node did not fully process the mark-for-rollover cluster
state, resulting in the documents ending up in the first backing index.

Therefore, we use `awaitClusterState` to wait for the data stream to be
marked for rollover, which waits on the master node by default.

Fixes elastic#127692

Author: nielsbauman

muted-tests.yml

@@ -435,9 +435,6 @@ tests:
 - class: org.elasticsearch.snapshots.SnapshotShutdownIT
   method: testSnapshotShutdownProgressTracker
   issue: https://github.com/elastic/elasticsearch/issues/127690
-- class: org.elasticsearch.xpack.core.template.IndexTemplateRegistryRolloverIT
-  method: testRollover
-  issue: https://github.com/elastic/elasticsearch/issues/127692
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
   method: test {p0=search/350_point_in_time/point-in-time with index filter}
   issue: https://github.com/elastic/elasticsearch/issues/127741

x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryRolloverIT.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.project.TestProjectResolvers;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.datastreams.DataStreamsPlugin;
@@ -68,7 +69,8 @@ public void setup() {
     public void testRollover() throws Exception {
         ClusterState state = clusterService.state();
         registry.clusterChanged(new ClusterChangedEvent(IndexTemplateRegistryRolloverIT.class.getName(), state, state));
-        assertBusy(() -> { assertTrue(clusterService.state().metadata().getProject().templatesV2().containsKey(TEST_INDEX_TEMPLATE_ID)); });
+        final var projectId = ProjectId.DEFAULT;
+        awaitClusterState(s -> s.metadata().getProject(projectId).templatesV2().containsKey(TEST_INDEX_TEMPLATE_ID));
         String dsName = TEST_INDEX_PATTERN.replace('*', '1');
         CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
             TEST_REQUEST_TIMEOUT,
@@ -80,7 +82,7 @@ public void testRollover() throws Exception {
         assertNumberOfBackingIndices(1);
         registry.incrementVersion();
         registry.clusterChanged(new ClusterChangedEvent(IndexTemplateRegistryRolloverIT.class.getName(), clusterService.state(), state));
-        assertBusy(() -> assertTrue(getDataStream().rolloverOnWrite()));
+        awaitClusterState(s -> s.metadata().getProject(projectId).dataStreams().get(dsName).rolloverOnWrite());
         assertNumberOfBackingIndices(1);
 
         String timestampValue = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis());
@@ -89,7 +91,7 @@ public void testRollover() throws Exception {
                 .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, timestampValue), XContentType.JSON)
         ).actionGet();
         assertThat(docWriteResponse.status().getStatus(), equalTo(201));
-        assertBusy(() -> assertNumberOfBackingIndices(2));
+        awaitClusterState(s -> s.metadata().getProject(projectId).dataStreams().get(dsName).getIndices().size() == 2);
     }
 
     private void assertNumberOfBackingIndices(final int expected) {