[ML] Making indices/aliases hidden takes a while after cluster startup (#93148) (#93149)

When a cluster is upgraded from a version that didn't have hidden
indices/aliases to one that does, we make the relevant ML indices
and aliases hidden. However, the steps to do this are split over
5 actions that run sequentially, so the work might take a second or
two after the upgraded cluster starts up. This can cause the test
that asserts on the final state to fail if it runs very quickly
after the upgraded cluster startup. The solution is to retry the
test for a few seconds to give a chance for all the upgrade actions
to complete.

Relates #93062

Author: droberts195

x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlHiddenIndicesFullClusterRestartIT.java

@@ -102,34 +102,41 @@ public void testMlIndicesBecomeHidden() throws Exception {
                 }
             }
         } else {
-            Map<String, Object> indexSettingsMap = contentAsMap(getMlIndicesSettings());
-            Map<String, Object> aliasesMap = contentAsMap(getMlAliases());
-
-            assertThat("Index settings map was: " + indexSettingsMap, indexSettingsMap, is(aMapWithSize(greaterThanOrEqualTo(4))));
-            for (Map.Entry<String, Object> e : indexSettingsMap.entrySet()) {
-                String indexName = e.getKey();
-                @SuppressWarnings("unchecked")
-                Map<String, Object> settings = (Map<String, Object>) e.getValue();
-                assertThat(settings, is(notNullValue()));
-                assertThat(
-                    "Index " + indexName + " expected to be hidden but wasn't, settings = " + settings,
-                    XContentMapValues.extractValue(settings, "settings", "index", "hidden"),
-                    is(equalTo("true"))
-                );
-            }
+            // The 5 operations in MlInitializationService.makeMlInternalIndicesHidden() run sequentially, so might
+            // not all be finished when this test runs. The desired state should exist within a few seconds of startup,
+            // hence the assertBusy().
+            assertBusy(() -> {
+                Map<String, Object> indexSettingsMap = contentAsMap(getMlIndicesSettings());
+                Map<String, Object> aliasesMap = contentAsMap(getMlAliases());
 
-            for (Tuple<List<String>, String> indexAndAlias : EXPECTED_INDEX_ALIAS_PAIRS) {
-                List<String> indices = indexAndAlias.v1();
-                String alias = indexAndAlias.v2();
-                assertThat(
-                    indexAndAlias + " expected to be hidden but wasn't, aliasesMap = " + aliasesMap,
-                    indices.stream()
-                        .anyMatch(
-                            index -> Boolean.TRUE.equals(XContentMapValues.extractValue(aliasesMap, index, "aliases", alias, "is_hidden"))
-                        ),
-                    is(true)
-                );
-            }
+                assertThat("Index settings map was: " + indexSettingsMap, indexSettingsMap, is(aMapWithSize(greaterThanOrEqualTo(4))));
+                for (Map.Entry<String, Object> e : indexSettingsMap.entrySet()) {
+                    String indexName = e.getKey();
+                    @SuppressWarnings("unchecked")
+                    Map<String, Object> settings = (Map<String, Object>) e.getValue();
+                    assertThat(settings, is(notNullValue()));
+                    assertThat(
+                        "Index " + indexName + " expected to be hidden but wasn't, settings = " + settings,
+                        XContentMapValues.extractValue(settings, "settings", "index", "hidden"),
+                        is(equalTo("true"))
+                    );
+                }
+
+                for (Tuple<List<String>, String> indexAndAlias : EXPECTED_INDEX_ALIAS_PAIRS) {
+                    List<String> indices = indexAndAlias.v1();
+                    String alias = indexAndAlias.v2();
+                    assertThat(
+                        indexAndAlias + " expected to be hidden but wasn't, aliasesMap = " + aliasesMap,
+                        indices.stream()
+                            .anyMatch(
+                                index -> Boolean.TRUE.equals(
+                                    XContentMapValues.extractValue(aliasesMap, index, "aliases", alias, "is_hidden")
+                                )
+                            ),
+                        is(true)
+                    );
+                }
+            });
         }
     }