Use assertBusy for waiting on file settings watching in readiness test (elastic#136131)

The readiness check depends on file settings being applied. The
readiness tests need to wait on the file settings service watching for
changes. This commit updates the readiness tests to spin on the file
settings not yet watching.

closes elastic#108613

Author: rjernst

server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java

@@ -109,7 +109,7 @@ protected Collection<Class<? extends Plugin>> getMockPlugins() {
         plugins.add(MockReadinessService.TestPlugin.class);
         return Collections.unmodifiableList(plugins);
     }
-
+    
     private void assertMasterNode(Client client, String node) {
         assertThat(
             client.admin().cluster().prepareState(TEST_REQUEST_TIMEOUT).get().getState().nodes().getMasterNode().getName(),
@@ -124,7 +124,6 @@ private void expectMasterNotFound() {
         );
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/108613")
     public void testReadinessDuringRestarts() throws Exception {
         internalCluster().setBootstrapMasterNodeIndex(0);
         writeFileSettings(testJSON);
@@ -276,7 +275,7 @@ public void testNotReadyOnBadFileSettings() throws Exception {
 
         FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
 
-        assertTrue(masterFileSettingsService.watching());
+        assertBusy(() -> assertTrue(masterFileSettingsService.watching()));
         assertFalse(dataFileSettingsService.watching());
 
         boolean awaitSuccessful = savedClusterState.await(20, TimeUnit.SECONDS);
@@ -333,7 +332,7 @@ public void testReadyWhenMissingFileSettings() throws Exception {
 
         FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
 
-        assertTrue(masterFileSettingsService.watching());
+        assertBusy(() -> assertTrue(masterFileSettingsService.watching()));
 
         boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
         assertTrue(awaitSuccessful);
@@ -387,7 +386,7 @@ public void testReadyAfterCorrectFileSettings() throws Exception {
 
         FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
 
-        assertTrue(masterFileSettingsService.watching());
+        assertBusy(() -> assertTrue(masterFileSettingsService.watching()));
         assertFalse(dataFileSettingsService.watching());
 
         boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);