Fix test: set up mocks before starting the service (#126719)

* Change testInvalidJSON to set up mocks before starting the service

* Move another mock setup before the service starts

Author: prdoyle

muted-tests.yml

@@ -150,9 +150,6 @@ tests:
 - class: org.elasticsearch.xpack.ccr.FollowIndexSecurityIT
   method: testCleanShardFollowTaskAfterDeleteFollower
   issue: https://github.com/elastic/elasticsearch/issues/120339
-- class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
-  method: testInvalidJSON
-  issue: https://github.com/elastic/elasticsearch/issues/120482
 - class: org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeToCharProcessorTests
   issue: https://github.com/elastic/elasticsearch/issues/120575
 - class: org.elasticsearch.xpack.inference.DefaultEndPointsIT

server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java

@@ -275,6 +275,7 @@ public void testProcessFileChanges() throws Exception {
             return null;
         }).when(controller).process(any(), any(XContentParser.class), any(), any());
 
+        // Await on some latches when files change so we can sync up
         CountDownLatch processFileCreationLatch = new CountDownLatch(1);
         doAnswer(i -> {
             try {
@@ -283,25 +284,28 @@ public void testProcessFileChanges() throws Exception {
                 processFileCreationLatch.countDown();
             }
         }).when(fileSettingsService).processFile(eq(watchedFile), eq(true));
+        CountDownLatch processFileChangeLatch = new CountDownLatch(1);
+        doAnswer(i -> {
+            try {
+                return i.callRealMethod();
+            } finally {
+                processFileChangeLatch.countDown();
+            }
+        }).when(fileSettingsService).processFile(eq(watchedFile), eq(false));
 
         Files.createDirectories(fileSettingsService.watchedFileDir());
         // contents of the JSON don't matter, we just need a file to exist
         writeTestFile(watchedFile, "{}");
 
+        // It's important to configure all the mocks before calling start() here,
+        // because otherwise there can be races between configuration and use of mocks
+        // which leads to a UnfinishedStubbingException.
+
         fileSettingsService.start();
         fileSettingsService.clusterChanged(new ClusterChangedEvent("test", clusterService.state(), ClusterState.EMPTY_STATE));
 
         longAwait(processFileCreationLatch);
 
-        CountDownLatch processFileChangeLatch = new CountDownLatch(1);
-        doAnswer(i -> {
-            try {
-                return i.callRealMethod();
-            } finally {
-                processFileChangeLatch.countDown();
-            }
-        }).when(fileSettingsService).processFile(eq(watchedFile), eq(false));
-
         verify(fileSettingsService, times(1)).processFile(eq(watchedFile), eq(true));
         verify(controller, times(1)).process(any(), any(XContentParser.class), eq(ReservedStateVersionCheck.HIGHER_OR_SAME_VERSION), any());
 
@@ -328,10 +332,8 @@ public void testInvalidJSON() throws Exception {
         // Don't really care about the initial state
         Files.createDirectories(fileSettingsService.watchedFileDir());
         doNothing().when(fileSettingsService).processInitialFilesMissing();
-        fileSettingsService.start();
-        fileSettingsService.clusterChanged(new ClusterChangedEvent("test", clusterService.state(), ClusterState.EMPTY_STATE));
 
-        // Now break the JSON and wait
+        // Prepare to await on a barrier when the file changes so we can sync up
         CyclicBarrier fileChangeBarrier = new CyclicBarrier(2);
         doAnswer((Answer<?>) invocation -> {
             try {
@@ -340,6 +342,12 @@ public void testInvalidJSON() throws Exception {
                 awaitOrBust(fileChangeBarrier);
             }
         }).when(fileSettingsService).onProcessFileChangesException(eq(watchedFile), any());
+
+        // Kick off the service
+        fileSettingsService.start();
+        fileSettingsService.clusterChanged(new ClusterChangedEvent("test", clusterService.state(), ClusterState.EMPTY_STATE));
+
+        // Now break the JSON and wait
         writeTestFile(watchedFile, "test_invalid_JSON");
         awaitOrBust(fileChangeBarrier);