Change bad file settings when node is shut down (elastic#137978) (elastic#138932)

testNewErrorOnRestartProcessing modifies file settings with a different
error case. However, it checks the error hasn't yet been processed
before stopping the node. The point of the test is to check restarting
processes the file, but since there is an error, the version check is
never applied, so the new error state may materialize before the node is
restarted. This commit moves the file settings modification to occur
after the node is stopped.

closes elastic#131098

Author: rjernst

server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java

@@ -138,9 +138,7 @@ public static void writeJSONFile(String node, String json, Logger logger, Long v
     }
 
     public static void writeJSONFile(String node, String json, Logger logger, Long version, Path targetPath) throws Exception {
-        FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, node);
-
-        Files.createDirectories(fileSettingsService.watchedFileDir());
+        Files.createDirectories(targetPath.getParent());
         Path tempFilePath = createTempFile();
 
         String jsonWithVersion = Strings.format(json, version);
@@ -519,11 +517,19 @@ public void testNewErrorOnRestartReprocessing() throws Exception {
         assertClusterStateNotSaved(savedClusterState.v1(), metadataVersion);
         assertHasErrors(metadataVersion, "not_cluster_settings");
 
-        // write json with new error without version increment to simulate ES failing to process settings after a restart for a new reason
-        // (usually, this would be due to a code change)
-        writeJSONFile(masterNode, testOtherErrorJSON, logger, versionCounter.get());
-        assertHasErrors(metadataVersion, "not_cluster_settings");
-        internalCluster().restartNode(masterNode);
+        // capture the watched file settings file before shutting down the master node
+        FileSettingsService fileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
+        Path fileSettingsFile = fileSettingsService.watchedFile();
+
+        internalCluster().restartNode(masterNode, new InternalTestCluster.RestartCallback() {
+            @Override
+            public Settings onNodeStopped(String nodeName) throws Exception {
+                // write json with new error without version increment to simulate ES failing to process settings after a restart for
+                // a new reason (usually, this would be due to a code change)
+                writeJSONFile(masterNode, testOtherErrorJSON, logger, versionCounter.get(), fileSettingsFile);
+                return Settings.EMPTY;
+            }
+        });
         ensureGreen();
 
         assertBusy(() -> assertHasErrors(metadataVersion, "bad_cluster_settings"));