onProcessFileChangesException (elastic#115038) (elastic#134909)

Author: prdoyle

server/src/main/java/org/elasticsearch/common/file/AbstractFileWatchingService.java

@@ -315,12 +315,20 @@ void processSettingsOnServiceStartAndNotifyListeners() throws InterruptedExcepti
     void processSettingsAndNotifyListeners() throws InterruptedException {
         try {
             processFileChanges();
-            for (var listener : eventListeners) {
-                listener.watchedFileChanged();
-            }
         } catch (IOException | ExecutionException e) {
-            logger.error(() -> "Error processing watched file: " + watchedFile(), e);
+            onProcessFileChangesException(e);
+            return;
         }
+        for (var listener : eventListeners) {
+            listener.watchedFileChanged();
+        }
+    }
+
+    /**
+     * Called for checked exceptions only.
+     */
+    protected void onProcessFileChangesException(Exception e) {
+        logger.error(() -> "Error processing watched file: " + watchedFile(), e);
     }
 
     // package private for testing

server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsService.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.ReservedStateMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -149,7 +150,16 @@ private void processFileChanges(ReservedStateVersionCheck versionCheck) throws I
     }
 
     @Override
-    protected void processInitialFileMissing() throws ExecutionException, InterruptedException {
+    protected void onProcessFileChangesException(Exception e) {
+        if (e instanceof ExecutionException && e.getCause() instanceof FailedToCommitClusterStateException f) {
+            logger.error("Unable to commit cluster state", e);
+        } else {
+            super.onProcessFileChangesException(e);
+        }
+    }
+
+    @Override
+    protected void processInitialFileMissing() throws ExecutionException, InterruptedException, IOException {
         PlainActionFuture<ActionResponse.Empty> completion = new PlainActionFuture<>();
         logger.info("setting file [{}] not found, initializing [{}] as empty", watchedFile(), NAMESPACE);
         stateService.initEmpty(NAMESPACE, completion);