Log warning instead of error for some SLM logs during master shutdown

Author: samxbr

server/src/main/java/org/elasticsearch/shutdown/PluginShutdownService.java

@@ -46,6 +46,14 @@ public static Set<String> shutdownNodes(final ClusterState clusterState) {
         return clusterState.metadata().nodeShutdowns().getAllNodeIds();
     }
 
+    /**
+     * Return if the current node is requested to shut down
+     */
+    public static boolean isLocalNodeShutdown(final ClusterState clusterState) {
+        String localNodeId = clusterState.nodes().getLocalNodeId();
+        return localNodeId != null && shutdownNodes(clusterState).contains(localNodeId);
+    }
+
     /**
      * Return all nodes shutting down with the given shutdown types from the given cluster state
      */

x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java

@@ -151,7 +151,11 @@ public void onResponse(CreateSnapshotResponse createSnapshotResponse) {
 
                 @Override
                 public void onFailure(Exception e) {
-                    logger.error("failed to create snapshot for snapshot lifecycle policy [{}]: {}", policyMetadata.getPolicy().getId(), e);
+                    SnapshotHistoryStore.logErrorOrWarning(
+                        clusterService.state(),
+                        () -> format("failed to create snapshot for snapshot lifecycle policy [{}]", policyMetadata.getPolicy().getId()),
+                        e
+                    );
                     final long timestamp = Instant.now().toEpochMilli();
                     submitUnbatchedTask(
                         clusterService,

x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/history/SnapshotHistoryStore.java

@@ -9,12 +9,15 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.shutdown.PluginShutdownService;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
@@ -83,20 +86,31 @@ public void putAsync(SnapshotHistoryItem item) {
                     SLM_HISTORY_DATA_STREAM,
                     item
                 );
-            }, exception -> {
-                logger.error(
+            },
+                exception -> logErrorOrWarning(
+                    clusterService.state(),
                     () -> format("failed to index snapshot history item in data stream [%s]: [%s]", SLM_HISTORY_DATA_STREAM, item),
                     exception
-                );
-            }));
+                )
+            ));
         } catch (IOException exception) {
-            logger.error(
+            logErrorOrWarning(
+                clusterService.state(),
                 () -> format("failed to index snapshot history item in data stream [%s]: [%s]", SLM_HISTORY_DATA_STREAM, item),
                 exception
             );
         }
     }
 
+    // On node shutdown, some operations are expected to fail, we log a warning instead of error during node shutdown for those exceptions
+    public static void logErrorOrWarning(ClusterState clusterState, Supplier<?> failureMsgSupplier, Exception exception) {
+        if (PluginShutdownService.isLocalNodeShutdown(clusterState)) {
+            logger.warn(failureMsgSupplier, exception);
+        } else {
+            logger.error(failureMsgSupplier, exception);
+        }
+    }
+
     public void setSlmHistoryEnabled(boolean slmHistoryEnabled) {
         this.slmHistoryEnabled = slmHistoryEnabled;
     }