Suppress MasterService exception log at shutdown (#94976)

Today if the `MasterService` closes mid-publication then it emits a
large `WARN` log to report the `EsRejectedExecutionException`. This is
unnecessary and potentially confusing since it is not unexpected and
needs no action from the user. With this commit we drop the logging
level to `DEBUG`.

Author: DaveCTurner

server/src/main/java/org/elasticsearch/cluster/service/MasterService.java

@@ -394,9 +394,21 @@ public void onFailure(Exception exception) {
                             clusterStatePublicationEvent,
                             notificationMillis
                         );
+                    } else if (exception instanceof EsRejectedExecutionException esRejectedExecutionException) {
+                        assert esRejectedExecutionException.isExecutorShutdown();
+                        clusterStateUpdateStatsTracker.onPublicationFailure(
+                            threadPool.rawRelativeTimeInMillis(),
+                            clusterStatePublicationEvent,
+                            0L
+                        );
+                        final long version = newClusterState.version();
+                        logger.debug(
+                            () -> format("shut down during publication of cluster state version [%s]: [%s]", version, summary),
+                            exception
+                        );
+                        // TODO also bubble the failure up to the tasks too, see https://github.com/elastic/elasticsearch/issues/94930
                     } else {
-                        assert publicationMayFail() || (exception instanceof EsRejectedExecutionException esre && esre.isExecutorShutdown())
-                            : exception;
+                        assert publicationMayFail() : exception;
                         clusterStateUpdateStatsTracker.onPublicationFailure(
                             threadPool.rawRelativeTimeInMillis(),
                             clusterStatePublicationEvent,

server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java

@@ -38,6 +38,7 @@
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
@@ -2099,7 +2100,7 @@ public void onFailure(Exception e) {
         }
     }
 
-    public void testRejectionBehaviour() {
+    public void testRejectionBehaviourAtSubmission() {
 
         final var deterministicTaskQueue = new DeterministicTaskQueue();
         final var threadPool = deterministicTaskQueue.getThreadPool();
@@ -2167,6 +2168,81 @@ public void onFailure(Exception e) {
         }
     }
 
+    @TestLogging(reason = "verifying DEBUG logs", value = "org.elasticsearch.cluster.service.MasterService:DEBUG")
+    public void testRejectionBehaviourAtCompletion() {
+
+        final var deterministicTaskQueue = new DeterministicTaskQueue();
+        final var threadPool = deterministicTaskQueue.getThreadPool();
+        final var threadPoolExecutor = new StoppableExecutorServiceWrapper(threadPool.generic()) {
+
+            boolean executedTask = false;
+
+            @Override
+            public void execute(Runnable command) {
+                if (command instanceof AbstractRunnable abstractRunnable) {
+                    if (executedTask) {
+                        abstractRunnable.onRejection(new EsRejectedExecutionException("simulated", true));
+                    } else {
+                        executedTask = true;
+                        super.execute(command);
+                    }
+                } else {
+                    fail("not an AbstractRunnable: " + command);
+                }
+            }
+        };
+
+        final var appender = new MockLogAppender();
+        appender.addExpectation(
+            new MockLogAppender.UnseenEventExpectation("warning", MasterService.class.getCanonicalName(), Level.WARN, "*")
+        );
+        appender.addExpectation(
+            new MockLogAppender.SeenEventExpectation(
+                "debug",
+                MasterService.class.getCanonicalName(),
+                Level.DEBUG,
+                "shut down during publication of cluster state version*"
+            )
+        );
+
+        try (
+            var ignored = appender.capturing(MasterService.class);
+            var masterService = createMasterService(true, null, threadPool, threadPoolExecutor)
+        ) {
+
+            final var testHeader = "test-header";
+
+            class TestTask implements ClusterStateTaskListener {
+                private final String expectedHeader = threadPool.getThreadContext().getHeader(testHeader);
+
+                @Override
+                public void onFailure(Exception e) {
+                    // post-publication rejections are currently just dropped, see https://github.com/elastic/elasticsearch/issues/94930
+                    throw new AssertionError("unexpected exception", e);
+                }
+            }
+
+            final var queue = masterService.createTaskQueue("queue", randomFrom(Priority.values()), batchExecutionContext -> {
+                for (var taskContext : batchExecutionContext.taskContexts()) {
+                    taskContext.success(() -> fail("should not succeed"));
+                }
+                return ClusterState.builder(batchExecutionContext.initialState()).build();
+            });
+
+            try (var ignoredContext = threadPool.getThreadContext().stashContext()) {
+                threadPool.getThreadContext().putHeader(testHeader, randomAlphaOfLength(10));
+                queue.submitTask("batched", new TestTask(), null);
+            }
+
+            threadPool.getThreadContext().markAsSystemContext();
+            deterministicTaskQueue.runAllTasks();
+            assertFalse(deterministicTaskQueue.hasRunnableTasks());
+            assertFalse(deterministicTaskQueue.hasDeferredTasks());
+
+            appender.assertAllExpectationsMatched();
+        }
+    }
+
     public void testLifecycleBehaviour() {
 
         final var deterministicTaskQueue = new DeterministicTaskQueue();