IGNITE-27324 Stop the node in a separate thread in failure handler (#7235)

Author: ibessonov

modules/failure-handler/src/integrationTest/java/org/apache/ignite/internal/failure/handlers/FailureHandlerTest.java

@@ -46,7 +46,7 @@ protected int initialNodes() {
 
     @Test
     void testStopNodeFailureHandler() {
-        testFailureHandler(node -> new StopNodeFailureHandler(node::shutdown));
+        testFailureHandler(node -> new StopNodeFailureHandler("test-node", node::shutdown));
     }
 
     @Test

modules/failure-handler/src/main/java/org/apache/ignite/internal/failure/FailureManager.java

@@ -65,6 +65,9 @@ public class FailureManager implements FailureProcessor, IgniteComponent {
     private static final String IGNORED_FAILURE_LOG_MSG = "Possible failure suppressed according to a configured handler "
             + "[hnd={}, failureCtx={}, failureCtxId={}]";
 
+    /** Ignite node name. */
+    private final String nodeName;
+
     /** Failure processor configuration. */
     private final FailureProcessorConfiguration configuration;
 
@@ -100,7 +103,9 @@ public class FailureManager implements FailureProcessor, IgniteComponent {
      *
      * @param handler Handler.
      */
+    @TestOnly
     public FailureManager(FailureHandler handler) {
+        this.nodeName = "test-node";
         this.nodeStopper = () -> {};
         this.handler = handler;
         this.configuration = null;
@@ -109,10 +114,12 @@ public FailureManager(FailureHandler handler) {
     /**
      * Creates a new instance of a failure processor.
      *
+     * @param nodeName Node name.
      * @param nodeStopper Node stopper.
      * @param configuration Failure processor configuration.
      */
-    public FailureManager(NodeStopper nodeStopper, FailureProcessorConfiguration configuration) {
+    public FailureManager(String nodeName, NodeStopper nodeStopper, FailureProcessorConfiguration configuration) {
+        this.nodeName = nodeName;
         this.nodeStopper = nodeStopper;
         this.configuration = configuration;
     }
@@ -263,11 +270,11 @@ private synchronized void reconfigure(FailureProcessorView newConfiguration) {
                 break;
 
             case StopNodeFailureHandlerConfigurationSchema.TYPE:
-                hnd = new StopNodeFailureHandler(nodeStopper);
+                hnd = new StopNodeFailureHandler(nodeName, nodeStopper);
                 break;
 
             case StopNodeOrHaltFailureHandlerConfigurationSchema.TYPE:
-                hnd = new StopNodeOrHaltFailureHandler(nodeStopper, (StopNodeOrHaltFailureHandlerView) handlerView);
+                hnd = new StopNodeOrHaltFailureHandler(nodeName, nodeStopper, (StopNodeOrHaltFailureHandlerView) handlerView);
                 break;
 
             default:

modules/failure-handler/src/main/java/org/apache/ignite/internal/failure/handlers/StopNodeFailureHandler.java

@@ -19,23 +19,37 @@
 
 import org.apache.ignite.internal.failure.FailureContext;
 import org.apache.ignite.internal.failure.NodeStopper;
+import org.apache.ignite.internal.logger.IgniteLogger;
+import org.apache.ignite.internal.logger.Loggers;
+import org.apache.ignite.internal.thread.IgniteThreadFactory;
+import org.apache.ignite.internal.thread.ThreadOperation;
 import org.apache.ignite.internal.tostring.IgniteToStringExclude;
 import org.apache.ignite.internal.tostring.S;
 
 /**
  * Handler will stop node in case of critical error using provided {@link NodeStopper}.
  */
 public class StopNodeFailureHandler extends AbstractFailureHandler {
+    private static final IgniteLogger LOG = Loggers.forClass(StopNodeFailureHandler.class);
+
+    /** Ignite node name. */
+    private final String nodeName;
+
     @IgniteToStringExclude
     private final NodeStopper nodeStopper;
 
-    public StopNodeFailureHandler(NodeStopper nodeStopper) {
+    public StopNodeFailureHandler(String nodeName, NodeStopper nodeStopper) {
+        this.nodeName = nodeName;
         this.nodeStopper = nodeStopper;
     }
 
     @Override
     protected boolean handle(FailureContext failureCtx) {
-        nodeStopper.stopNode();
+        IgniteThreadFactory threadFactory = IgniteThreadFactory.create(nodeName, "node-stopper", true, LOG, ThreadOperation.values());
+
+        Thread nodeStopperThread = threadFactory.newThread(nodeStopper::stopNode);
+
+        nodeStopperThread.start();
 
         return true;
     }

modules/failure-handler/src/main/java/org/apache/ignite/internal/failure/handlers/StopNodeOrHaltFailureHandler.java

@@ -24,9 +24,11 @@
 import org.apache.ignite.internal.failure.handlers.configuration.StopNodeOrHaltFailureHandlerView;
 import org.apache.ignite.internal.logger.IgniteLogger;
 import org.apache.ignite.internal.logger.Loggers;
-import org.apache.ignite.internal.thread.LogUncaughtExceptionHandler;
+import org.apache.ignite.internal.thread.IgniteThreadFactory;
+import org.apache.ignite.internal.thread.ThreadOperation;
 import org.apache.ignite.internal.tostring.IgniteToStringExclude;
 import org.apache.ignite.internal.tostring.S;
+import org.jetbrains.annotations.TestOnly;
 
 /**
  * Handler will try to stop node if {@code tryStop} value is {@code true}.
@@ -42,6 +44,9 @@ public class StopNodeOrHaltFailureHandler extends AbstractFailureHandler {
      */
     private static final int KILL_EXIT_CODE = 130;
 
+    /** Ignite node name. */
+    private String nodeName;
+
     /** Node stopper. */
     @IgniteToStringExclude
     private final NodeStopper nodeStopper;
@@ -59,6 +64,7 @@ public class StopNodeOrHaltFailureHandler extends AbstractFailureHandler {
      * @param tryStop Try stop.
      * @param timeout Stop node timeout in milliseconds.
      */
+    @TestOnly
     public StopNodeOrHaltFailureHandler(NodeStopper nodeStopper, boolean tryStop, long timeout) {
         this.nodeStopper = nodeStopper;
         this.tryStop = tryStop;
@@ -68,10 +74,12 @@ public StopNodeOrHaltFailureHandler(NodeStopper nodeStopper, boolean tryStop, lo
     /**
      * Creates a new instance of a failure processor.
      *
+     * @param nodeName Node name.
      * @param nodeStopper Node stopper.
      * @param view Configuration view.
      */
-    public StopNodeOrHaltFailureHandler(NodeStopper nodeStopper, StopNodeOrHaltFailureHandlerView view) {
+    public StopNodeOrHaltFailureHandler(String nodeName, NodeStopper nodeStopper, StopNodeOrHaltFailureHandlerView view) {
+        this.nodeName = nodeName;
         this.nodeStopper = nodeStopper;
         tryStop = view.tryStop();
         timeout = view.timeoutMillis();
@@ -82,18 +90,33 @@ protected boolean handle(FailureContext failureCtx) {
         if (tryStop) {
             CountDownLatch latch = new CountDownLatch(1);
 
-            Thread stopperThread = new Thread(
+            IgniteThreadFactory stopThreadFactory = IgniteThreadFactory.create(
+                    nodeName,
+                    "node-stopper",
+                    true,
+                    LOG,
+                    ThreadOperation.values()
+            );
+
+            Thread stopperThread = stopThreadFactory.newThread(
                     () -> {
                         nodeStopper.stopNode();
 
                         latch.countDown();
-                    },
-                    "node-stopper"
+                    }
             );
-            stopperThread.setUncaughtExceptionHandler(new LogUncaughtExceptionHandler(LOG));
+
             stopperThread.start();
 
-            Thread haltOnStopTimeoutThread = new Thread(
+            IgniteThreadFactory haltThreadFactory = IgniteThreadFactory.create(
+                    nodeName,
+                    "jvm-halt-on-stop-timeout",
+                    true,
+                    LOG,
+                    ThreadOperation.values()
+            );
+
+            Thread haltOnStopTimeoutThread = haltThreadFactory.newThread(
                     () -> {
                         try {
                             if (!latch.await(timeout, TimeUnit.MILLISECONDS)) {
@@ -102,10 +125,9 @@ protected boolean handle(FailureContext failureCtx) {
                         } catch (InterruptedException e) {
                             // No-op.
                         }
-                    },
-                    "jvm-halt-on-stop-timeout"
+                    }
             );
-            haltOnStopTimeoutThread.setUncaughtExceptionHandler(new LogUncaughtExceptionHandler(LOG));
+
             haltOnStopTimeoutThread.start();
         } else {
             Runtime.getRuntime().halt(KILL_EXIT_CODE);

modules/failure-handler/src/test/java/org/apache/ignite/internal/failure/FailureProcessorTest.java

@@ -94,7 +94,7 @@ public void testIgnoredFailureTypes() {
 
     @Test
     public void testDefaultFailureHandlerConfiguration() {
-        FailureManager failureManager = new FailureManager(() -> {}, failureProcessorConfiguration);
+        FailureManager failureManager = new FailureManager("test-node", () -> {}, failureProcessorConfiguration);
 
         try {
             assertThat(failureManager.startAsync(new ComponentContext()), willSucceedFast());
@@ -114,7 +114,7 @@ public void testDefaultFailureHandlerConfiguration() {
 
     @Test
     public void testFailureProcessorReconfiguration() {
-        FailureManager failureManager = new FailureManager(() -> {}, failureProcessorConfiguration);
+        FailureManager failureManager = new FailureManager("test-node", () -> {}, failureProcessorConfiguration);
 
         try {
             assertThat(failureManager.startAsync(new ComponentContext()), willSucceedFast());

modules/failure-handler/src/test/java/org/apache/ignite/internal/failure/FailureProcessorThreadDumpThrottlingTest.java

@@ -72,7 +72,7 @@ public void testNoThreadDumps() {
 
         logInspector.start();
         try {
-            FailureManager failureManager = new FailureManager(() -> {}, disabledThreadDumpConfiguration);
+            FailureManager failureManager = new FailureManager("test-node", () -> {}, disabledThreadDumpConfiguration);
 
             try {
                 assertThat(failureManager.startAsync(new ComponentContext()), willSucceedFast());
@@ -217,7 +217,7 @@ public void testThrottlingPerFailureType() {
      * Creates a new instance of {@link FailureManager} with the given configuration and runs the test represented by {@code test} closure.
      */
     static void testFailureProcessing(FailureProcessorConfiguration configuration, Consumer<FailureProcessor> test) {
-        FailureManager failureManager = new FailureManager(() -> {}, configuration);
+        FailureManager failureManager = new FailureManager("test-node", () -> {}, configuration);
 
         try {
             assertThat(failureManager.startAsync(new ComponentContext()), willSucceedFast());

modules/runner/src/main/java/org/apache/ignite/internal/app/IgniteImpl.java

@@ -598,7 +598,7 @@ public class IgniteImpl implements Ignite {
 
         FailureProcessorConfiguration failureProcessorConfiguration = nodeConfigRegistry.getConfiguration(
                 FailureProcessorExtensionConfiguration.KEY).failureHandler();
-        failureManager = new FailureManager(node::shutdown, failureProcessorConfiguration);
+        failureManager = new FailureManager(name, node::shutdown, failureProcessorConfiguration);
 
         SystemLocalConfiguration systemConfiguration = nodeConfigRegistry.getConfiguration(SystemLocalExtensionConfiguration.KEY).system();