Merge pull request #1371 from myronkscott/fix_galvan2

fix thread shutdown issues

Author: myronkscott

.gitignore

@@ -10,7 +10,7 @@ dependency-reduced-pom.xml
 
 *.iml
 .idea/
-/bin/
+**/bin/
 *~
 nb-configuration.xml
 nbactions.xml

common/src/main/java/com/tc/lang/TCThreadGroup.java

@@ -46,7 +46,7 @@ public TCThreadGroup(ThrowableHandler throwableHandler, String name) {
   }
 
   public TCThreadGroup(ThrowableHandler throwableHandler, String name, boolean stoppable) {
-    this(throwableHandler, name, stoppable, stoppable);
+    this(throwableHandler, name, stoppable, !stoppable);
   }
 
   public TCThreadGroup(ThrowableHandler throwableHandler, String name, boolean stoppable, boolean ignorePool) {
@@ -81,10 +81,6 @@ public void printLiveThreads(Consumer<String> reporter) {
     }
   }
 
-  public void interruptThreads() {
-    threads().stream().filter(t -> t != Thread.currentThread() && t.isAlive()).forEach(Thread::interrupt);
-  }
-
   public boolean retire(long timeout, Consumer<InterruptedException> interruptHandler) {
     boolean complete = false;
     long killStart = System.currentTimeMillis();
@@ -97,9 +93,8 @@ public boolean retire(long timeout, Consumer<InterruptedException> interruptHand
           }
         }
       }
-      if (activeCount() == 0) {
-        destroy();
-      }
+    } else {
+      return true;
     }
     if (complete) {
       LOGGER.debug("finished thread exiting in {} seconds", TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - killStart));

common/src/main/java/com/tc/util/concurrent/LifeCycleState.java

@@ -238,7 +238,7 @@ private synchronized boolean initializeServer(OutputStream out) throws Exception
     public synchronized boolean shutdown() {
       if (running) {
         running = false;
-        String result = invokeOnServerMBean(server, "Server","stopAndWait",null);
+        String result = invokeOnServerMBean(server, "Server","halt",null);
         serverLogger.output("stopping. " + result);
         return !Boolean.parseBoolean(result);
       } else {

common/src/main/java/com/tc/util/concurrent/NullLifeCycleState.java

@@ -43,6 +43,8 @@ public interface TCServerInfoMBean extends TerracottaMBean, RuntimeStatisticCons
 
   boolean stopAndWait();
 
+  boolean halt();
+
   boolean isShutdownable();
 
   void shutdown();

galvan/src/main/java/org/terracotta/testing/master/InlineServer.java

@@ -57,6 +57,7 @@
 import javax.management.MBeanNotificationInfo;
 import javax.management.NotCompliantMBeanException;
 import org.terracotta.server.ServerEnv;
+import org.terracotta.server.StopAction;
 
 public class TCServerInfo extends AbstractTerracottaMBean implements TCServerInfoMBean, StateChangeListener {
   private static final Logger logger = LoggerFactory.getLogger(TCServerInfo.class);
@@ -156,6 +157,12 @@ public boolean stopAndWait() {
     return server.waitUntilShutdown();
   }
 
+  @Override
+  public boolean halt() {
+    server.stop(StopAction.IMMEDIATE);
+    return server.waitUntilShutdown();
+  }
+  
   @Override
   public boolean isShutdownable() {
     return server.canShutdown();

management/src/main/java/com/tc/management/beans/TCServerInfoMBean.java

@@ -869,26 +869,19 @@ public synchronized void openNetworkPorts() throws ConfigurationException {
 
   public CompletableFuture<Void> destroy(boolean immediate) throws Exception {
     CompletableFuture<Void> finished = new CompletableFuture<>();
-    if (this.threadGroup.isStoppable()) {
-      if (this.stopping.attemptSet()) {
+    if (!immediate) {
+      try {
+        stopped.get();
+      } catch (ExecutionException ee) {
+        finished.completeExceptionally(ee.getCause());
+        return finished;
+      }
+    }
+    if (this.stopping.attemptSet() && this.threadGroup.isStoppable()) {
         ThreadUtil.executeInThread(threadGroup.getParent(), ()->{
-          try {
-            if (!immediate) {
-              try {
-                stopped.get();
-              } catch (ExecutionException ee) {
-                logger.warn("stop not complete", ee.getCause());
-              }
-            }
-            killThreads(finished, immediate);
-          }catch(InterruptedException ie) {
-            logger.warn("shutdown thread failed", ie);
-            finished.completeExceptionally(ie);
-          }
+          killThreads(finished);
         }, "server shutdown thread", true);
-      }
     } else {
-      consoleLogger.info("Server Exiting...");
       finished.complete(null);
     }
     return finished;
@@ -911,17 +904,14 @@ private void shutdown() {
     }
   }
 
-  private void killThreads(CompletableFuture<Void> stopped, boolean immediate) {
+  private void killThreads(CompletableFuture<Void> stopped) {
     try {
       this.seda.getStageManager().stopAll();
-      if (immediate) {
-        threadGroup.interrupt();
-      } else if (!threadGroup.retire(TimeUnit.SECONDS.toMillis(30L), e->L2Utils.handleInterrupted(logger, e))) {
+      while (!threadGroup.retire(TimeUnit.SECONDS.toMillis(10L), e->L2Utils.handleInterrupted(logger, e))) {
         consoleLogger.warn("unable to retire server threads");
         threadGroup.printLiveThreads(logger::warn);
         threadGroup.interrupt();
       }
-      consoleLogger.info("Server Exiting...");
     } finally {
       stopped.complete(null);
     }

tc-server/src/main/java/com/tc/management/beans/TCServerInfo.java

@@ -59,6 +59,7 @@
 import javax.management.NotCompliantMBeanException;
 import com.tc.text.PrettyPrinter;
 import java.lang.management.ManagementFactory;
+import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
@@ -144,24 +145,26 @@ public void stopIfActive(StopAction...restartMode) throws PlatformStopException
   @Override
   public void stop(StopAction...restartMode) {
     audit("Stop invoked", new Properties());
-    TCLogging.getConsoleLogger().info("Stopping server");
     if (dsoServer != null) {
       try {
-        getStateManager().moveToStopStateIf(EnumSet.complementOf(EnumSet.of(ServerMode.STOP)));
-        EnumSet<StopAction> set = EnumSet.noneOf(StopAction.class);
-        for (StopAction s : restartMode) {
-          set.add(s);
-        }
-        if (set.contains(StopAction.ZAP)) {
-          TCLogging.getConsoleLogger().info("Setting data to dirty");
-          dsoServer.getPersistor().getClusterStatePersistor().setDBClean(false);
-        }
-        CompletableFuture<Void> dsoStop = dsoServer.destroy(set.contains(StopAction.IMMEDIATE));
-        if (set.contains(StopAction.RESTART)) {
-          TCLogging.getConsoleLogger().info("Requesting restart");
-          dsoStop.thenRun(()->shutdownGate.complete(true));
-        } else {
-          dsoStop.thenRun(()->shutdownGate.complete(false));
+        if (getStateManager().moveToStopStateIf(EnumSet.complementOf(EnumSet.of(ServerMode.STOP)))) {
+          consoleLogger.info("Stopping server");
+          EnumSet<StopAction> set = EnumSet.noneOf(StopAction.class);
+          set.addAll(Arrays.asList(restartMode));
+          if (set.contains(StopAction.ZAP)) {
+            TCLogging.getConsoleLogger().info("Setting data to dirty");
+            dsoServer.getPersistor().getClusterStatePersistor().setDBClean(false);
+          }
+          CompletableFuture<Void> dsoStop = dsoServer.destroy(set.contains(StopAction.IMMEDIATE));
+          dsoStop = dsoStop.thenRun(()->{
+            consoleLogger.info("Server Exiting...");
+          });
+          if (set.contains(StopAction.RESTART)) {
+            TCLogging.getConsoleLogger().info("Requesting restart");
+            dsoStop.thenRun(()->shutdownGate.complete(true));
+          } else {
+            dsoStop.thenRun(()->shutdownGate.complete(false));
+          }
         }
       } catch (Throwable e) {
         logger.error("trouble shutting down", e);
@@ -198,7 +201,6 @@ public boolean canShutdown() {
   @Override
   public synchronized void shutdown() {
     if (canShutdown()) {
-      consoleLogger.info("Server exiting...");
       stop();
     } else {
       logger.warn("Server in incorrect state (" + getStateManager().getCurrentMode().getName() + ") to be shutdown.");