Inline servers can respin on zap and restart while trying to be shutdown. Check if the the thread is running before respin.

Author: myronkscott

galvan-support/src/main/java/org/terracotta/testing/rules/BasicExternalCluster.java

@@ -155,30 +155,10 @@ protected void before() throws Throwable {
   }
 
   @Override
-  public TestManager getTestManager() {
-    return new TestManager() {
-      @Override
-      public void testFinished() {
-        stateManager.setTestDidPassIfNotFailed();
-      }
-
-      @Override
-      public void testDidFail(GalvanFailureException failure) {
-        stateManager.testDidFail(failure);
-      }
-
-      @Override
-      public boolean isComplete() throws GalvanFailureException {
-        return stateManager.checkDidPass();
-      }
-    };
+  public ClusterInfo getClusterInfo() {
+      return this.cluster.getClusterInfo();
   }
 
-    @Override
-    public ClusterInfo getClusterInfo() {
-        return this.cluster.getClusterInfo();
-    }
-
   private void internalStart(CompletableFuture<Void> checker) throws Throwable {
     VerboseLogger harnessLogger = new VerboseLogger(System.out, null);
     VerboseLogger fileHelpersLogger = new VerboseLogger(null, null);
@@ -269,34 +249,30 @@ public void run() {
           stateManager.waitForFinish();
           didPass = true;
         } catch (GalvanFailureException e) {
-          e.printStackTrace();
           checker.completeExceptionally(e);
           didPass = false;
         } finally {
-          // Whether we passed or failed, bring everything down.
+          setSafeForRun(false);
           try {
-            interlock.forceShutdown();
-          } catch (Exception e) {
-            e.printStackTrace();
-            didPass = false;
-          } finally {
-            setSafeForRun(false);
-
-            serverPortRefs.forEach(PortManager.PortRef::close);
-            groupPortRefs.forEach(PortManager.PortRef::close);
-            debugPortRefs.stream().filter(Objects::nonNull).forEach(PortManager.PortRef::close);
-
-            if (!didPass) {
-              // Typically, we want to interrupt the thread running as the "client" as it might be stuck in a connection
-              // attempt, etc.  When Galvan is run in the purely multi-process mode, this is typically where all
-              // sub-processes would be terminated.  Since we are running the client as another thread, in-process, the
-              // best we can do is interrupt it from a lower-level blocking call.
-              // NOTE:  the "client" is also the thread which created us and will join on our termination, before
-              // returning back to the user code so it is possible that this interruption could be experienced in its
-              // join() call (in which case, we can safely ignore it).
-              isInterruptingClient = true;
-              clientThread.interrupt();
-            }
+            // make sure all servers are down before releasing ports
+            interlock.waitForAllServerTerminated();
+          } catch (GalvanFailureException ge) {
+            
+          }
+          serverPortRefs.forEach(PortManager.PortRef::close);
+          groupPortRefs.forEach(PortManager.PortRef::close);
+          debugPortRefs.stream().filter(Objects::nonNull).forEach(PortManager.PortRef::close);
+
+          if (!didPass) {
+            // Typically, we want to interrupt the thread running as the "client" as it might be stuck in a connection
+            // attempt, etc.  When Galvan is run in the purely multi-process mode, this is typically where all
+            // sub-processes would be terminated.  Since we are running the client as another thread, in-process, the
+            // best we can do is interrupt it from a lower-level blocking call.
+            // NOTE:  the "client" is also the thread which created us and will join on our termination, before
+            // returning back to the user code so it is possible that this interruption could be experienced in its
+            // join() call (in which case, we can safely ignore it).
+            isInterruptingClient = true;
+            clientThread.interrupt();
           }
         }
       }
@@ -319,7 +295,6 @@ private void internalStop() {
     try {
       interlock.ignoreServerCrashes(true);
       interlock.forceShutdown();
-      interlock.waitForAllServerTerminated();
     } catch (GalvanFailureException gf) {
       stateManager.testDidFail(gf);
     }

galvan-support/src/main/java/org/terracotta/testing/rules/Cluster.java

@@ -38,8 +38,6 @@ public abstract class Cluster extends ExternalResource {
 
   public abstract ClusterControl getClusterControl();
 
-  public abstract TestManager getTestManager();
-
   public abstract void expectCrashes(boolean yes);
 
   public abstract CompletionStage<Void> manualStart(String display);

galvan/src/main/java/org/terracotta/testing/master/InlineServer.java

@@ -163,7 +163,10 @@ public void stop() throws InterruptedException {
         serverLogger.output("Crashing server process: " + serverThread);
         // Mark this as expected.
         this.setCrashExpected(true);
-        boolean result = serverThread.shutdown();
+        boolean result;
+        do {
+          result = serverThread.shutdown();
+        } while (!result);
         serverLogger.output("Server Stop Command Result: " + result);
       }
     } finally {
@@ -224,7 +227,7 @@ public void run() {
           try (OutputStream events = buildEventingStream(stdout)) {
             if (initializeServer(events)) {
               serverLogger.output("starting server");
-              returnValue = (Boolean)invokeOnObject(server, "waitUntilShutdown");
+              returnValue = (Boolean)invokeOnObject(server, "waitUntilShutdown") && isRunning();
               didTerminateWithStatus(returnValue);
               serverLogger.output("server process exit. restarting:" + returnValue);
             } else {
@@ -246,6 +249,10 @@ private Object getServerObject() {
       return server;
     }
 
+    private synchronized boolean isRunning() {
+      return running;
+    }
+
     private synchronized boolean initializeServer(OutputStream out) throws Exception {
       server = startIsolatedServer(serverWorkingDir, serverInstall, out, cmd, serverProperties);
       running = server != null;

galvan/src/main/java/org/terracotta/testing/master/ServerInstance.java

@@ -31,7 +31,6 @@
 import org.terracotta.testing.common.Assert;
 import org.terracotta.testing.common.SimpleEventingStream;
 import org.terracotta.testing.logging.ContextualLogger;
-import org.terracotta.testing.logging.VerboseManager;
 import static org.terracotta.testing.master.ServerMode.ACTIVE;
 
 
@@ -102,6 +101,7 @@ protected synchronized boolean isCrashExpected() {
     return this.isCrashExpected;
   }
 
+  @Override
   public synchronized void setCrashExpected(boolean expect) {
     this.isCrashExpected = expect;
   }

galvan/src/main/java/org/terracotta/testing/master/StateInterlock.java

@@ -236,7 +236,6 @@ private void safeStop(IGalvanServer server) {
       this.logger.output("< safeStop " + server.toString());
       this.logger.output("Stopping " + server);
       server.stop();
-      server.waitForState(ServerMode.TERMINATED);
       this.logger.output("> safeStop " + server.toString() + " " + (System.currentTimeMillis() - start) + "ms");
     } catch (InterruptedException e) {
       // Not expected in this usage - we are shutting down.