When running test-cljs, after 5 minutes print stacktraces and exit

A successful ClojureScript build takes 2~3 minutes, however sometimes it will
block until CI times out, and we're not sure yet why.

To further diagnose this issue we now start a background shell which after five
minutes will print the stacktraces of all threads of all active JVMs using
jcmd/jstack. Hopefully this provides some insight in what/where the tests get
stuck.

Additionally this kills all Java processes after five minutes so CI finishes a
little quicker.

Author: plexus

Makefile

@@ -1,4 +1,4 @@
-.PHONY: test-clj test-cljs eastwood cljfmt cloverage install smoketest release deploy clean check_node
+.PHONY: test-clj test-cljs eastwood cljfmt cloverage install smoketest release deploy clean check_node detect_timeout
 
 CLOJURE_VERSION ?= 1.9
 export CLOVERAGE_VERSION = 1.0.13
@@ -16,7 +16,7 @@ source-deps: .source-deps
 test-clj: .source-deps smoketest
 	lein with-profile +$(CLOJURE_VERSION),+plugin.mranderson/config,+test-clj test
 
-test-cljs: .source-deps check_node
+test-cljs: .source-deps check_node detect_timeout
 	lein with-profile +$(CLOJURE_VERSION),+plugin.mranderson/config,+test-cljs test
 
 eastwood:
@@ -50,6 +50,12 @@ smoketest: install
 check_node:
 	which node
 
+# Run a background process that prints all JVM stacktraces after five minutes,
+# then kills all JVMs, to help diagnose issues with ClojureScript tests getting
+# stuck.
+detect_timeout:
+	(bin/ci_detect_timeout &)
+
 # When releasing, the BUMP variable controls which field in the
 # version string will be incremented in the *next* snapshot
 # version. Typically this is either "major", "minor", or "patch".

bin/ci_detect_timeout

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# A successful ClojureScript build takes 2~3 minutes, however sometimes it will
+# block until CI times out, and we're not sure yet why.
+#
+# To further diagnose this issue we now start a background shell which after five
+# minutes will print the stacktraces of all threads of all active JVMs using
+# jcmd/jstack. Hopefully this provides some insight in what/where the tests get
+# stuck.
+#
+# Additionally this kills all Java processes after five minutes so CI finishes a
+# little quicker.
+
+sleep 300 # 5 minutes
+
+function hr() {
+    echo '━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━'
+}
+
+echo
+hr
+echo "           TIMEOUT, PRINTING STACK TRACES."
+hr
+
+jcmd | grep -v jcmd | while read pid cmd
+do
+    echo "---> " $cmd
+    echo
+    jstack $pid 2>&1
+    hr
+    echo
+done
+
+echo "KILLING JAVA PROCESSES"
+echo
+
+jcmd | grep -v jcmd | while read pid cmd
+do
+    echo "kill -9 ${pid} # ${cmd}"
+    kill -9 $pid
+done