Improve demo cluster start and stop handling

Author: bootjp

.github/workflows/jepsen-test.yml

@@ -33,20 +33,34 @@ jobs:
         working-directory: jepsen
         run: ~/lein test
       - name: Launch demo cluster
+        env:
+          GOTMPDIR: /tmp/go-tmp
         run: |
-          mkdir -p "$GOCACHE"
+          set -euo pipefail
+          mkdir -p "$GOCACHE" "$GOTMPDIR"
           nohup go run cmd/server/demo.go > /tmp/elastickv-demo.log 2>&1 &
           echo $! > /tmp/elastickv-demo.pid
-          for i in {1..30}; do
-            nc -z 127.0.0.1 63791 && nc -z 127.0.0.1 63792 && nc -z 127.0.0.1 63793 && break
+
+          echo "Waiting for redis listeners on 63791-63793..."
+          for i in {1..45}; do
+            if nc -z 127.0.0.1 63791 && nc -z 127.0.0.1 63792 && nc -z 127.0.0.1 63793; then
+              echo "Cluster is up"
+              exit 0
+            fi
             sleep 1
           done
+
+          echo "Demo cluster failed to start; dumping log:"
+          tail -n 200 /tmp/elastickv-demo.log || true
+          exit 1
       - name: Run Redis Jepsen workload against elastickv
         working-directory: jepsen
         run: |
           ~/lein run -m elastickv.redis-workload --time-limit 5 --rate 5 --concurrency 5 --ports 63791,63792,63793 --host 127.0.0.1
       - name: Stop demo cluster
         run: |
           if [ -f /tmp/elastickv-demo.pid ]; then
-            kill "$(cat /tmp/elastickv-demo.pid)" || true
+            pid=$(cat /tmp/elastickv-demo.pid)
+            kill "$pid" 2>/dev/null || true
+            wait "$pid" 2>/dev/null || true
           fi