Fix Jepsen CI: add missing provision script and improve wait logic

Author: bootjp

jepsen/.gitignore

@@ -0,0 +1,7 @@
+/target
+/tmp-home
+/.lein
+/.vagrant
+/.vagrant.d
+/store
+/results.tgz

jepsen/provision/base.sh

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROLE="${1:-db}"
+
+echo "[jepsen] provisioning role=${ROLE}"
+sudo apt-get update -y
+sudo apt-get install -y --no-install-recommends \
+  curl netcat-openbsd rsync iptables chrony libfaketime \
+  openjdk-17-jre-headless leiningen git build-essential jq
+
+sudo systemctl enable --now chrony
+
+# Append Jepsen nodes to /etc/hosts if not already present
+if ! grep -q "192.168.56.10 ctrl" /etc/hosts; then
+cat <<'EOF' | sudo tee -a /etc/hosts >/dev/null
+
+# Jepsen Cluster
+192.168.56.10 ctrl
+192.168.56.11 n1
+192.168.56.12 n2
+192.168.56.13 n3
+192.168.56.14 n4
+192.168.56.15 n5
+EOF
+fi
+
+install -d -m700 /home/vagrant/.ssh
+if [ "$ROLE" = "ctrl" ]; then
+  GO_VERSION=1.25.5
+  ARCH=$(dpkg --print-architecture)
+  if [ "$ARCH" = "arm64" ]; then
+    GO_ARCH="arm64"
+  else
+    GO_ARCH="amd64"
+  fi
+
+  if ! command -v go >/dev/null 2>&1 || [[ "$(go version | awk '{print $3}')" != "go${GO_VERSION}" ]]; then
+    echo "[jepsen] installing go ${GO_VERSION} for ${GO_ARCH}"
+    curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz" -o /tmp/go.tar.gz
+    sudo rm -rf /usr/local/go
+    sudo tar -C /usr/local -xzf /tmp/go.tar.gz
+  fi
+  echo 'export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin' | sudo tee /etc/profile.d/go.sh >/dev/null
+
+  if [ ! -f /home/vagrant/.ssh/id_rsa ]; then
+    cat <<'KEY' > /home/vagrant/.ssh/id_rsa
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI
+w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP
+kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2
+hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO
+Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW
+yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd
+ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1
+Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf
+TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK
+iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A
+sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf
+4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP
+cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk
+EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN
+CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX
+3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG
+YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj
+3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+
+dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz
+6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC
+P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF
+llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ
+kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH
++vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ
+NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s=
+-----END RSA PRIVATE KEY-----
+KEY
+    chmod 600 /home/vagrant/.ssh/id_rsa
+    chown vagrant:vagrant /home/vagrant/.ssh/id_rsa
+  fi
+  cat <<'EOF' > /home/vagrant/.ssh/config
+Host n1 n2 n3 n4 n5
+  User vagrant
+  IdentityFile /home/vagrant/.ssh/id_rsa
+  StrictHostKeyChecking no
+  UserKnownHostsFile /dev/null
+  LogLevel QUIET
+EOF
+  chown vagrant:vagrant /home/vagrant/.ssh/config
+fi
+
+# authorize the same key on all nodes
+cat <<'PUB' >> /home/vagrant/.ssh/authorized_keys
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDo0XyJqWW9BWnbZYOROSyu2+n15ZbrgPGFa/pM+E4xmHu4B8yMPp4jbWRhR8w/Pr9SNmCeqF3r3LdWHktKPR2cjduPaoAoM1BbXTii7+iHnaZaqD5HJhXQhr3Y+QQOjcYVMFyQU8hMAzMF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key
+PUB
+chown vagrant:vagrant /home/vagrant/.ssh/authorized_keys
+chmod 600 /home/vagrant/.ssh/authorized_keys
+
+sudo mkdir -p /opt/elastickv/bin /var/lib/elastickv

jepsen/src/elastickv/db.clj

@@ -113,8 +113,8 @@
   [node grpc-port]
   (c/on node
     (c/exec :bash "-c"
-            (format "for i in $(seq 1 60); do nc -z %s %s && exit 0; sleep 1; done; exit 1"
-                    (name node) grpc-port))))
+            (format "for i in $(seq 1 60); do if nc -z -w 1 %s %s; then exit 0; fi; sleep 1; done; echo 'Timed out waiting for %s:%s'; exit 1"
+                    (name node) grpc-port (name node) grpc-port))))
 
 (defn- join-node!
   "Join peer into cluster via raftadmin, executed on bootstrap node."