Add re-try logic to ssh sync and check (#274)

Author: tellet-q

tools/hetzner/check_ssh_connection.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 SERVER_NAME=${SERVER_NAME:-test-server-1}
 
 SERVER_IP=$(hcloud server ip "${SERVER_NAME}")
@@ -8,4 +10,16 @@ echo "Server IP: ${SERVER_IP}"
 
 ssh-keygen -f "$HOME/.ssh/known_hosts" -R "${SERVER_IP}" || true
 
-ssh -oStrictHostKeyChecking=no root@${SERVER_IP} echo "Server is ready"
+max_retries=10
+retry_delay=2
+
+for ((i=1; i<=max_retries; i++)); do
+    if ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "root@${SERVER_IP}" echo "Server is ready"; then
+        exit 0
+    fi
+    echo "SSH connection failed (attempt $i/$max_retries), retrying in ${retry_delay}s..."
+    sleep $retry_delay
+done
+
+echo "Failed to establish SSH connection after $max_retries attempts"
+exit 1

tools/sync_servers.sh

@@ -2,5 +2,13 @@
 
 PROJECT_PATH=$(realpath "$(dirname "$0")/..")
 
-rsync -e "ssh -o ServerAliveInterval=10 -o ServerAliveCountMax=10" -avP --mkpath\
-   "$PROJECT_PATH/engine/servers/" $1:./projects/vector-db-benchmark/engine/servers/
+max_retries=5
+retry_delay=5
+for ((i=1; i<=max_retries; i++)); do
+    if rsync -e "ssh -o ConnectTimeout=30 -o ServerAliveInterval=10 -o ServerAliveCountMax=10" \
+       -avP --mkpath "$PROJECT_PATH/engine/servers/" "$1:./projects/vector-db-benchmark/engine/servers/"; then
+        break
+    fi
+    echo "rsync failed (attempt $i/$max_retries), retrying in ${retry_delay}s..."
+    sleep $retry_delay
+done