Merge pull request #166 from github/snh/stop-multiple-restores

Prevent multiple restores from running at the same time

Author: snh

bin/ghe-restore

@@ -146,10 +146,6 @@ fi
 echo "Starting restore of $GHE_HOSTNAME from snapshot $GHE_RESTORE_SNAPSHOT"
 ghe_remote_logger "Starting restore from $(hostname) / snapshot $GHE_RESTORE_SNAPSHOT ..."
 
-# Update remote restore state file and setup failure trap
-trap "update_restore_status failed" EXIT
-update_restore_status "restoring"
-
 # Verify the host has been fully configured at least once if when running
 # against v11.10.x appliances and the -c option wasn't specified.
 if [ "$GHE_VERSION_MAJOR" -le 1 ] && ! $restore_settings && ! $instance_configured; then
@@ -176,6 +172,16 @@ if $instance_configured; then
     fi
 fi
 
+# Make sure the appliance doesn't already have a restore underway
+if [ "$GHE_VERSION_MAJOR" -ge 2 ] && ghe-ssh "$GHE_HOSTNAME" -- "sudo grep -q restoring $GHE_REMOTE_DATA_USER_DIR/common/ghe-restore-status 2>/dev/null"; then
+    echo "Error: $GHE_HOSTNAME already has a restore underway. Aborting." 1>&2
+    exit 1
+fi
+
+# Update remote restore state file and setup failure trap
+trap "update_restore_status failed" EXIT
+update_restore_status "restoring"
+
 # Restore settings and license if restoring to an unconfigured appliance or when
 # specified manually.
 if $restore_settings; then

test/test-ghe-restore.sh

@@ -410,3 +410,37 @@ begin_test "ghe-restore with tarball strategy"
     echo "$output" | grep -q 'fake ghe-export-repositories data'
 )
 end_test
+
+begin_test "ghe-restore aborts when another restore is underway"
+(
+    set -e
+    # This test is only valid for version 2 and above
+    if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
+      rm -rf "$GHE_REMOTE_ROOT_DIR"
+      setup_remote_metadata
+
+      # create file used to determine if instance has been configured.
+      touch "$GHE_REMOTE_ROOT_DIR/etc/github/configured"
+
+      # create file used to determine if instance is in maintenance mode.
+      mkdir -p "$GHE_REMOTE_DATA_DIR/github/current/public/system"
+      touch "$GHE_REMOTE_DATA_DIR/github/current/public/system/maintenance.html"
+
+      # create file to indicate restore is underway
+      echo "restoring" > "$GHE_REMOTE_DATA_USER_DIR/common/ghe-restore-status"
+
+      # set restore host environ var
+      GHE_RESTORE_HOST=127.0.0.1
+      export GHE_RESTORE_HOST
+
+      # run ghe-restore and write output to file for asserting against
+      # this should fail due to the appliance being in an unconfigured state
+      ! ghe-restore -v > "$TRASHDIR/restore-out" 2>&1
+
+      cat $TRASHDIR/restore-out
+
+      # verify that ghe-restore failed due a restore already being underway
+      grep -q -e "already has a restore underway" "$TRASHDIR/restore-out"
+    fi
+)
+end_test