Merge pull request #883 from github/restart-cron-on-restore-failure

on restore failure, restart cron on target host via trap call

Author: donal

bin/ghe-restore

@@ -86,6 +86,19 @@ while true; do
   esac
 done
 
+start_cron () {
+  echo "Starting cron ..."
+  if $CLUSTER; then
+    if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service cron start"; then
+      echo "* Warning: Failed to start cron on one or more nodes"
+    fi
+  else
+    if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service cron start"; then
+      echo "* Warning: Failed to start cron"
+    fi
+  fi
+}
+
 cleanup () {
   if [ -n "$1" ]; then
     update_restore_status "$1"
@@ -103,6 +116,10 @@ cleanup () {
     fi
   fi
 
+  if ! $CRON_RUNNING; then
+    start_cron
+  fi
+
   # Cleanup SSH multiplexing
   ghe-ssh --clean
 }
@@ -249,6 +266,7 @@ update_restore_status () {
   fi
 }
 
+CRON_RUNNING=true
 # Update remote restore state file and setup failure trap
 trap "cleanup failed" EXIT
 update_restore_status "restoring"
@@ -324,7 +342,7 @@ else
     fi
   fi
 fi
-
+CRON_RUNNING=false
 
 # Restore settings and license if restoring to an unconfigured appliance or when
 # specified manually.
@@ -492,16 +510,8 @@ if ! $RESTORE_SETTINGS; then
 fi
 
 # Start cron. Timerd will start automatically as part of the config run.
-echo "Starting cron ..."
-if $CLUSTER; then
-  if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo service cron start"; then
-    echo "* Warning: Failed to start cron on one or more nodes"
-  fi
-else
-  if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo service cron start"; then
-    echo "* Warning: Failed to start cron"
-  fi
-fi
+start_cron
+CRON_RUNNING=true
 
 # Clean up all stale replicas on configured instances.
 if ! $CLUSTER && $instance_configured; then