Merge pull request #670 from github/hao/cleanup-nomad-container

Cleanup nomad container when restore for 2.22

Author: ashishkeshan

bin/ghe-host-check

@@ -84,6 +84,23 @@ if [ $rc -ne 0 ]; then
   exit $rc
 fi
 
+CLUSTER=false
+if ghe-ssh "$host" -- \
+  "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ]"; then
+  CLUSTER=true
+fi
+
+# ensure all nodes in the cluster are running the same version
+if "$CLUSTER"; then
+  node_version_list=$(ghe-ssh "$host" ghe-cluster-each -- ghe-version)
+  distinct_versions=$(echo "$node_version_list" | awk '{split($0, a, ":"); print a[2]}' | awk '{print $4}' | uniq | wc -l)
+  if [ "$distinct_versions" -ne 1 ]; then
+    echo "$node_version_list" 1>&2
+    echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&2
+    exit 1
+  fi
+fi
+
 version=$(echo "$output" | grep "GitHub Enterprise" | awk '{print $NF}')
 
 if [ -z "$version" ]; then

bin/ghe-restore

@@ -410,12 +410,16 @@ if $CLUSTER; then
   echo "Configuring cluster ..."
   if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
     ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-nomad-cleanup" 1>&3 2>&3
+  elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then
+    ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- /usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3
   fi
   ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-config-apply" 1>&3 2>&3
 elif $instance_configured; then
   echo "Configuring appliance ..."
   if [ "$GHE_VERSION_MAJOR" -eq "3" ]; then
     ghe-ssh "$GHE_HOSTNAME" -- "ghe-nomad-cleanup" 1>&3 2>&3
+  elif [ "$GHE_VERSION_MAJOR" -eq "2" ] && [ "$GHE_VERSION_MINOR" -eq "22" ]; then
+    ghe-ssh "$GHE_HOSTNAME" -- "/usr/local/share/enterprise/ghe-nomad-cleanup" 1>&3 2>&3
   fi
   ghe-ssh "$GHE_HOSTNAME" -- "ghe-config-apply" 1>&3 2>&3
 fi

share/github-backup-utils/ghe-restore-settings

@@ -28,7 +28,7 @@ GHE_RESTORE_SNAPSHOT_PATH="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"
 echo "Restoring license ..."
 ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-license' < "$GHE_RESTORE_SNAPSHOT_PATH/enterprise.ghl" 1>&3
 
-echo "Restoring settings ..."
+echo "Restoring settings and applying configuration ..."
 
 # Restore external MySQL password if running external MySQL DB.
 restore-secret "external MySQL password" "external-mysql-password" "secrets.external.mysql"

test/bin/ghe-cluster-each

@@ -16,9 +16,34 @@ for _ in "$@"; do
         shift
         shift
         ;;
+    --)
+      if [ "$1" = "--" ]; then
+        shift
+        COMMAND="$*"
+      fi
+      break
+      ;;
   esac
 done
 
+if [ "$COMMAND" == "ghe-version" ]; then
+  if [ -z "$DIFFERENT_VERSIONS" ]; then
+    echo "fake-uuid: GitHub Enterprise Server 3.1.0 lxc 2020-12-16 5e97c07602"
+    echo "fake-uuid1: GitHub Enterprise Server 3.1.0 lxc 2020-12-16 5e97c07602"
+    echo "fake-uuid2: GitHub Enterprise Server 3.1.0 lxc 2020-12-16 5e97c07602"
+  else
+    echo "fake-uuid: GitHub Enterprise Server 3.1.0 lxc 2020-12-16 5e97c07602"
+    echo "fake-uuid1: GitHub Enterprise Server 3.1.0 lxc 2020-12-16 5e97c07602"
+    echo "fake-uuid2: GitHub Enterprise Server 2.19 lxc 2020-12-13 5e97c07622"
+  fi
+  exit 0
+fi
+
+if [ "$COMMAND" == "/usr/local/share/enterprise/ghe-nomad-cleanup" ]; then
+  echo "nomad cleanup"
+  exit 0
+fi
+
 if $SHOW_UUID; then
   CONFIG="$GHE_REMOTE_DATA_USER_DIR/common/cluster.conf"
 
@@ -36,4 +61,4 @@ if $SHOW_UUID; then
       echo $hostname
     done
   fi
-fi
+fi

test/test-ghe-restore.sh

@@ -535,7 +535,7 @@ setup_test_data "$GHE_DATA_DIR/1"
 # Make the current symlink
 ln -s 1 "$GHE_DATA_DIR/current"
 
-begin_test "ghe-restore cluster"
+begin_test "ghe-restore cluster with matching node versions"
 (
   set -e
   rm -rf "$GHE_REMOTE_ROOT_DIR"
@@ -598,6 +598,32 @@ begin_test "ghe-restore cluster"
 )
 end_test
 
+begin_test "ghe-restore cluster with different node versions should fail at ghe-host-check"
+(
+  set -e
+  rm -rf "$GHE_REMOTE_ROOT_DIR"
+  setup_moreutils_parallel
+  setup_remote_metadata
+  setup_remote_cluster
+  echo "cluster" > "$GHE_DATA_DIR/current/strategy"
+
+  # set that versions should not match for this test
+  DIFFERENT_VERSIONS=1
+  export DIFFERENT_VERSIONS
+
+  # set as configured, enable maintenance mode and create required directories
+  setup_maintenance_mode "configured"
+
+  # set restore host environ var
+  GHE_RESTORE_HOST=127.0.0.1
+  export GHE_RESTORE_HOST
+
+  ! output=$(ghe-restore -v -f 2>&1)
+
+  echo "$output" | grep -q "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils."
+)
+end_test
+
 begin_test "ghe-restore missing directories or files from source snapshot displays warning"
 (
     # Tests the scenario where something exists in the database, but not on disk.