Merge pull request #668 from github/add-node-version-check

Add node version check and fail out if they don't match

Author: ashishkeshan

bin/ghe-host-check

@@ -15,32 +15,48 @@ set -e
 
 while true; do
   case "$1" in
-    -h|--help)
-      export GHE_SHOW_HELP=true
-      shift
-      ;;
-    --version)
-      export GHE_SHOW_VERSION=true
-      shift
-      ;;
-    -*)
-      echo "Error: invalid argument: '$1'" 1>&2
-      exit 1
-      ;;
-    *)
-      break
-      ;;
+  -h | --help)
+    export GHE_SHOW_HELP=true
+    shift
+    ;;
+  --version)
+    export GHE_SHOW_VERSION=true
+    shift
+    ;;
+  -*)
+    echo "Error: invalid argument: '$1'" 1>&2
+    exit 1
+    ;;
+  *)
+    break
+    ;;
   esac
 done
 
 # Bring in the backup configuration
 # shellcheck source=share/github-backup-utils/ghe-backup-config
-. "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config"
+. "$(dirname "${BASH_SOURCE[0]}")/../share/github-backup-utils/ghe-backup-config"
 
 # Use the host provided on the command line if provided, or fallback on the
 # $GHE_HOSTNAME configured in backup.config when not present.
 host="${1:-$GHE_HOSTNAME}"
 
+CLUSTER=false
+if ghe-ssh "$host" -- \
+  "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ]"; then
+  CLUSTER=true
+fi
+
+if "$CLUSTER"; then
+  node_version_list=$(ghe-ssh "$host" ghe-cluster-each -- ghe-version)
+  echo "$node_version_list" 1>&2
+  distinct_versions=$(echo "$node_version_list" | awk '{split($0, a, ":"); print a[2]}' | uniq | wc -l)
+  if [ "$distinct_versions" -ne 1 ]; then
+    echo "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils." 1>&2
+    exit 1
+  fi
+fi
+
 # Options to pass to SSH during connection check
 options="
   -o PasswordAuthentication=no
@@ -59,26 +75,26 @@ set -e
 
 if [ $rc -ne 0 ]; then
   case $rc in
-    255)
-      if echo "$output" | grep -i "port 22: Network is unreachable\|port 22: connection refused\|port 22: no route to host\|ssh_exchange_identification: Connection closed by remote host\|Connection timed out during banner exchange\|port 22: Connection timed out" >/dev/null; then
-        exec "$(basename $0)" "$hostname:122"
-      fi
-
+  255)
+    if echo "$output" | grep -i "port 22: Network is unreachable\|port 22: connection refused\|port 22: no route to host\|ssh_exchange_identification: Connection closed by remote host\|Connection timed out during banner exchange\|port 22: Connection timed out" >/dev/null; then
+      exec "$(basename $0)" "$hostname:122"
+    fi
+
+    echo "$output" 1>&2
+    echo "Error: ssh connection with '$host' failed" 1>&2
+    echo "Note that your SSH key needs to be setup on $host as described in:" 1>&2
+    echo "* https://enterprise.github.com/help/articles/adding-an-ssh-key-for-shell-access" 1>&2
+    ;;
+  101)
+    echo "Error: couldn't read GitHub Enterprise Server fingerprint on '$host' or this isn't a GitHub appliance." 1>&2
+    ;;
+  1)
+    if [ "${port:-22}" -eq 22 ] && echo "$output" | grep "use port 122" >/dev/null; then
+      exec "$(basename $0)" "$hostname:122"
+    else
       echo "$output" 1>&2
-      echo "Error: ssh connection with '$host' failed" 1>&2
-      echo "Note that your SSH key needs to be setup on $host as described in:" 1>&2
-      echo "* https://enterprise.github.com/help/articles/adding-an-ssh-key-for-shell-access" 1>&2
-      ;;
-    101)
-      echo "Error: couldn't read GitHub Enterprise Server fingerprint on '$host' or this isn't a GitHub appliance." 1>&2
-      ;;
-    1)
-      if [ "${port:-22}" -eq 22 ] && echo "$output" | grep "use port 122" >/dev/null; then
-        exec "$(basename $0)" "$hostname:122"
-      else
-        echo "$output" 1>&2
-      fi
-      ;;
+    fi
+    ;;
 
   esac
   exit $rc

test/bin/ghe-cluster-each

@@ -19,6 +19,19 @@ for _ in "$@"; do
   esac
 done
 
+if [ "$2" == "ghe-version" ]; then
+  if [ -z "$DIFFERENT_VERSIONS" ]; then
+    echo "fake-uuid: GitHub Enterprise Server 3.1.0 lxc 2020-12-16 5e97c07602"
+    echo "fake-uuid1: GitHub Enterprise Server 3.1.0 lxc 2020-12-16 5e97c07602"
+    echo "fake-uuid2: GitHub Enterprise Server 3.1.0 lxc 2020-12-16 5e97c07602"
+  else
+    echo "fake-uuid: GitHub Enterprise Server 3.1.0 lxc 2020-12-16 5e97c07602"
+    echo "fake-uuid1: GitHub Enterprise Server 3.1.0 lxc 2020-12-16 5e97c07602"
+    echo "fake-uuid2: GitHub Enterprise Server 2.19 lxc 2020-12-13 5e97c07622"
+  fi
+  exit 0
+fi
+
 if $SHOW_UUID; then
   CONFIG="$GHE_REMOTE_DATA_USER_DIR/common/cluster.conf"
 

test/test-ghe-restore.sh

@@ -535,7 +535,7 @@ setup_test_data "$GHE_DATA_DIR/1"
 # Make the current symlink
 ln -s 1 "$GHE_DATA_DIR/current"
 
-begin_test "ghe-restore cluster"
+begin_test "ghe-restore cluster with matching node versions"
 (
   set -e
   rm -rf "$GHE_REMOTE_ROOT_DIR"
@@ -598,6 +598,32 @@ begin_test "ghe-restore cluster"
 )
 end_test
 
+begin_test "ghe-restore cluster with different node versions should fail at ghe-host-check"
+(
+  set -e
+  rm -rf "$GHE_REMOTE_ROOT_DIR"
+  setup_moreutils_parallel
+  setup_remote_metadata
+  setup_remote_cluster
+  echo "cluster" > "$GHE_DATA_DIR/current/strategy"
+
+  # set that versions should not match for this test
+  DIFFERENT_VERSIONS=1
+  export DIFFERENT_VERSIONS
+
+  # set as configured, enable maintenance mode and create required directories
+  setup_maintenance_mode "configured"
+
+  # set restore host environ var
+  GHE_RESTORE_HOST=127.0.0.1
+  export GHE_RESTORE_HOST
+
+  ! output=$(ghe-restore -v -f 2>&1)
+
+  echo "$output" | grep -q "Error: Not all nodes are running the same version! Please ensure all nodes are running the same version before using backup-utils."
+)
+end_test
+
 begin_test "ghe-restore missing directories or files from source snapshot displays warning"
 (
     # Tests the scenario where something exists in the database, but not on disk.