Merge pull request #700 from github/check-ro-filesystem

Check for RO file-system [Cluster]

Author: chuckp22

backup.config-example

@@ -32,6 +32,10 @@ GHE_NUM_SNAPSHOTS=10
 # disk space validation and software version checks on the backup-host will be disabled.
 #GHE_SKIP_CHECKS=false
 
+# Cluster filesystem to check if it's writable as part of ghe-host-check
+# By default it is /data/user/tmp but can be updated if needed
+#GHE_FILE_SYSTEM_WRITE_CHECK="/data/user/tmp"
+
 # The hostname of the GitHub appliance to restore. If you've set up a separate
 # GitHub appliance to act as a standby for recovery, specify its IP or hostname
 # here. The host to restore to may also be specified directly when running

bin/ghe-host-check

@@ -85,7 +85,9 @@ if [ $rc -ne 0 ]; then
   exit $rc
 fi
 
-CLUSTER=false
+if [ -z "$CLUSTER" ]; then
+  CLUSTER=false
+fi
 if ghe-ssh "$host" -- \
   "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ]"; then
   CLUSTER=true
@@ -118,6 +120,37 @@ if [ -z "$version" ]; then
   exit 2
 fi
 
+NON_WRITABLE=""
+# ensure all nodes are writable
+if [ "$CLUSTER" == "true" ] ; then
+  if [ -z "$GHE_FILE_SYSTEM_WRITE_CHECK" ]; then
+    if [ -d "/data/user/tmp" ]; then
+      WRITE_CHECK_FILE="/data/user/tmp/test-ro-file.txt"
+    else
+      WRITE_CHECK_FILE="/tmp/test-ro-file.txt"
+    fi
+  else
+    WRITE_CHECK_FILE="$GHE_FILE_SYSTEM_CHECK/test-ro-file.txt"
+  fi
+
+  # Iterate through each node in the cluster
+  nodes=$(ghe-ssh "$host" ghe-cluster-nodes)
+  for node in $nodes; do
+    if ! echo "set -o pipefail; ssh $node -- 'touch $WRITE_CHECK_FILE && rm $WRITE_CHECK_FILE'" | ghe-ssh "$host" /bin/bash; then
+      echo "File system is not writeable or no permission on $node" 1>&2
+      NON_WRITABLE+="$node "
+    fi || true
+  done
+  # Display the comma-separated list of non-writable nodes
+  if [ -n "$NON_WRITABLE" ]; then
+    NON_WRITABLE=$(echo "$NON_WRITABLE" | sed 's/ /, /g; s/, $//')
+    log_error "Error: Following nodes are non-writable - $NON_WRITABLE. Please make sure the filesystem for all GHES nodes are writable." 1>&2
+    exit 1
+  else
+    log_info "All nodes are writable."
+  fi
+fi
+
 # Block restoring snapshots to older releases of GitHub Enterprise Server
 if [ -n "$GHE_RESTORE_SNAPSHOT_PATH" ]; then
   snapshot_version=$(cat $GHE_RESTORE_SNAPSHOT_PATH/version)

test/test-ghe-host-check.sh

@@ -123,3 +123,18 @@ begin_test "ghe-host-check blocks restore to old release"
   ! GHE_TEST_REMOTE_VERSION=$bu_version_major.$((bu_version_minor-1)).$bu_version_patch ghe-restore -v
 )
 end_test
+
+# Check ghe-host-check detects RO file system
+begin_test "ghe-host-check fails when encountering RO file-system"
+(
+  set -e
+
+  ghe-ssh "$GHE_HOSTNAME" -- 'mkdir -p "~/tmp"'
+  # Remove write access in ~/tmp
+  ghe-ssh "$GHE_HOSTNAME" -- 'chmod a-w -R "~/tmp"'
+
+  # File creation fails for CLUSTER
+  ! WRITE_CHECK_FILE="$HOME/tmp/test" CLUSTER=true GHE_ALLOW_REPLICA_BACKUP=no ghe-host-check
+  WRITE_CHECK_FILE="$HOME/tmp/test" CLUSTER=false GHE_ALLOW_REPLICA_BACKUP=no ghe-host-check
+)
+end_test