Merge pull request #156 from github/cluster

GitHub Enterprise Cluster support + SAML fixes

Author: rubiojr

bin/ghe-backup

@@ -86,6 +86,13 @@ echo "Starting backup of $GHE_HOSTNAME in snapshot $GHE_SNAPSHOT_TIMESTAMP"
 ghe_remote_version_required
 echo "$GHE_REMOTE_VERSION" > version
 
+# Figure out if we're restoring into cluster
+cluster=false
+if ghe-ssh "$GHE_HOSTNAME" -- \
+  "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ]"; then
+  cluster=true
+fi
+
 # Log backup start message in /var/log/syslog on remote instance
 ghe_remote_logger "Starting backup from $(hostname) in snapshot $GHE_SNAPSHOT_TIMESTAMP ..."
 
@@ -99,8 +106,13 @@ if [ $GHE_VERSION_MAJOR -eq 1 -a $GHE_VERSION_PATCH -lt 340 ]; then
     GHE_BACKUP_STRATEGY="tarball"
 fi
 
+if $cluster; then
+  GHE_BACKUP_STRATEGY="cluster"
+fi
+
 # Record the strategy with the snapshot so we will know how to restore.
 echo "$GHE_BACKUP_STRATEGY" > strategy
+export GHE_BACKUP_STRATEGY
 
 # If we're using the tarball backup strategy, put the appliance in maintenance
 # mode and wait for all writing processes to bleed out.
@@ -127,8 +139,21 @@ ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > mysql.sql.gz ||
 failures="$failures mysql"
 
 echo "Backing up Redis database ..."
-ghe-backup-redis > redis.rdb ||
-failures="$failures redis"
+if $cluster; then
+  ghe-backup-redis-cluster > redis.rdb ||
+    failures="$failures redis"
+else
+  ghe-backup-redis > redis.rdb ||
+    failures="$failures redis"
+fi
+
+echo "Backing up audit log ..."
+ghe-backup-es-audit-log ||
+failures="$failures audit-log"
+
+echo "Backing up hookshot logs ..."
+ghe-backup-es-hookshot ||
+failures="$failures hookshot"
 
 echo "Backing up Git repositories ..."
 ghe-backup-repositories-${GHE_BACKUP_STRATEGY} ||
@@ -139,18 +164,30 @@ ghe-backup-pages-${GHE_BACKUP_STRATEGY} ||
 failures="$failures pages"
 
 if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
-    echo "Backing up asset attachments ..."
-    ghe-backup-userdata alambic_assets ||
-    failures="$failures alambic_assets"
-
-    echo "Backing up hook deliveries ..."
-    ghe-backup-userdata hookshot ||
-    failures="$failures hookshot"
+    if $cluster; then
+        echo "Backing up asset attachments ..."
+        ghe-backup-alambic-cluster ||
+        failures="$failures alambic_assets"
+    else
+        echo "Backing up asset attachments ..."
+        ghe-backup-userdata alambic_assets ||
+        failures="$failures alambic_assets"
+
+        echo "Backing up storage data ..."
+        ghe-backup-userdata storage ||
+        failures="$failures storage"
+
+        echo "Backing up hook deliveries ..."
+        ghe-backup-userdata hookshot ||
+        failures="$failures hookshot"
+    fi
 fi
 
-echo "Backing up Elasticsearch indices ..."
-ghe-backup-es-${GHE_BACKUP_STRATEGY} ||
-failures="$failures elasticsearch"
+if ! $cluster; then
+  echo "Backing up Elasticsearch indices ..."
+  ghe-backup-es-${GHE_BACKUP_STRATEGY} ||
+  failures="$failures elasticsearch"
+fi
 
 # If we're using the tarball backup strategy, bring the appliance out of
 # maintenance mode now instead of waiting until after pruning stale snapshots.

bin/ghe-host-check

@@ -37,7 +37,7 @@ set -e
 if [ $rc -ne 0 ]; then
     case $rc in
         255)
-            if echo "$output" | grep -i "port 22: connection refused" >/dev/null; then
+            if echo "$output" | grep -i "port 22: connection refused\|Connection timed out during banner exchange" >/dev/null; then
                 exec "bin/$(basename $0)" "$hostname:122"
             fi
 

bin/ghe-restore

@@ -97,6 +97,15 @@ elif [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
     restore_settings=true
 fi
 
+# Figure out if we're restoring into cluster
+cluster=false
+if ghe-ssh "$GHE_HOSTNAME" -- \
+  "[ -f '$GHE_REMOTE_ROOT_DIR/etc/github/cluster' ]"; then
+  cluster=true
+  instance_configured=true
+  restore_settings=false
+fi
+
 # Figure out if this instance is in a replication pair
 if ghe-ssh "$GHE_HOSTNAME" -- "ghe-repl-status -r 2>/dev/null" \
   | grep -Eq "replica|primary"; then
@@ -176,40 +185,66 @@ fi
 # Make sure mysql and elasticsearch are prep'd and running before restoring into
 # appliances v2.x or greater. These services will not have been started on appliances
 # that have not been configured yet.
-if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
+if ! $cluster; then
+  if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
     echo "sudo ghe-service-ensure-mysql && sudo ghe-service-ensure-elasticsearch" |
     ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3
+  fi
 fi
 
-# Remove temporary 2.2 storage migration directory if it exists
-echo "if [ -d /data/user/repositories-nw-backup ]; then sudo rm -rf /data/user/repositories-nw-backup; fi" |
-ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3
+echo "Restoring MySQL database ..."
+gzip -dc "$GHE_RESTORE_SNAPSHOT_PATH/mysql.sql.gz" | ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-mysql'
 
-echo "Restoring Git repositories ..."
-ghe-restore-repositories-${GHE_BACKUP_STRATEGY} "$GHE_HOSTNAME" 1>&3
+echo "Restoring Redis database ..."
+ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-redis' < "$GHE_RESTORE_SNAPSHOT_PATH/redis.rdb" 1>&3
 
-echo "Restoring GitHub Pages ..."
-ghe-restore-pages-${GHE_BACKUP_STRATEGY} "$GHE_HOSTNAME" 1>&3
+if $cluster; then
+  echo "Restoring Git repositories into cluster ..."
+  ghe-restore-repositories-dgit "$GHE_HOSTNAME" 1>&3
 
-if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
-    echo "Restoring asset attachments ..."
-    ghe-restore-userdata alambic_assets "$GHE_HOSTNAME" 1>&3
+  echo "Restoring Gists into cluster ..."
+  ghe-restore-repositories-gist "$GHE_HOSTNAME" 1>&3
+else
+  # Remove temporary 2.2 storage migration directory if it exists
+  echo "if [ -d /data/user/repositories-nw-backup ]; then sudo rm -rf /data/user/repositories-nw-backup; fi" |
+  ghe-ssh "$GHE_HOSTNAME" -- /bin/sh 1>&3
 
-    echo "Restoring hook deliveries ..."
-    ghe-restore-userdata hookshot "$GHE_HOSTNAME" 1>&3
+  echo "Restoring Git repositories ..."
+  ghe-restore-repositories-${GHE_BACKUP_STRATEGY} "$GHE_HOSTNAME" 1>&3
 fi
 
-echo "Restoring MySQL database ..."
-gzip -dc "$GHE_RESTORE_SNAPSHOT_PATH/mysql.sql.gz" | ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-mysql' 1>&3
-
-echo "Restoring Redis database ..."
-ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-redis' < "$GHE_RESTORE_SNAPSHOT_PATH/redis.rdb" 1>&3
+if $cluster; then
+  echo "Restoring GitHub Pages into DPages..."
+  ghe-restore-pages-dpages "$GHE_HOSTNAME" 1>&3
+else
+  echo "Restoring GitHub Pages ..."
+  ghe-restore-pages-${GHE_BACKUP_STRATEGY} "$GHE_HOSTNAME" 1>&3
+fi
 
 echo "Restoring SSH authorized keys ..."
 ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-authorized-keys' < "$GHE_RESTORE_SNAPSHOT_PATH/authorized-keys.json" 1>&3
 
-echo "Restoring Elasticsearch indices ..."
-ghe-restore-es-${GHE_BACKUP_STRATEGY} "$GHE_HOSTNAME" 1>&3
+if $cluster; then
+  echo "Restoring storage data ..."
+  ghe-restore-alambic-cluster "$GHE_HOSTNAME" 1>&3
+elif [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
+  echo "Restoring asset attachments ..."
+  ghe-restore-userdata alambic_assets "$GHE_HOSTNAME" 1>&3
+
+  echo "Restoring storage data ..."
+  ghe-restore-userdata storage "$GHE_HOSTNAME" 1>&3
+
+  echo "Restoring hook deliveries ..."
+  ghe-restore-userdata hookshot "$GHE_HOSTNAME" 1>&3
+fi
+
+if $cluster; then
+  echo "Restoring ElasticSearch Audit logs"
+  ghe-restore-es-audit-log "$GHE_HOSTNAME" 1>&3
+else
+  echo "Restoring Elasticsearch indices ..."
+  ghe-restore-es-${GHE_BACKUP_STRATEGY} "$GHE_HOSTNAME" 1>&3
+fi
 
 # Restart an already running memcached to reset the cache after restore
 if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
@@ -220,10 +255,13 @@ fi
 
 # When restoring to a host that has already been configured, kick off a
 # config run to perform data migrations.
-if $instance_configured; then
+if $cluster; then
+  echo "Configuring cluster ..."
+  ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-config-apply" 1>&3 2>&3
+elif $instance_configured; then
   echo "Configuring storage ..."
   if [ "$GHE_VERSION_MAJOR" -ge 2 ]; then
-    ghe-ssh "$GHE_HOSTNAME" -- "sudo ghe-config-apply --full" 1>&3
+    ghe-ssh "$GHE_HOSTNAME" -- "ghe-config-apply --full" 1>&3 2>&3
   else
     echo "   This will take several minutes to complete..."
     ghe-ssh "$GHE_HOSTNAME" -- "sudo enterprise-configure" 1>&3 2>&3
@@ -239,8 +277,13 @@ update_restore_status "complete"
 # Log restore complete message in /var/log/syslog on remote instance
 ghe_remote_logger "Completed restore from $(hostname) / snapshot ${GHE_SNAPSHOT_TIMESTAMP}."
 
-echo "Restoring SSH host keys ..."
-ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-ssh-host-keys' < "$GHE_RESTORE_SNAPSHOT_PATH/ssh-host-keys.tar" 1>&3
+if ! $cluster; then
+  echo "Restoring SSH host keys ..."
+  ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-ssh-host-keys' < "$GHE_RESTORE_SNAPSHOT_PATH/ssh-host-keys.tar" 1>&3
+fi
 
 echo "Completed restore of $GHE_HOSTNAME from snapshot $GHE_RESTORE_SNAPSHOT"
-echo "Visit https://$hostname/setup/settings to review appliance configuration."
+
+if ! $cluster; then
+  echo "Visit https://$hostname/setup/settings to review appliance configuration."
+fi

debian/changelog

@@ -1,3 +1,11 @@
+github-backup-utils (2.5.0) UNRELEASED; urgency=medium
+
+  * Adds GitHub Enterpise 2.5 support
+  * Adds GitHub Enterprise Cluster support
+  * Backups and restores SAML keypairs
+
+ -- Sergio Rubio <[email protected]>  Tue, 9 Feb 2016 00:02:37 +0000
+
 github-backup-utils (2.4.0) UNRELEASED; urgency=medium
 
   * Moves the in-progress detection to a separate file with PID which is

share/github-backup-utils/ghe-backup-alambic-cluster

@@ -0,0 +1,86 @@
+#!/bin/sh
+#/ Usage: ghe-backup-alambic-cluster
+#/ Take an online, incremental snapshot of all Alambic Storage data
+#/
+#/ Note: This command typically isn't called directly. It's invoked by
+#/ ghe-backup when the cluster strategy is used.
+set -e
+
+# Bring in the backup configuration
+cd $(dirname "$0")/../..
+. share/github-backup-utils/ghe-backup-config
+
+# Set up remote host and root backup snapshot directory based on config
+host="$GHE_HOSTNAME"
+backup_dir="$GHE_SNAPSHOT_DIR/storage"
+
+# Verify rsync is available.
+if ! rsync --version 1>/dev/null 2>&1; then
+    echo "Error: rsync not found." 1>&2
+    exit 1
+fi
+
+# Perform a host-check and establish GHE_REMOTE_XXX variables.
+ghe_remote_version_required "$host"
+
+# Generate SSH config for forwarding
+
+config=""
+
+# Split host:port into parts
+port=$(ssh_port_part "$GHE_HOSTNAME")
+host=$(ssh_host_part "$GHE_HOSTNAME")
+
+# Add user / -l option
+user="${host%@*}"
+[ "$user" = "$host" ] && user="admin"
+
+# git server hostnames
+hostnames=$(ghe_cluster_online_nodes "storage-server")
+
+for hostname in $hostnames; do
+  config="$config
+Host $hostname
+  ProxyCommand ssh -q $GHE_EXTRA_SSH_OPTS -p $port $user@$host nc.openbsd %h %p
+  StrictHostKeyChecking=no
+"
+done
+
+config_file=$(mktemp -t cluster-backup-restore-XXXXXX)
+echo "$config" > "$config_file"
+
+opts="$GHE_EXTRA_SSH_OPTS -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no"
+
+# Make sure root backup dir exists if this is the first run
+mkdir -p "$backup_dir"
+
+# Removes the remote sync-in-progress file on exit, re-enabling GC operations
+# on the remote instance.
+cleanup() {
+  rm -f $config_file
+}
+trap 'cleanup' EXIT INT
+
+# If we have a previous increment and it is not empty, avoid transferring existing files via rsync's
+# --link-dest support. This also decreases physical space usage considerably.
+if [ -d "$GHE_DATA_DIR/current/storage" ] && [ "$(ls -A $GHE_DATA_DIR/current/storage)" ]; then
+  link_dest="--link-dest=../../current/storage"
+fi
+
+for hostname in $hostnames; do
+  echo 1>&3
+  echo "* Starting backup for host: $hostname"
+  # Sync all auxiliary repository data. This includes files and directories like
+  # HEAD, audit_log, config, description, info/, etc. No refs or object data
+  # should be transferred here.
+  echo 1>&3
+  echo "* Transferring storage files ..." 1>&3
+
+  # Transfer all data from the user data directory using rsync.
+  ghe-rsync -az \
+      -e "ssh -q $opts -p 122 -F $config_file -l $user" \
+      --rsync-path='sudo -u git rsync' \
+      $link_dest \
+      "$hostname:$GHE_REMOTE_DATA_USER_DIR/storage/" \
+      "$GHE_SNAPSHOT_DIR/storage" 1>&3
+done

share/github-backup-utils/ghe-backup-config

@@ -130,6 +130,10 @@ GHE_SNAPSHOT_DIR="$GHE_DATA_DIR"/"$GHE_SNAPSHOT_TIMESTAMP"
 # Set "true" to get verbose logging of all ssh commands on stderr
 : ${GHE_VERBOSE_SSH:=false}
 
+# The location of the cluster configuration file file on the remote side.
+# This is always "/data/user/common/cluster.conf" for GitHub Cluster instances.
+# Use of this variable allows the location to be overridden in tests.
+: ${GHE_REMOTE_CLUSTER_CONF_FILE:="$GHE_REMOTE_DATA_DIR/user/common/cluster.conf"}
 
 ###############################################################################
 ### Dynamic remote version config
@@ -241,3 +245,10 @@ ghe_remote_logger () {
     echo "$@" |
     ghe-ssh "$GHE_HOSTNAME" -- logger -t backup-utils || true
 }
+
+# Usage: ghe_cluster_online_nodes role
+# Returns the online nodes with a certain role in cluster
+ghe_cluster_online_nodes () {
+    role=$1
+    echo "ghe-config --get-regexp cluster.*.$role | egrep 'true$' | awk '{ print \$1; }' | awk 'BEGIN { FS=\".\" }; { print \$2 };' | xargs -I{} -n1 bash -c 'if [ \"\$(ghe-config cluster.\$hostname.offline)\" != true ]; then ghe-config cluster.{}.hostname; fi'" | ghe-ssh "$GHE_HOSTNAME" /bin/bash
+}

share/github-backup-utils/ghe-backup-es-audit-log

@@ -0,0 +1,32 @@
+#!/bin/sh
+#/ Usage: ghe-backup-es-audit-log
+#/ Take a backup of audit logs in ElasticSearch.
+#/
+#/ Note: This command typically isn't called directly. It's invoked by
+#/ ghe-backup.
+set -e
+
+# Bring in the backup configuration
+cd $(dirname "$0")/../..
+. share/github-backup-utils/ghe-backup-config
+
+# Set up remote host and root elastic backup directory based on config
+host="$GHE_HOSTNAME"
+
+# Perform a host-check and establish GHE_REMOTE_XXX variables.
+ghe_remote_version_required "$host"
+
+# Make sure root backup dir exists if this is the first run
+mkdir -p "$GHE_SNAPSHOT_DIR/audit-log"
+
+indices=$(ghe-ssh "$host" 'curl -s "localhost:9201/_cat/indices/audit_log*"' | cut -d ' ' -f 3)
+current_index=audit_log-$(ghe-ssh "$host" 'date +"%Y-%m"')
+
+for index in $indices; do
+  if [ -f $GHE_DATA_DIR/current/audit-log/$index.gz -a $index \< $current_index ]; then
+    # Hard link any older indices since they are read only and won't change
+    ln $GHE_DATA_DIR/current/audit-log/$index.gz $GHE_SNAPSHOT_DIR/audit-log/$index.gz
+  else
+    ghe-ssh "$host" "/usr/local/share/enterprise/ghe-es-dump-json 'http://localhost:9201/$index'" | gzip > $GHE_SNAPSHOT_DIR/audit-log/$index.gz
+  fi
+done