Merge pull request #41 from github/upstream-merge

Upstream merge

Author: rubiojr

backup.config-example

@@ -32,8 +32,14 @@ GHE_NUM_SNAPSHOTS=10
 # Any extra options passed to the rsync command. Nothing required by default.
 #
 #GHE_EXTRA_RSYNC_OPTS=""
-#
+
 # If set to 'no', GHE_DATA_DIR will not be created automatically
 # and restore/backup will exit 8
 #
 #GHE_CREATE_DATA_DIR=yes
+
+# If set to 'yes', git fsck will run on the repositories
+# and print some additional info.
+#
+# WARNING: do not enable this, only useful for debugging/development
+#GHE_BACKUP_FSCK=no

bin/ghe-backup

@@ -210,6 +210,12 @@ if $GHE_MAINTENANCE_MODE_ENABLED; then
     GHE_MAINTENANCE_MODE_ENABLED=false
 fi
 
+# git fsck repositories after the backup
+if [ "$GHE_BACKUP_FSCK" = "yes" ]; then
+  ghe-backup-fsck $GHE_SNAPSHOT_DIR ||
+    failures="$failures fsck"
+fi
+
 # If everything was successful, mark the snapshot as complete, update the
 # current symlink to point to the new snapshot and prune expired and failed
 # snapshots.

share/github-backup-utils/ghe-backup-config

@@ -21,9 +21,6 @@ BACKUP_UTILS_VERSION="$(cat share/github-backup-utils/version)"
 # Add the bin and share/github-backup-utils dirs to PATH
 PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH"
 
-# The backup config file. This may be set in the environment.
-: ${GHE_BACKUP_CONFIG:="$GHE_BACKUP_ROOT/backup.config"}
-
 # Parse out -v (verbose) argument
 if [ "$1" = "-v" ]; then
     GHE_VERBOSE=true
@@ -43,10 +40,11 @@ fi
 # backup.config value when set.
 GHE_HOSTNAME_PRESERVE="$GHE_HOSTNAME"
 
-# Source in the backup config file from the local working copy location first
-# and then falling back to the system location.
+# Source in the backup config file from the copy specified in the environment
+# first and then fall back to the backup-utils root, home directory and system.
 config_found=false
-for f in "$GHE_BACKUP_CONFIG" "/etc/github-backup-utils/backup.config"; do
+for f in "$GHE_BACKUP_CONFIG" "$GHE_BACKUP_ROOT/backup.config" \
+  "$HOME/.github-backup-utils/backup.config" "/etc/github-backup-utils/backup.config"; do
     if [ -f "$f" ]; then
         GHE_BACKUP_CONFIG="$f"
         . "$GHE_BACKUP_CONFIG"
@@ -58,7 +56,9 @@ done
 # Check that the config file exists before we source it in.
 if ! $config_found; then
     echo "Error: No backup configuration file found. Tried:" 1>&2
-    echo " - $GHE_BACKUP_CONFIG" 1>&2
+    [ -n "$GHE_BACKUP_CONFIG" ] && echo " - $GHE_BACKUP_CONFIG" 1>&2
+    echo " - $GHE_BACKUP_ROOT/backup.config" 1>&2
+    echo " - $HOME/.github-backup-utils/backup.config" 1>&2
     echo " - /etc/github-backup-utils/backup.config" 1>&2
     exit 2
 fi

share/github-backup-utils/ghe-backup-fsck

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-backup-fsck <snapshot-dir> [--print-nwo]
+#/
+#/ Run git fsck on backed up repositories.
+set -e
+
+# Bring in the backup configuration
+cd $(dirname "$0")/../..
+. share/github-backup-utils/ghe-backup-config
+
+echo "Running git fsck on repos..."
+
+# Verify git is available.
+if ! git --version 1>/dev/null 2>&1; then
+  echo "Error: git not found." 1>&2
+  exit 1
+fi
+
+sdir=$1
+repos=0
+errors=0
+log=$(mktemp)
+t_start=$(date +%s)
+
+if [ -z "$sdir" ] || [ ! -d "$sdir" ]; then
+  print_usage
+fi
+
+if [ ! -d "$sdir/repositories" ]; then
+  echo "Error: $sdir is not a valid snapshot." >&2
+  exit 1
+fi
+
+for repo in $(find $sdir/repositories/ -type d -name \*.git); do
+  repos=$(($repos+1))
+  before_time=$(date +%s)
+
+  status=$(
+    set -e
+
+    cd $repo
+
+    nwo="-"
+    if [ "$2" = "--print-nwo" ] && [ -f info/nwo ]; then
+      nwo="$(cat info/nwo)"
+    fi
+
+    if [ ! -f objects/info/alternates ] || grep -q '^\.\.' objects/info/alternates; then
+      git fsck --no-dangling >$log 2>&1 && {
+        echo "OK $repo $nwo"; exit
+      }
+    else
+      GIT_ALTERNATE_OBJECT_DIRECTORIES=../network.git/objects git fsck --no-dangling >$log 2>&1 && {
+        echo "WARN $repo $nwo (alternates absolute path)"; exit
+      }
+    fi
+
+    echo "ERROR $repo $nwo"
+  )
+
+  elapsed_time=$(($(date +%s) - before_time))
+
+  if [[ ! "$status" =~ ^OK ]] || [ $elapsed_time -gt 5 ]; then
+    echo "$status ${elapsed_time}s" 1>&3
+    [ -n "$GHE_VERBOSE" ] && cat $log
+  fi
+
+  case "$status" in
+    OK*)
+      ;;
+    ERROR*)
+      errors=$(($errors+1))
+      ;;
+   esac
+
+done
+
+echo "* Repos verified: $repos, Errors: $errors, Took: $(($(date +%s) - $t_start))s"
+
+rm -f $log

share/github-backup-utils/ghe-backup-repositories-cluster

@@ -84,68 +84,26 @@ echo "$config" > "$config_file"
 
 opts="$GHE_EXTRA_SSH_OPTS -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PasswordAuthentication=no"
 
-# Remote sync-in-progress file location. When this file exists, Git GC
-# operations are disabled on the GitHub instance.
-sync_in_progress_file="$GHE_REMOTE_DATA_USER_DIR/repositories/.sync_in_progress"
-
 # Make sure root backup dir exists if this is the first run
 mkdir -p "$backup_dir"
 
 # Removes the remote sync-in-progress file on exit, re-enabling GC operations
 # on the remote instance.
 cleanup() {
+  # Enable remote GC operations
   for hostname in $hostnames; do
-    ghe-ssh -F $config_file "$hostname:122" -- "sudo rm -f '$sync_in_progress_file'"
+    ghe-gc-enable -F $config_file $hostname:$port
   done
   rm -f $config_file
 }
 trap 'cleanup' EXIT
 trap 'exit $?' INT # ^C always terminate
 
-wait_pids=""
+# Disable remote GC operations
 for hostname in $hostnames; do
-  # Touch the sync-in-progress file, disabling GC operations, and wait for all
-  # active GC processes to finish on the remote side.
-  echo "
-      set -e
-      sudo -u git touch '$sync_in_progress_file'
-
-      sanity=0
-      while [ \$sanity -lt $GHE_GIT_COOLDOWN_PERIOD ]; do
-          # note: the bracket synta[x] below is to prevent matches against the
-          # grep process itself.
-          if ps axo args | grep -E -e '^git( -.*)? nw-repac[k]( |$)' -e '^git( -.*)? g[c]( |$)' >/dev/null; then
-              sleep 1
-              sanity=\$(( sanity + 1 ))
-          else
-              exit 0
-          fi
-      done
-      exit 7
-  " | ssh -q $opts -p $port -F $config_file -l $user $hostname -- /bin/sh
-  wait_pids="$wait_pids $hostname:$!"
-done
-
-# strip leading space
-wait_pids="$(echo -e "${wait_pids}" | sed -e 's/^[[:space:]]*//')"
-
-# wait for jobs one by one, bail on the first one
-for pid in $wait_pids; do
-  # pull out pid:hostname pairs into vars
-  wait_hostname="$(echo $pid | awk 'BEGIN {FS = ":" }; { print $1; }')"
-  wait_pid="$(echo $pid | awk 'BEGIN {FS = ":" }; { print $2; }')"
-  wait $wait_pid || {
-      res=$?
-      if [ $res = 7 ]; then
-          echo "Error: Git GC processes remain after $GHE_GIT_COOLDOWN_PERIOD seconds on host: $wait_hostname. Aborting..." 1>&2
-      fi
-      exit $res
-  }
-  unset wait_hostname wait_pid
+  ghe-gc-disable -F $config_file $hostname:$port
 done
 
-unset wait_pids
-
 # If we have a previous increment, avoid transferring existing files via rsync's
 # --link-dest support. This also decreases physical space usage considerably.
 if [ -d "$backup_current" ]; then

share/github-backup-utils/ghe-backup-repositories-rsync

@@ -51,46 +51,20 @@ fi
 # Perform a host-check and establish GHE_REMOTE_XXX variables.
 ghe_remote_version_required "$host"
 
-# Remote sync-in-progress file location. When this file exists, Git GC
-# operations are disabled on the GitHub instance.
-sync_in_progress_file="$GHE_REMOTE_DATA_USER_DIR/repositories/.sync_in_progress"
-
 # Make sure root backup dir exists if this is the first run
 mkdir -p "$backup_dir"
 
 # Removes the remote sync-in-progress file on exit, re-enabling GC operations
 # on the remote instance.
 cleanup() {
-  ghe-ssh "$host" -- "sudo rm -f '$sync_in_progress_file'"
+  # Enable remote GC operations
+  ghe-gc-enable $host
 }
 trap 'cleanup' EXIT
 trap 'exit $?' INT # ^C always terminate
 
-# Touch the sync-in-progress file, disabling GC operations, and wait for all
-# active GC processes to finish on the remote side.
-echo "
-    set -e
-    sudo -u git touch '$sync_in_progress_file'
-
-    sanity=0
-    while [ \$sanity -lt $GHE_GIT_COOLDOWN_PERIOD ]; do
-        # note: the bracket synta[x] below is to prevent matches against the
-        # grep process itself.
-        if ps axo args | grep -E -e '^git( -.*)? nw-repac[k]( |$)' -e '^git( -.*)? g[c]( |$)' >/dev/null; then
-            sleep 1
-            sanity=\$(( sanity + 1 ))
-        else
-            exit 0
-        fi
-    done
-    exit 7
-" | ghe-ssh "$host" -- /bin/sh || {
-    res=$?
-    if [ $res = 7 ]; then
-        echo "Error: Git GC processes remain after $GHE_GIT_COOLDOWN_PERIOD seconds. Aborting..." 1>&2
-    fi
-    exit $res
-}
+# Disable remote GC operations
+ghe-gc-disable $host
 
 # Transfer repository data from a GitHub instance to the current snapshot
 # directory, using a previous snapshot to avoid transferring files that have

share/github-backup-utils/ghe-backup-store-version

@@ -14,5 +14,5 @@ if [ -d .git ]; then
     version_info="$version_info:$ref"
   fi
   echo "$version_info" |
-  ghe-ssh "$GHE_HOSTNAME" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/common/backup-utils-version"
+  ghe-ssh "$GHE_HOSTNAME" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/common/backup-utils-version >/dev/null 2>&1"
 fi

test/test-ghe-backup.sh

@@ -54,13 +54,15 @@ fi
 mkdir "$GHE_REMOTE_DATA_USER_DIR/repositories"
 cd "$GHE_REMOTE_DATA_USER_DIR/repositories"
 mkdir alice bob
-mkdir alice/repo1.git alice/repo2.git bob/repo3.git
+mkdir alice/repo1.git alice/repo2.git bob/repo3.git alice/broken.git
 
 # Initialize test repositories with a fake commit
 for repo in */*.git; do
     git init -q --bare "$repo"
     git --git-dir="$repo" --work-tree=. commit -q --allow-empty -m 'test commit'
 done
+# Break a repo to test fsck
+rm -f alice/broken.git/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904
 
 begin_test "ghe-backup first snapshot"
 (
@@ -303,3 +305,18 @@ begin_test "ghe-backup empty git-hooks directory"
   [ ! "$(grep "[l]ink-dest arg does not exist" $TRASHDIR/out)" ]
 )
 end_test
+
+begin_test "ghe-backup fsck"
+(
+  set -e
+
+  export GHE_BACKUP_FSCK=yes
+  ghe-backup | grep -q "Repos verified: 4, Errors: 1, Took:"
+  # Verbose mode disabled by default
+  ! ghe-backup | grep -q "missing tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904"
+  ghe-backup -v | grep -q "missing tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904"
+
+  export GHE_BACKUP_FSCK=no
+  ! ghe-backup | grep -q "Repos verified:"
+)
+end_test