Merge pull request #341 from github/separate-pruning-snapshots

Remove ghe-prune-snapshots from ghe-backup so it can be called separately. 
Incremental logic handling to be done outside of this PR.

Author: chuckp22

backup.config-example

@@ -16,6 +16,10 @@ GHE_DATA_DIR="data"
 # be available for the past N days ...
 GHE_NUM_SNAPSHOTS=10
 
+# Pruning snapshots can be scheduled outside of the backup process. If set to 'yes'
+# ghe-pruning-snapshots will need to be invoked separately via cron
+#GHE_PRUNING_SCHEDULED=yes
+
 # The hostname of the GitHub appliance to restore. If you've set up a separate
 # GitHub appliance to act as a standby for recovery, specify its IP or hostname
 # here. The host to restore to may also be specified directly when running

bin/ghe-backup

@@ -327,7 +327,11 @@ if [ -z "$failures" ]; then
   rm -f "../current"
   ln -s "$GHE_SNAPSHOT_TIMESTAMP" "../current"
 
-  ghe-prune-snapshots
+  if [[ $GHE_PRUNING_SCHEDULED != "yes" ]]; then
+    ghe-prune-snapshots
+  else
+    log_info "Expired and incomplete snapshots to be pruned separately"
+  fi
 else 
   log_info "Skipping pruning snapshots, since some backups failed..."
 fi

docs/scheduling-backups.md

@@ -1,4 +1,4 @@
-# Scheduling backups
+# Scheduling backups & snapshot pruning
 
 Regular backups should be scheduled using `cron(8)` or similar command
 scheduling service on the backup host. The backup frequency will dictate the
@@ -17,6 +17,13 @@ based on the frequency of backups. The ten most recent snapshots are retained by
 default. The number should be adjusted based on backup frequency and available
 storage.
 
+By default all expired and incomplete snapshots are deleted at the end of the main
+backup process `ghe-backup`. If pruning these snapshots takes a long time you can
+choose to disable the pruning process from the backup run and schedule it separately.
+This can be achieved by enabling the `GHE_PRUNING_SCHEDULED` option in `backup.config`.
+Please note if this option is enabled, you will need to schedule the pruning script `ghe-prune-snapshots`
+using `cron` or similar command scheduling service on the backup host.
+
 To schedule hourly backup snapshots with verbose informational output written to
 a log file and errors generating an email:
 
@@ -30,5 +37,10 @@ To schedule nightly backup snapshots instead, use:
 
     0 0 * * * /opt/backup-utils/bin/ghe-backup -v 1>>/opt/backup-utils/backup.log 2>&1
 
+To schedule daily snapshot pruning, use:
+
+    [email protected]
+
+    0 3 * * * /opt/backup-utils/share/github-backup-utils/ghe-prune-snapshots 1>>/opt/backup-utils/prune-snapshots.log 2>&1
 
 [1]: https://en.wikipedia.org/wiki/Recovery_point_objective

share/github-backup-utils/ghe-prune-snapshots

@@ -20,21 +20,30 @@ prune_snapshot() {
   done
 }
 
-# First prune all incomplete / failed snapshot directories
-prune_dirs="$(ls -1 "$GHE_DATA_DIR"/[0-9]*/incomplete 2>/dev/null || true)"
-prune_num=$(echo "$prune_dirs" | grep -v '^$' | wc -l)
+# Prune if backup is not running
+#if [ ! -f "$GHE_DATA_DIR/in-progress" ] && [ ! -f "$GHE_DATA_DIR/in-progress-restore" ]; then
+# Check for backup or restore in-progress file
+inprogress_file=$(find $GHE_DATA_DIR -maxdepth 1 -type f \( -name "in-progress" -o -name "in-progress-restore" \) -print -quit)
+if [[ "$CALLING_SCRIPT" == "ghe-backup" ]] || [ -z "$inprogress_file" ]; then
+  # First prune all incomplete / failed snapshot directories
+  prune_dirs="$(ls -1 "$GHE_DATA_DIR"/[0-9]*/incomplete 2>/dev/null || true)"
+  prune_num=$(echo "$prune_dirs" | grep -v '^$' | wc -l)
 
-if [ $prune_num -gt 0 ]; then
-  log_info Pruning $prune_num "failed snapshot(s) ..."
-  echo "$prune_dirs" | sed 's@/incomplete$@@' | prune_snapshot
-fi
+  if [ $prune_num -gt 0 ]; then
+    log_info Pruning $prune_num "failed snapshot(s) ..."
+    echo "$prune_dirs" | sed 's@/incomplete$@@' | prune_snapshot
+  fi
 
-# Now prune all expired snapshots. Keep GHE_NUM_SNAPSHOTS around.
-snapshot_count=$(ls -1d "$GHE_DATA_DIR"/[0-9]* 2>/dev/null | wc -l)
+  # Now prune all expired snapshots. Keep GHE_NUM_SNAPSHOTS around.
+  snapshot_count=$(ls -1d "$GHE_DATA_DIR"/[0-9]* 2>/dev/null | wc -l)
 
-if [ "$snapshot_count" -gt "$GHE_NUM_SNAPSHOTS" ]; then
-  prune_dirs="$(ls -1d "$GHE_DATA_DIR"/[0-9]* | sort -r | awk "NR>$GHE_NUM_SNAPSHOTS")"
-  prune_num=$(echo "$prune_dirs" | grep -v '^$' | wc -l)
-  log_info Pruning $prune_num "expired snapshot(s) ..."
-  echo "$prune_dirs" | prune_snapshot
+  if [ "$snapshot_count" -gt "$GHE_NUM_SNAPSHOTS" ]; then
+    prune_dirs="$(ls -1d "$GHE_DATA_DIR"/[0-9]* | sort -r | awk "NR>$GHE_NUM_SNAPSHOTS")"
+    prune_num=$(echo "$prune_dirs" | grep -v '^$' | wc -l)
+    log_info Pruning $prune_num "expired snapshot(s) ..."
+    echo "$prune_dirs" | prune_snapshot
+  fi
+elif [ "$CALLING_SCRIPT" != "ghe-backup" ] && [ -n "$inprogress_file" ]; then
+  log_info "Detected a running backup/restore process, please wait until that process is complete to prune expired/incomplete snapshots." 1>&2
+  log_info "If no such process is running, please remove the "$GHE_DATA_DIR/in-progress*" file and retry again." 1>&2
 fi

test/test-ghe-prune-snapshots.sh

@@ -89,3 +89,23 @@ begin_test "ghe-prune-snapshots incomplete snapshot pruning"
   [ ! -d "$GHE_DATA_DIR/04" ]
 )
 end_test
+
+begin_test "ghe-prune-snapshots scheduled snapshot pruning"
+(
+  set -e
+  # Create the backup data dir and fake remote repositories dirs
+  mkdir -p "$GHE_DATA_DIR" "$GHE_REMOTE_DATA_USER_DIR"
+
+  setup_test_data $GHE_REMOTE_DATA_USER_DIR
+
+  generate_prune_files 5
+
+  pre_num_files=$(file_count_no_current)
+
+  GHE_NUM_SNAPSHOTS=3 GHE_PRUNING_SCHEDULED=yes ghe-backup
+
+  post_num_files=$(file_count_no_current)
+
+  [ "$((pre_num_files + 1))" = "$post_num_files" ]
+)
+end_test