Merge branch 'master' into ddivad195/container-prefix-fix

Author: ddivad195

.gitignore

@@ -1,3 +1,5 @@
 /backup.config
 /data
 /dist
+dash
+parallel

backup.config-example

@@ -16,6 +16,10 @@ GHE_DATA_DIR="data"
 # be available for the past N days ...
 GHE_NUM_SNAPSHOTS=10
 
+# Pruning snapshots can be scheduled outside of the backup process. If set to 'yes'
+# ghe-pruning-snapshots will need to be invoked separately via cron
+#GHE_PRUNING_SCHEDULED=yes
+
 # The hostname of the GitHub appliance to restore. If you've set up a separate
 # GitHub appliance to act as a standby for recovery, specify its IP or hostname
 # here. The host to restore to may also be specified directly when running

bin/ghe-backup

@@ -27,6 +27,10 @@ while true; do
       export GHE_VERBOSE=true
       shift
       ;;
+    -i|--incremental)
+      export GHE_INCREMENTAL=true
+      shift
+      ;;
     -*)
       echo "Error: invalid argument: '$1'" 1>&2
       exit 1
@@ -46,12 +50,36 @@ export CALLING_SCRIPT="ghe-backup"
 
 # Setup progress tracking
 init-progress
+export PROGRESS_TOTAL=14 # Minimum number of steps in backup is 14
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
 export PROGRESS_TYPE="Backup"
 echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
 export PROGRESS=0 # Used to track progress of backup
 echo "$PROGRESS" > /tmp/backup-utils-progress
-export PROGRESS_TOTAL=18 # Maximum number of steps in backup
 
+OPTIONAL_STEPS=0
+# Backup actions+mssql
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2))
+fi
+
+# Backup fsck
+if [ "$GHE_BACKUP_FSCK" = "yes" ]; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+# Backup minio
+if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+# Backup pages
+if [ "$GHE_BACKUP_PAGES" != "no" ]; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+fi
+
+PROGRESS_TOTAL=$((OPTIONAL_STEPS + PROGRESS_TOTAL)) # Minimum number of steps in backup is 14
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
 # Check to make sure moreutils parallel is installed and working properly
 ghe_parallel_check
 
@@ -149,17 +177,34 @@ if [ -f ../in-progress ]; then
   fi
 fi
 
+# Perform a host connection check and establish the remote appliance version.
+# The version is available in the GHE_REMOTE_VERSION variable and also written
+# to a version file in the snapshot directory itself.
+ghe_remote_version_required
+echo "$GHE_REMOTE_VERSION" > version
+
+# check that incremental settings are valid if set
+is_inc=$(is_incremental_backup_feature_on)
+
+if [ "$is_inc" = true ]; then
+if [ "$GHE_VERSION_MAJOR" -lt 3 ] && [ "$GHE_VERSION_MINOR" -lt 10 ]; then
+  log_error "Cannot only perform incremental backups on enterprise version 3.10 or higher"
+  exit 1
+fi
+  incremental_backup_check
+  # If everything is ok, check if we have hit GHE_MAX_INCREMENTAL_BACKUPS, performing pruning actions if necessary
+  check_for_incremental_max_backups
+  # initialize incremental backup if it hasn't been done yet
+  incremental_backup_init
+fi
+
 echo "$GHE_SNAPSHOT_TIMESTAMP $$" > ../in-progress
 echo "$GHE_SNAPSHOT_TIMESTAMP $$" > "${GHE_DATA_DIR}/in-progress-backup"
 
 START_TIME=$(date +%s)
 log_info "Starting backup of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION in snapshot $GHE_SNAPSHOT_TIMESTAMP"
 
-# Perform a host connection check and establish the remote appliance version.
-# The version is available in the GHE_REMOTE_VERSION variable and also written
-# to a version file in the snapshot directory itself.
-ghe_remote_version_required
-echo "$GHE_REMOTE_VERSION" > version
+
 
 if [ -n "$GHE_ALLOW_REPLICA_BACKUP" ]; then
   echo "Warning: backing up a high availability replica may result in inconsistent or unreliable backups."
@@ -269,6 +314,7 @@ echo \"$cmd_title\"
 ghe-backup-git-hooks || printf %s \"git-hooks \" >> \"$failures_file\"")
 
 if [ "$GHE_BACKUP_STRATEGY" = "rsync" ]; then
+  increment-progress-total-count 1
   cmd_title=$(log_info "Backing up Elasticsearch indices ...")
   commands+=("
   echo \"$cmd_title\"
@@ -302,7 +348,11 @@ if [ -z "$failures" ]; then
   rm -f "../current"
   ln -s "$GHE_SNAPSHOT_TIMESTAMP" "../current"
 
-  ghe-prune-snapshots
+  if [[ $GHE_PRUNING_SCHEDULED != "yes" ]]; then
+    ghe-prune-snapshots
+  else
+    log_info "Expired and incomplete snapshots to be pruned separately"
+  fi
 else 
   log_info "Skipping pruning snapshots, since some backups failed..."
 fi

bin/ghe-restore

@@ -71,6 +71,10 @@ while true; do
       export GHE_VERBOSE=true
       shift
       ;;
+     -i|--incremental)
+      export GHE_INCREMENTAL=true
+      shift
+      ;;
     -*)
       echo "Error: invalid argument: '$1'" 1>&2
       exit 1
@@ -275,17 +279,14 @@ fi
 # taking into account the options passed to the script and the appliance configuration
 # calculate restore steps
 OPTIONAL_STEPS=0
-# Cluster restores add an additional step
-if $CLUSTER ; then
-  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
-fi
+
 # Restoring UUID
 if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then
   OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
 fi
-# Restoring Actions
+# Restoring Actions + MSSQL
 if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
-  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 2))
 fi
 # Restoring minio
 if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.minio.enabled'; then
@@ -305,10 +306,16 @@ fi
 if ! $CLUSTER && $instance_configured; then
   OPTIONAL_STEPS=$((OPTIONAL_STEPS + 1))
 fi
-# Maximum restore steps
-export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 6))
+# Restoring settings + restore-chat-integration + restore-packages
+if $RESTORE_SETTINGS; then
+  OPTIONAL_STEPS=$((OPTIONAL_STEPS + 3))
+fi
+
+# Minimum number of steps is 7
+export PROGRESS_TOTAL=$((OPTIONAL_STEPS + 7))
 
 init-progress
+echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
 export PROGRESS_TYPE="Restore"
 echo "$PROGRESS_TYPE" > /tmp/backup-utils-progress-type
 export PROGRESS=0 # Used to track progress of restore
@@ -318,6 +325,21 @@ echo "$PROGRESS" > /tmp/backup-utils-progress
 START_TIME=$(date +%s)
 log_info "Starting restore of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION from snapshot $GHE_RESTORE_SNAPSHOT"
 
+if [ "$GHE_INCREMENTAL" ]; then
+  log_info "Incremental restore from snapshot $GHE_RESTORE_SNAPSHOT"
+  # If we see 'inc_previous' prepended to the snapshot name, then
+  # we set $INC_FULL_BACKUP and $INC_SNAPSHOT_DATA to $INC_PREVIOUS_FULL_BACKUP and
+  # $INC_PREVIOUS_SNAPSHOT_DATA respectively. Otherwise, leave them at default setting
+  # so that incremental restore is from current cycle
+  if [[ "$GHE_RESTORE_SNAPSHOT" =~ ^inc_previous ]]; then
+    INC_FULL_BACKUP=$INC_PREVIOUS_FULL_BACKUP
+    INC_SNAPSHOT_DATA=$INC_PREVIOUS_SNAPSHOT_DATA
+    log_info "Incremental restore from previous cycle snapshot. Using $INC_FULL_BACKUP"
+    log_info "Incremental restore from previous cycle snapshot. Using $INC_SNAPSHOT_DATA"
+  fi 
+  log_info "Validating snapshot $GHE_RESTORE_SNAPSHOT"
+  validate_inc_snapshot_data "$GHE_RESTORE_SNAPSHOT"
+fi
 ghe_remote_logger "Starting restore from $(hostname) with backup-utils v$BACKUP_UTILS_VERSION / snapshot $GHE_RESTORE_SNAPSHOT ..."
 # Create an in-progress-restore file to prevent simultaneous backup or restore runs
 echo "${START_TIME} $$" > "${GHE_DATA_DIR}/in-progress-restore"
@@ -446,6 +468,7 @@ ghe-restore-column-encryption-keys "$GHE_HOSTNAME"
 # Always restore secret scanning encryption keys
 if [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 3.8.0)" ]; then
   log_info "Always restore secret scanning encryption keys on GHES verions 3.8.0+"
+  increment-progress-total-count 1
   ghe-restore-secret-scanning-encryption-keys "$GHE_HOSTNAME"
 fi
 
@@ -490,9 +513,12 @@ if is_external_database_target_or_snapshot && $SKIP_MYSQL; then
   log_info "Skipping MySQL restore."
 else
   log_info "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..."
+  increment-progress-total-count 2
   ghe-restore-mysql "$GHE_HOSTNAME" 1>&3
 fi
 
+
+
 if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
   log_info "Stopping Actions before restoring databases ..."
   # We mark Actions as stopped even if the `ghe-actions-stop`
@@ -596,7 +622,7 @@ fi
 log_info  "Restarting memcached ..." 1>&3
 bm_start "$(basename $0) - Restarting memcached"
 echo "sudo restart -q memcached 2>/dev/null || true" |
-  ghe-ssh "$GHE_HOSTNAME" -- /bin/sh
+ghe-ssh "$GHE_HOSTNAME" -- /bin/sh
 bm_end "$(basename $0) - Restarting memcached"
 
 # Prevent GitHub Connect jobs running before we've had a chance to reset

docs/scheduling-backups.md

@@ -1,10 +1,9 @@
-# Scheduling backups
+# Scheduling backups & snapshot pruning
 
 Regular backups should be scheduled using `cron(8)` or similar command
 scheduling service on the backup host. The backup frequency will dictate the
 worst case [recovery point objective (RPO)][1] in your backup plan. We recommend
 hourly backups at the least.
-
 ## Example scheduling usage
 
 The following examples assume the Backup Utilities are installed under
@@ -19,16 +18,34 @@ storage.
 
 To schedule hourly backup snapshots with verbose informational output written to
 a log file and errors generating an email:
+```
+[email protected]
 
-    [email protected]
-
-    0 * * * * /opt/backup-utils/bin/ghe-backup -v 1>>/opt/backup-utils/backup.log 2>&1
+0 * * * * /opt/backup-utils/bin/ghe-backup -v 1>>/opt/backup-utils/backup.log 2>&1
+```
 
 To schedule nightly backup snapshots instead, use:
 
-    [email protected]
+```
+[email protected]
+
+0 0 * * * /opt/backup-utils/bin/ghe-backup -v 1>>/opt/backup-utils/backup.log 2>&1
+```
+
+### Example snapshot pruning 
+
+By default all expired and incomplete snapshots are deleted at the end of the main
+backup process `ghe-backup`. If pruning these snapshots takes a long time you can
+choose to disable the pruning process from the backup run and schedule it separately.
+This can be achieved by enabling the `GHE_PRUNING_SCHEDULED` option in `backup.config`.
+Please note that this option is only avilable for `backup-utils` >= v3.10.0, if this option is enabled you will need to schedule the pruning script `ghe-prune-snapshots` using `cron` or a similar command scheduling service on the backup host.
+
+To schedule daily snapshot pruning, use:
 
-    0 0 * * * /opt/backup-utils/bin/ghe-backup -v 1>>/opt/backup-utils/backup.log 2>&1
+```
+[email protected]
 
+0 3 * * * /opt/backup-utils/share/github-backup-utils/ghe-prune-snapshots 1>>/opt/backup-utils/prune-snapshots.log 2>&1
+```
 
 [1]: https://en.wikipedia.org/wiki/Recovery_point_objective

share/github-backup-utils/ghe-backup-config

@@ -183,6 +183,8 @@ fi
 PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH"
 # shellcheck source=share/github-backup-utils/bm.sh
 . "$GHE_BACKUP_ROOT/share/github-backup-utils/bm.sh"
+# shellcheck source=share/github-backup-utils/ghe-incremental-backup-restore
+. "$GHE_BACKUP_ROOT/share/github-backup-utils/ghe-incremental-backup-restore"
 # shellcheck source=share/github-backup-utils/track-progress
 . "$GHE_BACKUP_ROOT/share/github-backup-utils/track-progress"
 # Save off GHE_HOSTNAME from the environment since we want it to override the
@@ -713,5 +715,8 @@ init-progress() {
 }
 
 
-
-
+#increase total count of progress
+increment-progress-total-count() {
+  ((PROGRESS_TOTAL += $1))
+  echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
+}

share/github-backup-utils/ghe-backup-mysql

@@ -33,6 +33,14 @@ else
   if is_binary_backup_feature_on; then
     ghe-backup-mysql-binary
   else
+    # if incremental backups are turned on, we can't do them with
+    # logical backups, so we need to tell the user and exit
+    is_inc=$(is_incremental_backup_feature_on)
+    if [ $is_inc = true ]; then
+      log_warn "Incremental backups are configured on the target environment."
+      log_warn "Incremental backup is not supported with a logical MySQL backup. Please disable incremental backups with 'ghe-config mysql.backup.incremental false'"
+      exit 1
+    fi
     ghe-backup-mysql-logical
   fi
 fi

share/github-backup-utils/ghe-backup-mysql-binary

@@ -16,9 +16,45 @@ bm_start "$(basename $0)"
 ghe_remote_version_required "$GHE_HOSTNAME"
 
 log_verbose "Backing up MySQL database using binary backup strategy ..."
-
-echo "set -o pipefail; ghe-export-mysql" |
-ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > "$GHE_SNAPSHOT_DIR/mysql.sql.gz"
-echo "NO_ADDITIONAL_COMPRESSION" > "$GHE_SNAPSHOT_DIR/mysql-binary-backup-sentinel"
-
+is_inc=$(is_incremental_backup_feature_on)
+if [ $is_inc = true ]; then
+    log_verbose "Incremental backups are configured on the target environment."
+    log_info "Performing incremental backup of MySQL database ..." 1>&3
+    INC_TYPE=$(full_or_incremental_backup)
+    INC_LSN=""
+    if [ "$INC_TYPE" == "full" ]; then
+        log_info "Incremental backup type: $INC_TYPE"  1>&3
+        INC_LSN=0 # 0 means full backup
+    else
+        validate_inc_snapshot_data
+        log_info "Incremental backup type: $INC_TYPE" 1>&3
+        INC_LSN=$(retrieve_last_lsn)
+    fi
+    echo "set -o pipefail; env INC_BACKUP=$INC_LSN ghe-export-mysql" |
+    ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > "$GHE_SNAPSHOT_DIR/mysql.sql.gz"
+    echo "NO_ADDITIONAL_COMPRESSION" > "$GHE_SNAPSHOT_DIR/mysql-binary-backup-sentinel"
+    # Ensure that we capture the xtrabackup_checkpoints file from the remote host
+    log_info "Checking if incremental backup is part of a cluster"
+    GET_LSN=$(get_cluster_lsn "$GHE_HOSTNAME")
+    ghe-ssh "$GHE_HOSTNAME" "$GET_LSN" > "$GHE_SNAPSHOT_DIR/xtrabackup_checkpoints"
+    if [ "$INC_TYPE" == "full" ]; then
+        log_info "Adding $GHE_SNAPSHOT_DIR to the list of full backups" 1>&3
+        update_inc_full_backup "$GHE_SNAPSHOT_DIR"
+    else
+        log_info "Adding $GHE_SNAPSHOT_DIR to the list of incremental backups" 1>&3
+        update_inc_snapshot_data "$GHE_SNAPSHOT_DIR"
+    fi
+    bm_end "$(basename $0)"
+    exit 0
+fi
+# if incremental backup isn't enabled, or we are performing a full backup as part of the process,
+# fall through and do a full backup
+    echo "set -o pipefail; ghe-export-mysql" |
+    ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > "$GHE_SNAPSHOT_DIR/mysql.sql.gz"
+    echo "NO_ADDITIONAL_COMPRESSION" > "$GHE_SNAPSHOT_DIR/mysql-binary-backup-sentinel"
+    is_inc=$(is_incremental_backup_feature_on)
+    if [ $is_inc = true ]; then
+        update_inc_full_backup "$GHE_SNAPSHOT_DIR"
+    fi
 bm_end "$(basename $0)"
+

share/github-backup-utils/ghe-backup-pages

@@ -63,6 +63,7 @@ if [ -d "$GHE_DATA_DIR/current/pages" ] && [ "$(ls -A $GHE_DATA_DIR/current/page
   link_dest="--link-dest=../../current/pages"
 fi
 
+count=0
 for hostname in $hostnames; do
   bm_start "$(basename $0) - $hostname"
   echo 1>&3
@@ -82,6 +83,7 @@ for hostname in $hostnames; do
     "$GHE_SNAPSHOT_DIR/pages" 1>&3
   log_rsync "END: pages rsync" 1>&3
   bm_end "$(basename $0) - $hostname"
+  count=$((count + 1))
 done
-
+increment-progress-total-count $count
 bm_end "$(basename $0)"