Incremental MySQL Backups and Restores (#366)

Implementing incremental mysql backups for 3.10

Co-authored-by: Dax Amin <[email protected]>
Co-authored-by: Manuel Bergler <[email protected]>
Co-authored-by: Chuck Pathanjali <[email protected]>

Author: 4 people

.gitignore

@@ -1,3 +1,5 @@
 /backup.config
 /data
 /dist
+dash
+parallel

bin/ghe-backup

@@ -27,6 +27,10 @@ while true; do
       export GHE_VERBOSE=true
       shift
       ;;
+    -i|--incremental)
+      export GHE_INCREMENTAL=true
+      shift
+      ;;
     -*)
       echo "Error: invalid argument: '$1'" 1>&2
       exit 1
@@ -173,17 +177,34 @@ if [ -f ../in-progress ]; then
   fi
 fi
 
+# Perform a host connection check and establish the remote appliance version.
+# The version is available in the GHE_REMOTE_VERSION variable and also written
+# to a version file in the snapshot directory itself.
+ghe_remote_version_required
+echo "$GHE_REMOTE_VERSION" > version
+
+# check that incremental settings are valid if set
+is_inc=$(is_incremental_backup_feature_on)
+
+if [ "$is_inc" = true ]; then
+if [ "$GHE_VERSION_MAJOR" -lt 3 ] && [ "$GHE_VERSION_MINOR" -lt 10 ]; then
+  log_error "Cannot only perform incremental backups on enterprise version 3.10 or higher"
+  exit 1
+fi
+  incremental_backup_check
+  # If everything is ok, check if we have hit GHE_MAX_INCREMENTAL_BACKUPS, performing pruning actions if necessary
+  check_for_incremental_max_backups
+  # initialize incremental backup if it hasn't been done yet
+  incremental_backup_init
+fi
+
 echo "$GHE_SNAPSHOT_TIMESTAMP $$" > ../in-progress
 echo "$GHE_SNAPSHOT_TIMESTAMP $$" > "${GHE_DATA_DIR}/in-progress-backup"
 
 START_TIME=$(date +%s)
 log_info "Starting backup of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION in snapshot $GHE_SNAPSHOT_TIMESTAMP"
 
-# Perform a host connection check and establish the remote appliance version.
-# The version is available in the GHE_REMOTE_VERSION variable and also written
-# to a version file in the snapshot directory itself.
-ghe_remote_version_required
-echo "$GHE_REMOTE_VERSION" > version
+
 
 if [ -n "$GHE_ALLOW_REPLICA_BACKUP" ]; then
   echo "Warning: backing up a high availability replica may result in inconsistent or unreliable backups."

bin/ghe-restore

@@ -71,6 +71,10 @@ while true; do
       export GHE_VERBOSE=true
       shift
       ;;
+     -i|--incremental)
+      export GHE_INCREMENTAL=true
+      shift
+      ;;
     -*)
       echo "Error: invalid argument: '$1'" 1>&2
       exit 1
@@ -321,6 +325,21 @@ echo "$PROGRESS" > /tmp/backup-utils-progress
 START_TIME=$(date +%s)
 log_info "Starting restore of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION from snapshot $GHE_RESTORE_SNAPSHOT"
 
+if [ "$GHE_INCREMENTAL" ]; then
+  log_info "Incremental restore from snapshot $GHE_RESTORE_SNAPSHOT"
+  # If we see 'inc_previous' prepended to the snapshot name, then
+  # we set $INC_FULL_BACKUP and $INC_SNAPSHOT_DATA to $INC_PREVIOUS_FULL_BACKUP and
+  # $INC_PREVIOUS_SNAPSHOT_DATA respectively. Otherwise, leave them at default setting
+  # so that incremental restore is from current cycle
+  if [[ "$GHE_RESTORE_SNAPSHOT" =~ ^inc_previous ]]; then
+    INC_FULL_BACKUP=$INC_PREVIOUS_FULL_BACKUP
+    INC_SNAPSHOT_DATA=$INC_PREVIOUS_SNAPSHOT_DATA
+    log_info "Incremental restore from previous cycle snapshot. Using $INC_FULL_BACKUP"
+    log_info "Incremental restore from previous cycle snapshot. Using $INC_SNAPSHOT_DATA"
+  fi 
+  log_info "Validating snapshot $GHE_RESTORE_SNAPSHOT"
+  validate_inc_snapshot_data "$GHE_RESTORE_SNAPSHOT"
+fi
 ghe_remote_logger "Starting restore from $(hostname) with backup-utils v$BACKUP_UTILS_VERSION / snapshot $GHE_RESTORE_SNAPSHOT ..."
 # Create an in-progress-restore file to prevent simultaneous backup or restore runs
 echo "${START_TIME} $$" > "${GHE_DATA_DIR}/in-progress-restore"
@@ -498,6 +517,8 @@ else
   ghe-restore-mysql "$GHE_HOSTNAME" 1>&3
 fi
 
+
+
 if ghe-ssh "$GHE_HOSTNAME" -- 'ghe-config --true app.actions.enabled'; then
   log_info "Stopping Actions before restoring databases ..."
   # We mark Actions as stopped even if the `ghe-actions-stop`
@@ -601,7 +622,7 @@ fi
 log_info  "Restarting memcached ..." 1>&3
 bm_start "$(basename $0) - Restarting memcached"
 echo "sudo restart -q memcached 2>/dev/null || true" |
-  ghe-ssh "$GHE_HOSTNAME" -- /bin/sh
+ghe-ssh "$GHE_HOSTNAME" -- /bin/sh
 bm_end "$(basename $0) - Restarting memcached"
 
 # Prevent GitHub Connect jobs running before we've had a chance to reset

share/github-backup-utils/ghe-backup-config

@@ -183,6 +183,8 @@ fi
 PATH="$GHE_BACKUP_ROOT/bin:$GHE_BACKUP_ROOT/share/github-backup-utils:$PATH"
 # shellcheck source=share/github-backup-utils/bm.sh
 . "$GHE_BACKUP_ROOT/share/github-backup-utils/bm.sh"
+# shellcheck source=share/github-backup-utils/ghe-incremental-backup-restore
+. "$GHE_BACKUP_ROOT/share/github-backup-utils/ghe-incremental-backup-restore"
 # shellcheck source=share/github-backup-utils/track-progress
 . "$GHE_BACKUP_ROOT/share/github-backup-utils/track-progress"
 # Save off GHE_HOSTNAME from the environment since we want it to override the
@@ -712,12 +714,9 @@ init-progress() {
   rm -f /tmp/backup-utils-progress*
 }
 
+
 #increase total count of progress
 increment-progress-total-count() {
   ((PROGRESS_TOTAL += $1))
   echo "$PROGRESS_TOTAL" > /tmp/backup-utils-progress-total
 }
-
-
-
-

share/github-backup-utils/ghe-backup-mysql

@@ -33,6 +33,14 @@ else
   if is_binary_backup_feature_on; then
     ghe-backup-mysql-binary
   else
+    # if incremental backups are turned on, we can't do them with
+    # logical backups, so we need to tell the user and exit
+    is_inc=$(is_incremental_backup_feature_on)
+    if [ $is_inc = true ]; then
+      log_warn "Incremental backups are configured on the target environment."
+      log_warn "Incremental backup is not supported with a logical MySQL backup. Please disable incremental backups with 'ghe-config mysql.backup.incremental false'"
+      exit 1
+    fi
     ghe-backup-mysql-logical
   fi
 fi

share/github-backup-utils/ghe-backup-mysql-binary

@@ -16,9 +16,45 @@ bm_start "$(basename $0)"
 ghe_remote_version_required "$GHE_HOSTNAME"
 
 log_verbose "Backing up MySQL database using binary backup strategy ..."
-
-echo "set -o pipefail; ghe-export-mysql" |
-ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > "$GHE_SNAPSHOT_DIR/mysql.sql.gz"
-echo "NO_ADDITIONAL_COMPRESSION" > "$GHE_SNAPSHOT_DIR/mysql-binary-backup-sentinel"
-
+is_inc=$(is_incremental_backup_feature_on)
+if [ $is_inc = true ]; then
+    log_verbose "Incremental backups are configured on the target environment."
+    log_info "Performing incremental backup of MySQL database ..." 1>&3
+    INC_TYPE=$(full_or_incremental_backup)
+    INC_LSN=""
+    if [ "$INC_TYPE" == "full" ]; then
+        log_info "Incremental backup type: $INC_TYPE"  1>&3
+        INC_LSN=0 # 0 means full backup
+    else
+        validate_inc_snapshot_data
+        log_info "Incremental backup type: $INC_TYPE" 1>&3
+        INC_LSN=$(retrieve_last_lsn)
+    fi
+    echo "set -o pipefail; env INC_BACKUP=$INC_LSN ghe-export-mysql" |
+    ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > "$GHE_SNAPSHOT_DIR/mysql.sql.gz"
+    echo "NO_ADDITIONAL_COMPRESSION" > "$GHE_SNAPSHOT_DIR/mysql-binary-backup-sentinel"
+    # Ensure that we capture the xtrabackup_checkpoints file from the remote host
+    log_info "Checking if incremental backup is part of a cluster"
+    GET_LSN=$(get_cluster_lsn "$GHE_HOSTNAME")
+    ghe-ssh "$GHE_HOSTNAME" "$GET_LSN" > "$GHE_SNAPSHOT_DIR/xtrabackup_checkpoints"
+    if [ "$INC_TYPE" == "full" ]; then
+        log_info "Adding $GHE_SNAPSHOT_DIR to the list of full backups" 1>&3
+        update_inc_full_backup "$GHE_SNAPSHOT_DIR"
+    else
+        log_info "Adding $GHE_SNAPSHOT_DIR to the list of incremental backups" 1>&3
+        update_inc_snapshot_data "$GHE_SNAPSHOT_DIR"
+    fi
+    bm_end "$(basename $0)"
+    exit 0
+fi
+# if incremental backup isn't enabled, or we are performing a full backup as part of the process,
+# fall through and do a full backup
+    echo "set -o pipefail; ghe-export-mysql" |
+    ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > "$GHE_SNAPSHOT_DIR/mysql.sql.gz"
+    echo "NO_ADDITIONAL_COMPRESSION" > "$GHE_SNAPSHOT_DIR/mysql-binary-backup-sentinel"
+    is_inc=$(is_incremental_backup_feature_on)
+    if [ $is_inc = true ]; then
+        update_inc_full_backup "$GHE_SNAPSHOT_DIR"
+    fi
 bm_end "$(basename $0)"
+