Add incremental MySQL restore test (#530)

Add basic tests for incremental MySQL restores

Author: gamefiend

test/test-ghe-host-check.sh

@@ -83,7 +83,10 @@ begin_test "ghe-host-check detects unsupported GitHub Enterprise Server versions
        ix=$(( $ix + 1 ))
       done
       GHE_TEST_REMOTE_VERSION="${keys[$ix]}.0" ghe-host-check
-      GHE_TEST_REMOTE_VERSION="${keys[$(( $ix - 1 ))]}.0" ghe-host-check
+      # sometimes when the latest.json is updated during a release this test gets broken.
+      if [ "${keys[$(( $ix - 1 ))]}" != "latest" ]; then
+        GHE_TEST_REMOTE_VERSION="${keys[$(( $ix - 1 ))]}.0" ghe-host-check
+      fi
       GHE_TEST_REMOTE_VERSION="${keys[$(( $ix - 2 ))]}.0" ghe-host-check
 
   fi

test/test-ghe-incremental-restore.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+# ghe-restore command tests
+
+# Bring in testlib
+# shellcheck source=test/testlib.sh
+. "$(dirname "$0")/testlib.sh"
+
+setup_incremental_restore_data 
+setup_actions_enabled_settings_for_restore true
+
+# Make the current symlink
+ln -s 1 "$GHE_DATA_DIR/current"
+begin_test "ghe_restore -i doesn't run on unsupported versions"
+(
+  set -e
+  GHE_RESTORE_HOST=127.0.0.1
+  export GHE_RESTORE_HOST
+
+  # restore should fail on versions older than 3.10
+  ! GHE_TEST_REMOTE_VERSION=3.9.0 ghe-restore -i -v
+  ! GHE_TEST_REMOTE_VERSION=3.7.0 ghe-restore -i -v
+  ! GHE_TEST_REMOTE_VERSION=3.1.0 ghe-restore -i -v
+)
+end_test
+
+begin_test "ghe-restore -i into configured vm from full backup"
+(
+  set -e
+  rm -rf "$GHE_REMOTE_ROOT_DIR"
+  setup_remote_metadata
+
+  # set as configured, enable maintenance mode and create required directories
+  setup_maintenance_mode "configured"
+
+  # set restore host environ var
+  GHE_RESTORE_HOST=127.0.0.1
+  export GHE_RESTORE_HOST
+  # run ghe-restore and write output to file for asserting against
+  if ! GHE_TEST_REMOTE_VERSION=3.10.0 GHE_DEBUG=1 ghe-restore -i -v -f > "$TRASHDIR/restore-out" 2>&1; then
+output_debug_logs_and_fail_test
+  fi
+
+
+  # verify connect to right host
+  grep -q "Connect 127.0.0.1:122 OK" "$TRASHDIR/restore-out"
+
+  # verify stale servers were cleared
+  grep -q "Cleaning up stale nodes ..." "$TRASHDIR/restore-out"
+
+  # Verify all the data we've restored is as expected
+  verify_all_restored_data
+)
+end_test
+
+begin_test "ghe-restore -i fails when the lsn information for the listed files is out of order"
+(
+  set -e
+  rm -rf "$GHE_REMOTE_ROOT_DIR"
+  setup_remote_metadata
+
+  # set as configured, enable maintenance mode and create required directories
+  setup_maintenance_mode "configured"
+
+  # set restore host environ var
+  GHE_RESTORE_HOST=127.0.0.1
+  export GHE_RESTORE_HOST
+
+  inc_1="$GHE_DATA_DIR/2"
+  inc_2="$GHE_DATA_DIR/3"
+
+  # screw up the order of the LSNs in xtrabackup_checkpoints
+  setup_incremental_lsn $inc_1 100 200 incremental
+  setup_incremental_lsn $inc_2 50 50 incremental
+  # run ghe-restore and write output to file for asserting against
+  # we expect failure and need the right output.
+  if GHE_DEBUG=1 ghe-restore -i -v -f > "$TRASHDIR/restore-out" 2>&1; then
+    true
+  fi
+)
+end_test
+
+

test/testlib.sh

@@ -327,6 +327,40 @@ setup_test_data () {
   setup_minio_test_data "$GHE_DATA_DIR"
 }
 
+# Sets up test data for testing incremental restores.
+setup_incremental_restore_data() {
+  local full="$GHE_DATA_DIR/1"
+  local inc_1="$GHE_DATA_DIR/2"
+  local inc_2="$GHE_DATA_DIR/3"
+  # Run the setup_test_data function to create three directories: 1 for full backup and two incremental.
+  # we can use these directories for different types of tests
+  setup_test_data "$full"
+  setup_test_data "$inc_1"
+  setup_test_data "$inc_2" 
+  # Setup the metadata files that track which files are used to track full and incremental files
+  echo "$full" >> "$GHE_DATA_DIR/inc_full_backup"
+  echo -e "$inc_1\n$inc_2" >> "$GHE_DATA_DIR/inc_snapshot_data"
+  # Configure lsn data in xtrabackup_checkpoints for the full backup and the incremental backup
+  setup_incremental_lsn $full 1 100 full
+  setup_incremental_lsn $inc_1 101 200 incremental
+  setup_incremental_lsn $inc_2 201 300 incremental
+}
+
+setup_incremental_lsn() {
+  local loc=$1
+  local start=$2
+  local end=$3
+  local type=$4
+
+cat <<LSN >> "$loc/xtrabackup_checkpoints"
+backup_type = $type
+from_lsn = $start
+to_lsn = $end
+last_lsn = $end
+flushed_lsn = $end
+LSN
+}
+
 setup_incremental_backup_config() {
   ghe-ssh "$GHE_HOSTNAME" -- 'mkdir -p /tmp/lsndir'
   ghe-ssh "$GHE_HOSTNAME" -- 'echo "fake xtrabackup checkpoint" > /tmp/lsndir/xtrabackup_checkpoints'