Merge branch 'master' of github.com:github/backup-utils

Author: juruen

backup.config-example

@@ -48,10 +48,3 @@ GHE_NUM_SNAPSHOTS=10
 #
 # WARNING: do not enable this, only useful for debugging/development
 #GHE_BACKUP_FSCK=no
-
-# If set to 'no', Elasticsearch audit log indices will not be backed up.
-# Note that they will still be backed up from MySQL. This will reduce
-# the time and size of the backup process but it will take longer
-# for the audit log entries to be searchable as they need to be reindexed
-# in Elasticsearch.
-#GHE_BACKUP_ES_AUDIT_LOGS=no

share/github-backup-utils/ghe-backup-es-rsync

@@ -27,17 +27,6 @@ fi
 # Make sure root backup dir exists if this is the first run
 mkdir -p "$GHE_SNAPSHOT_DIR/elasticsearch"
 
-# Create exclude file
-exclude_file="$(mktemp)"
-echo elasticsearch.yml >"$exclude_file"
-
-# Exclude audit log indices when configuration says so and import to MySQL is complete
-# as those indices will be rebuilt from MySQL during a restore
-if [ "$GHE_BACKUP_ES_AUDIT_LOGS" = "no" ] && ghe-ssh "$host" test -e "/data/user/common/audit-log-import/complete"; then
-  ghe_verbose "* Excluding Audit Log indices"
-  ghe-ssh "$host" curl -s 'http://localhost:9201/_cat/indices/audit_log?h=uuid' >>$exclude_file 2>&3
-fi
-
 # Verify that the /data/elasticsearch directory exists.
 if ! ghe-ssh "$host" -- "[ -d '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' ]"; then
   ghe_verbose "* The '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' directory doesn't exist."
@@ -58,7 +47,6 @@ ghe-rsync -avz \
   -e "ghe-ssh -p $(ssh_port_part "$host")" \
   --rsync-path="sudo -u elasticsearch rsync" \
   $link_dest \
-  --exclude-from="$exclude_file" \
   "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
   "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3
 
@@ -67,7 +55,6 @@ cleanup () {
   ghe_verbose "* Enabling ES index flushing ..."
   echo '{"index":{"translog.disable_flush":false}}' |
   ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
-  rm -rf "$exclude_file"
 }
 trap 'cleanup' EXIT
 trap 'exit $?' INT # ^C always terminate
@@ -84,7 +71,6 @@ ghe-rsync -avz \
   -e "ghe-ssh -p $(ssh_port_part "$host")" \
   --rsync-path="sudo -u elasticsearch rsync" \
   $link_dest \
-  --exclude-from="$exclude_file" \
   "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
   "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3
 

share/github-backup-utils/ghe-restore-audit-log

@@ -4,7 +4,7 @@
 #/
 #/ Note: This command typically isn't called directly. It's invoked by
 #/ ghe-backup.
-set -ex
+set -e
 
 # Bring in the backup configuration
 base_path="$( dirname "${BASH_SOURCE[0]}" )"
@@ -34,30 +34,46 @@ mysql_dump_available(){
   ls -A "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"/audit-log-mysql/20*.gz >/dev/null 2>&1
 }
 
+# Check whether the snapshot contains the audit log table schema
+mysql_table_schema_available(){
+  ls -A "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"/audit-log-mysql/schema.gz >/dev/null 2>&1
+}
+
 # Check whether the remote host is running a version where the MySQL backend
 # is supported, i.e: < 2.19
 is_mysql_supported(){
-   [ "$(version $GHE_REMOTE_VERSION)" -lt "$(version 2.19.0)" ]
+   [ "$(version "$GHE_REMOTE_VERSION")" -lt "$(version 2.19.0)" ]
 }
 
+# Helper function to set remote flags in `/data/user/common/audit-log-import`
+# if it's supported, i.e: directory exists.
 set_remote_flag(){
   local flag=$1
   local msg=$2
 
-  ghe_verbose "$2"
-  ghe-ssh "$GHE_HOSTNAME" -- "sudo touch $GHE_REMOTE_ROOT_DIR/data/user/common/audit-log-import/$flag" 1>&3 2>&3
+  local dir="/data/user/common/audit-log-import"
+
+  if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo test -d $GHE_REMOTE_ROOT_DIR/$dir" 1>&3 2>&3; then
+    ghe_verbose "Remote version doesn't support audit log import, skipping '$msg'"
+    return
+  fi
+
+  ghe_verbose "$msg"
+  ghe-ssh "$GHE_HOSTNAME" -- "sudo touch $GHE_REMOTE_ROOT_DIR/$dir/$flag" 1>&3 2>&3
 
   if $CLUSTER; then
-    if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo touch /data/user/common/audit-log-import/$flag" 1>&3 2>&3; then
+    if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo touch $dir/$flag" 1>&3 2>&3; then
       ghe_verbose "Failed to $msg in all instances in cluster"
     fi
   fi
 }
 
+# Add flag to not trigger transitions from MySQL to Elasticsearch
 set_skip_transition_flag(){
   set_remote_flag "skip" "Add flag to skip audit log import to MySQL"
 }
 
+# Add flag to not trigger the truncation of the MySQL audit log table
 set_skip_truncate_flag(){
   set_remote_flag "skip_truncate" "Add flag to skip truncating audit log table in MySQL"
 }
@@ -111,9 +127,13 @@ do_restore(){
     return
   fi
 
-  # Only the table schema is available, restore it
-  ghe_verbose "Only audit_entries schema is available"
-  restore_mysql --only-schema
+  if mysql_table_schema_available; then
+    # Only the table schema is available, restore it
+    ghe_verbose "Only audit_entries schema is available"
+    restore_mysql --only-schema
+  else
+    ghe_verbose "MySQL table schema is not available"
+  fi
 }
 
 main(){

share/github-backup-utils/ghe-restore-es-audit-log

@@ -4,7 +4,7 @@
 #/
 #/ Note: This command typically isn't called directly. It's invoked by
 #/ ghe-restore.
-set -ex
+set -e
 
 # Bring in the backup configuration
 # shellcheck source=share/github-backup-utils/ghe-backup-config

share/github-backup-utils/ghe-restore-mysql-audit-log

@@ -4,7 +4,7 @@
 #/
 #/ Note: This command typically isn't called directly. It's invoked by
 #/ ghe-restore-audit-log
-set -ex
+set -e
 
 # Bring in the backup configuration
 base_path="$( dirname "${BASH_SOURCE[0]}" )"

test/test-ghe-restore.sh

@@ -280,32 +280,6 @@ begin_test "ghe-restore with no pages backup"
 )
 end_test
 
-begin_test "ghe-restore removes audit log import to MySQL flag when is a < 2.17 snapshot"
-(
-  set -e
-
-  rm -rf "$GHE_REMOTE_ROOT_DIR"
-  setup_remote_metadata
-
-  # set as configured, enable maintenance mode and create required directories
-  setup_maintenance_mode "configured"
-
-  flag="$GHE_REMOTE_ROOT_DIR/data/user/common/audit-log-import/complete"
-  mkdir -p "$(dirname $flag)"
-  touch "$flag"
-
-  if ! output=$(ghe-restore -v -f localhost 2>&1); then
-    echo "Error: failed to restore $output" >&2
-    exit 1
-  fi
-
-  ! test -e "$flag" || {
-    echo "Error: the restore process should've removed $flag" >&2
-    exit 1
-  }
-)
-end_test
-
 begin_test "ghe-restore cluster backup to non-cluster appliance"
 (
   set -e

test/testlib.sh

@@ -253,7 +253,9 @@ setup_test_data () {
   mkdir -p "$loc/audit-log/"
   cd "$loc/audit-log/"
   echo "fake audit log last yr last mth" | gzip > audit_log-1-$last_yr-$last_mth-1.gz
+  echo "1" > audit_log-1-$last_yr-$last_mth-1.size
   echo "fake audit log this yr this mth" | gzip > audit_log-1-$this_yr-$this_mth-1.gz
+  echo "1" > audit_log-1-$this_yr-$this_mth-1.size
 
   # Create hookshot logs
   mkdir -p "$loc/hookshot/"