Merge pull request #487 from github/juruen/fix-auditlog-flag

Fix audit log import to MySQL flag removal for old snapshots and skip rsync'ed indices

Author: juruen

share/github-backup-utils/ghe-backup-es-rsync

@@ -27,6 +27,17 @@ fi
 # Make sure root backup dir exists if this is the first run
 mkdir -p "$GHE_SNAPSHOT_DIR/elasticsearch"
 
+# Create exclude file
+exclude_file="$(mktemp)"
+echo elasticsearch.yml >"$exclude_file"
+
+# Exclude audit log indices when configuration says so and import to MySQL is complete
+# as those indices will be rebuilt from MySQL during a restore
+if [ "$GHE_BACKUP_ES_AUDIT_LOGS" = "no" ] && ghe-ssh "$host" test -e "/data/user/common/audit-log-import/complete"; then
+  ghe_verbose "* Excluding Audit Log indices"
+  ghe-ssh "$host" curl -s 'http://localhost:9201/_cat/indices/audit_log?h=uuid' >>$exclude_file 2>&3
+fi
+
 # Verify that the /data/elasticsearch directory exists.
 if ! ghe-ssh "$host" -- "[ -d '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' ]"; then
   ghe_verbose "* The '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' directory doesn't exist."
@@ -47,15 +58,16 @@ ghe-rsync -avz \
   -e "ghe-ssh -p $(ssh_port_part "$host")" \
   --rsync-path="sudo -u elasticsearch rsync" \
   $link_dest \
-  --exclude='elasticsearch.yml' \
+  --exclude-from="$exclude_file" \
   "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
   "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3
 
-# Set up a trap to re-enable flushing on exit
+# Set up a trap to re-enable flushing on exit and remove temp file
 cleanup () {
   ghe_verbose "* Enabling ES index flushing ..."
   echo '{"index":{"translog.disable_flush":false}}' |
   ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
+  ghe-ssh "$host" rm -rf "$exclude_file"
 }
 trap 'cleanup' EXIT
 trap 'exit $?' INT # ^C always terminate
@@ -72,7 +84,7 @@ ghe-rsync -avz \
   -e "ghe-ssh -p $(ssh_port_part "$host")" \
   --rsync-path="sudo -u elasticsearch rsync" \
   $link_dest \
-  --exclude='elasticsearch.yml' \
+  --exclude-from="$exclude_file" \
   "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
   "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3
 

share/github-backup-utils/ghe-restore-audit-log

@@ -34,6 +34,17 @@ mysql_restored_enabled(){
   test -e "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/audit-log-mysql"
 }
 
+remove_complete_flag(){
+  ghe_verbose "Setting instance(s) as pending for audit log import to MySQL"
+  ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf $GHE_REMOTE_ROOT_DIR/data/user/common/audit-log-import/complete" 1>&3 2>&3
+
+  if $CLUSTER; then
+    if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo rm -rf /data/user/common/audit-log-import/complete" 1>&3 2>&3; then
+      ghe_verbose "Failed to set as pending for audit log import to MySQL all instances in cluster"
+    fi
+  fi
+}
+
 # Use `ghe-backup-mysql-audit-log` to dump the audit entries.
 # If the import to MySQL is complete, add a flag in the snapshot to indicate so.
 restore_mysql(){
@@ -42,8 +53,7 @@ restore_mysql(){
   "${base_path}/ghe-restore-mysql-audit-log" "$GHE_HOSTNAME"
 
   if ! is_import_complete; then
-    ghe_verbose "Audit log import to MySQL is not complete"
-    ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf $GHE_REMOTE_ROOT_DIR/data/user/common/audit-log-import/complete" 1>&3 2>&3
+    remove_complete_flag
     return
   fi
 
@@ -88,6 +98,7 @@ do_restore(){
     restore_mysql
   else
     ghe_verbose "MySQL audit log restore is not enabled"
+    remove_complete_flag
   fi
 
   if es_restore_enabled; then

test/test-ghe-restore.sh

@@ -246,6 +246,32 @@ begin_test "ghe-restore with no pages backup"
 )
 end_test
 
+begin_test "ghe-restore removes audit log import to MySQL flag when is a < 2.17 snapshot"
+(
+  set -e
+
+  rm -rf "$GHE_REMOTE_ROOT_DIR"
+  setup_remote_metadata
+
+  # set as configured, enable maintenance mode and create required directories
+  setup_maintenance_mode "configured"
+
+  flag="$GHE_REMOTE_ROOT_DIR/data/user/common/audit-log-import/complete"
+  mkdir -p "$(dirname $flag)"
+  touch "$flag"
+
+  if ! output=$(ghe-restore -v -f localhost 2>&1); then
+    echo "Error: failed to restore $output" >&2
+    exit 1
+  fi
+
+  ! test -e "$flag" || {
+    echo "Error: the restore process should've removed $flag" >&2
+    exit 1
+  }
+)
+end_test
+
 begin_test "ghe-restore cluster backup to non-cluster appliance"
 (
   set -e