Merge pull request #517 from github/juruen/audit-log-mysql-drop

Prepare for audit log MySQL backend deprecation

Author: juruen

backup.config-example

@@ -48,10 +48,3 @@ GHE_NUM_SNAPSHOTS=10
 #
 # WARNING: do not enable this, only useful for debugging/development
 #GHE_BACKUP_FSCK=no
-
-# If set to 'no', Elasticsearch audit log indices will not be backed up.
-# Note that they will still be backed up from MySQL. This will reduce
-# the time and size of the backup process but it will take longer
-# for the audit log entries to be searchable as they need to be reindexed
-# in Elasticsearch.
-#GHE_BACKUP_ES_AUDIT_LOGS=no

share/github-backup-utils/ghe-backup-audit-log

@@ -23,82 +23,20 @@ setup(){
   mkdir -p "$GHE_SNAPSHOT_DIR/audit-log"
 }
 
-# Check whether the MySQL backup should be enabled
-# by checking if the audit-log-import directory exists,
-# this makes it backwards-compatible with old snapshots
-mysql_backup_enabled(){
-  ghe-ssh "$host" test -d "$GHE_REMOTE_DATA_USER_DIR/common/audit-log-import"
+# Check whether the flag to skip the MySQL audit_entries table truncation
+# exists.
+is_skip_truncate_enabled(){
+  ghe-ssh "$host" test -e "$GHE_REMOTE_DATA_USER_DIR/common/audit-log-import/skip_truncate"
 }
 
-# Check whether the MySQL import is complete by checking if
-# /data/user/common/audit-log-import/complete exists
-is_import_complete(){
-  ghe-ssh "$host" test -e "$GHE_REMOTE_DATA_USER_DIR/common/audit-log-import/complete"
-}
-
-# Check whether the MySQL import is disabled by verifying if
-# /data/user/common/audit-log-import/skip exists
-is_import_disabled(){
-  ghe-ssh "$host" test -e "$GHE_REMOTE_DATA_USER_DIR/common/audit-log-import/skip"
-}
-
-# Check whether the instance ships an audit log reconciler, if it doesn't
-# we can't dump audit_entries data, only the schema
-is_reconciler_available(){
-  ghe-ssh "$GHE_HOSTNAME" -- "test -e /usr/local/share/enterprise/ghe-auditlog-repair"
-}
-
-# Check whether we only need to back up the audit_entries schema and
-# ignore the actual data.
-#
-# This is the case when:
-#   - The import to MySQL is not complete
-#   - The import is disabled
-#   - The reconciler tool is not available
-skip_mysql_entries(){
-  if ! is_import_complete; then
-    ghe_verbose "audit log import is not complete"
-    return
-  fi
-
-  if is_import_disabled; then
-    ghe_verbose "audit log import is disabled"
-    return
-  fi
-
-  if ! is_reconciler_available; then
-    ghe_verbose "audit log reconciler is not available"
-    return
-  fi
-
-  return 1
-}
-
-# If the import to MySQL is complete, add a flag in the snapshot to indicate so.
-# And also use `ghe-backup-mysql-audit-log` to dump the audit entries.
 backup_mysql(){
-  if skip_mysql_entries; then
-    ghe_verbose "only backing up audit log table schema"
+  if is_skip_truncate_enabled; then
+    # As skip_truncate exists, we need to also backup the audit entries
+    # in MySQL because Elasticsearch may not be fully synced.
+    "${base_path}/ghe-backup-mysql-audit-log"
+  else
     "${base_path}/ghe-backup-mysql-audit-log" --schema-only
-    return
-  fi
-
-  "${base_path}/ghe-backup-mysql-audit-log"
-  touch "$GHE_SNAPSHOT_DIR/audit-log/mysql-import-complete"
-}
-
-# Audit log indices in Elasticsearch are backed up when:
-#
-#   - Import is not complete
-#   - Import is disabled
-#   - Reconciler is not available
-#   - GHE_BACKUP_ES_AUDIT_LOGS is not set to 'no'
-es_backup_enabled(){
-  if skip_mysql_entries; then
-    return
   fi
-
-  [ -z "$GHE_BACKUP_ES_AUDIT_LOGS" ] || [ "$GHE_BACKUP_ES_AUDIT_LOGS" != "no" ]
 }
 
 # Use ghe-backup-es-audit-log to back up Elasticsearch indices
@@ -107,19 +45,8 @@ backup_es(){
 }
 
 backup(){
-  if mysql_backup_enabled; then
-    ghe_verbose "MySQL audit logs backup is enabled"
     backup_mysql
-  else
-    ghe_verbose "MySQL audit logs backup is disabled"
-  fi
-
-  if es_backup_enabled; then
-    ghe_verbose "Elasticsearch audit logs backup is enabled"
     backup_es
-  else
-    ghe_verbose "Elasticsearch audit logs backup is disabled"
-  fi
 }
 
 main(){

share/github-backup-utils/ghe-backup-es-rsync

@@ -27,17 +27,6 @@ fi
 # Make sure root backup dir exists if this is the first run
 mkdir -p "$GHE_SNAPSHOT_DIR/elasticsearch"
 
-# Create exclude file
-exclude_file="$(mktemp)"
-echo elasticsearch.yml >"$exclude_file"
-
-# Exclude audit log indices when configuration says so and import to MySQL is complete
-# as those indices will be rebuilt from MySQL during a restore
-if [ "$GHE_BACKUP_ES_AUDIT_LOGS" = "no" ] && ghe-ssh "$host" test -e "/data/user/common/audit-log-import/complete"; then
-  ghe_verbose "* Excluding Audit Log indices"
-  ghe-ssh "$host" curl -s 'http://localhost:9201/_cat/indices/audit_log?h=uuid' >>$exclude_file 2>&3
-fi
-
 # Verify that the /data/elasticsearch directory exists.
 if ! ghe-ssh "$host" -- "[ -d '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' ]"; then
   ghe_verbose "* The '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' directory doesn't exist."
@@ -58,7 +47,6 @@ ghe-rsync -avz \
   -e "ghe-ssh -p $(ssh_port_part "$host")" \
   --rsync-path="sudo -u elasticsearch rsync" \
   $link_dest \
-  --exclude-from="$exclude_file" \
   "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
   "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3
 
@@ -67,7 +55,6 @@ cleanup () {
   ghe_verbose "* Enabling ES index flushing ..."
   echo '{"index":{"translog.disable_flush":false}}' |
   ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
-  rm -rf "$exclude_file"
 }
 trap 'cleanup' EXIT
 trap 'exit $?' INT # ^C always terminate
@@ -84,7 +71,6 @@ ghe-rsync -avz \
   -e "ghe-ssh -p $(ssh_port_part "$host")" \
   --rsync-path="sudo -u elasticsearch rsync" \
   $link_dest \
-  --exclude-from="$exclude_file" \
   "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
   "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3
 

share/github-backup-utils/ghe-restore-audit-log

@@ -22,55 +22,70 @@ setup(){
   ghe_remote_version_required "$GHE_HOSTNAME"
 }
 
-# Check whether the snapshot comes from an instance
-# where the MySQL import was complete
-is_import_complete(){
-  test -e "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/audit-log/mysql-import-complete"
+# Check whether the snapshot contains audit logs that
+# were taken from Elasticsearch
+es_data_available(){
+  ls -A "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"/audit-log/*.size >/dev/null 2>&1
 }
 
-# Check whether the snapshot was taken on an instance
-# where MySQL audit logs were enabled
-mysql_restored_enabled(){
-  test -e "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT/audit-log-mysql"
+# Check whether the snapshot contains audit logs that
+# were taken from MySQL
+mysql_dump_available(){
+  ls -A "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"/audit-log-mysql/20*.gz >/dev/null 2>&1
 }
 
-remove_complete_flag(){
-  ghe_verbose "Setting instance(s) as pending for audit log import to MySQL"
-  ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf $GHE_REMOTE_ROOT_DIR/data/user/common/audit-log-import/complete" 1>&3 2>&3
+# Check whether the snapshot contains the audit log table schema
+mysql_table_schema_available(){
+  ls -A "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"/audit-log-mysql/schema.gz >/dev/null 2>&1
+}
 
-  if $CLUSTER; then
-    if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo rm -rf /data/user/common/audit-log-import/complete" 1>&3 2>&3; then
-      ghe_verbose "Failed to set as pending for audit log import to MySQL all instances in cluster"
-    fi
-  fi
+# Check whether the remote host is running a version where the MySQL backend
+# is supported, i.e: < 2.19
+is_mysql_supported(){
+   [ "$(version "$GHE_REMOTE_VERSION")" -lt "$(version 2.19.0)" ]
 }
 
-# Use `ghe-backup-mysql-audit-log` to dump the audit entries.
-# If the import to MySQL is complete, add a flag in the snapshot to indicate so.
-restore_mysql(){
-  ghe_verbose "Restoring MySQL audit logs ..."
+# Helper function to set remote flags in `/data/user/common/audit-log-import`
+# if it's supported, i.e: directory exists.
+set_remote_flag(){
+  local flag=$1
+  local msg=$2
 
-  "${base_path}/ghe-restore-mysql-audit-log" "$GHE_HOSTNAME"
+  local dir="/data/user/common/audit-log-import"
 
-  if ! is_import_complete; then
-    remove_complete_flag
+  if ! ghe-ssh "$GHE_HOSTNAME" -- "sudo test -d $GHE_REMOTE_ROOT_DIR/$dir" 1>&3 2>&3; then
+    ghe_verbose "Remote version doesn't support audit log import, skipping '$msg'"
     return
   fi
 
-  ghe_verbose "Audit log import to MySQL is complete"
-  ghe-ssh "$GHE_HOSTNAME" -- "sudo touch $GHE_REMOTE_ROOT_DIR/data/user/common/audit-log-import/complete" 
-}
+  ghe_verbose "$msg"
+  ghe-ssh "$GHE_HOSTNAME" -- "sudo touch $GHE_REMOTE_ROOT_DIR/$dir/$flag" 1>&3 2>&3
 
-# Audit log indices in Elasticsearch are restored when:
-#
-#   - import to MySQL is not complete
-#   - GHE_BACKUP_ES_AUDIT_LOGS is not set to 'no'
-es_restore_enabled(){
-  if ! is_import_complete; then
-    return
+  if $CLUSTER; then
+    if ! ghe-ssh "$GHE_HOSTNAME" -- "ghe-cluster-each -- sudo touch $dir/$flag" 1>&3 2>&3; then
+      ghe_verbose "Failed to $msg in all instances in cluster"
+    fi
   fi
+}
+
+# Add flag to not trigger transitions from MySQL to Elasticsearch
+set_skip_transition_flag(){
+  set_remote_flag "skip" "Add flag to skip audit log import to MySQL"
+}
 
-  [ -z "$GHE_BACKUP_ES_AUDIT_LOGS" ] || [ "$GHE_BACKUP_ES_AUDIT_LOGS" != "no" ]
+# Add flag to not trigger the truncation of the MySQL audit log table
+set_skip_truncate_flag(){
+  set_remote_flag "skip_truncate" "Add flag to skip truncating audit log table in MySQL"
+}
+
+# Use `ghe-backup-mysql-audit-log` to dump the audit entries.
+# If the import to MySQL is complete, add a flag in the snapshot to indicate so.
+restore_mysql(){
+  local only_schema=$1
+
+  ghe_verbose "Restoring MySQL audit logs ..."
+
+  "${base_path}/ghe-restore-mysql-audit-log" "$GHE_HOSTNAME" "$only_schema"
 }
 
 # Use ghe-restore-es-audit-log to restore Elasticsearch indices
@@ -80,37 +95,44 @@ restore_es(){
   "${base_path}/ghe-restore-es-audit-log" "$GHE_HOSTNAME"
 }
 
-# Whether or not we should trigger a reindex from MySQL into Elasticsearch
-should_start_reindex(){
-  if [ -z "$GHE_BACKUP_ES_AUDIT_LOGS" ] || [ "$GHE_BACKUP_ES_AUDIT_LOGS" != "no" ]; then
-    ghe_verbose "GHE_BACKUP_ES_AUDIT_LOGS is not set to 'no'"
-    return 1
-  fi
-
-  if ! ghe-ssh "$GHE_HOSTNAME" -- "test -e /usr/local/share/enterprise/ghe-auditlog-repair"; then
-    ghe_verbose "ghe-auditlog-repiar doesn't exist"
-    return 1
-  fi
-}
-
 do_restore(){
-  if mysql_restored_enabled; then
-    restore_mysql
-  else
-    ghe_verbose "MySQL audit log restore is not enabled"
-    remove_complete_flag
+  if is_mysql_supported; then
+    set_skip_transition_flag
   fi
 
-  if es_restore_enabled; then
+  # ES data is available, restore it along
+  # with the table schema
+  if es_data_available; then
+    ghe_verbose "Elasticsearch data is available"
+
     restore_es
+    restore_mysql --only-schema
     return
   fi
 
-  ghe_verbose "Elasticsearch audit log restore is not enabled"
+  # Only MySQL data is available, restore it
+  # and trigger a reindex
+  if mysql_dump_available; then
+    ghe_verbose "Only MySQL data is available"
+
+    restore_mysql
+
+    if ! is_mysql_supported; then
+      ghe_verbose "Add flag to skip MySQL audit log table truncation"
+      set_skip_truncate_flag
+    fi
 
-  if should_start_reindex; then
     ghe_verbose "Starting audit log reindex from MySQL to Elasticsearch"
     ghe-ssh "$GHE_HOSTNAME" -- "sudo systemctl --no-block restart auditlog-repair";
+    return
+  fi
+
+  if mysql_table_schema_available; then
+    # Only the table schema is available, restore it
+    ghe_verbose "Only audit_entries schema is available"
+    restore_mysql --only-schema
+  else
+    ghe_verbose "MySQL table schema is not available"
   fi
 }
 

share/github-backup-utils/ghe-restore-mysql-audit-log

@@ -16,6 +16,9 @@ base_path="$( dirname "${BASH_SOURCE[0]}" )"
 
 GHE_HOSTNAME="$1"
 
+# Whether we just need to imprt the table schema and no data
+only_schema="$2"
+
 # Setup GHE_REMOTE_XXX variables, snapshot_dir,
 # remote_dir, remote_dump and skip_prepare
 setup(){
@@ -202,6 +205,10 @@ restore(){
   fi
 
   restore_schema
+  if [ -n "$only_schema" ]; then
+    ghe_verbose "only table schema was imported"
+    return
+  fi
 
   IFS=$'\n'
   for month in $(notsynced_meta); do

test/test-ghe-restore.sh

@@ -280,32 +280,6 @@ begin_test "ghe-restore with no pages backup"
 )
 end_test
 
-begin_test "ghe-restore removes audit log import to MySQL flag when is a < 2.17 snapshot"
-(
-  set -e
-
-  rm -rf "$GHE_REMOTE_ROOT_DIR"
-  setup_remote_metadata
-
-  # set as configured, enable maintenance mode and create required directories
-  setup_maintenance_mode "configured"
-
-  flag="$GHE_REMOTE_ROOT_DIR/data/user/common/audit-log-import/complete"
-  mkdir -p "$(dirname $flag)"
-  touch "$flag"
-
-  if ! output=$(ghe-restore -v -f localhost 2>&1); then
-    echo "Error: failed to restore $output" >&2
-    exit 1
-  fi
-
-  ! test -e "$flag" || {
-    echo "Error: the restore process should've removed $flag" >&2
-    exit 1
-  }
-)
-end_test
-
 begin_test "ghe-restore cluster backup to non-cluster appliance"
 (
   set -e

test/testlib.sh

@@ -253,7 +253,9 @@ setup_test_data () {
   mkdir -p "$loc/audit-log/"
   cd "$loc/audit-log/"
   echo "fake audit log last yr last mth" | gzip > audit_log-1-$last_yr-$last_mth-1.gz
+  echo "1" > audit_log-1-$last_yr-$last_mth-1.size
   echo "fake audit log this yr this mth" | gzip > audit_log-1-$this_yr-$this_mth-1.gz
+  echo "1" > audit_log-1-$this_yr-$this_mth-1.size
 
   # Create hookshot logs
   mkdir -p "$loc/hookshot/"