Merge branch 'master' into ce-typo

Author: lildude

.travis.yml

@@ -1,4 +1,5 @@
 ---
+language: minimal
 matrix:
   include:
     - os: osx

backup.config-example

@@ -48,3 +48,10 @@ GHE_NUM_SNAPSHOTS=10
 #
 # WARNING: do not enable this, only useful for debugging/development
 #GHE_BACKUP_FSCK=no
+
+# If set to 'no', Elasticsearch audit log indices will not be backed up.
+# Note that they will still be backed up from MySQL. This will reduce
+# the time and size of the backup process but it will take longer
+# for the audit log entries to be searchable as they need to be reindexed
+# in Elasticsearch.
+#GHE_BACKUP_ES_AUDIT_LOGS=no

bin/ghe-backup

@@ -184,7 +184,7 @@ echo "Backing up Redis database ..."
 ghe-backup-redis > redis.rdb || failures="$failures redis"
 
 echo "Backing up audit log ..."
-ghe-backup-es-audit-log || failures="$failures audit-log"
+ghe-backup-audit-log || failures="$failures audit-log"
 
 echo "Backing up hookshot logs ..."
 ghe-backup-es-hookshot || failures="$failures hookshot"

bin/ghe-restore

@@ -301,8 +301,8 @@ fi
 # Restore exported audit and hookshot logs to 2.12.9 and newer single nodes and
 # all releases of cluster
 if $CLUSTER || [ "$(version $GHE_REMOTE_VERSION)" -ge "$(version 2.12.9)" ]; then
-  echo "Restoring Elasticsearch Audit logs ..."
-  ghe-restore-es-audit-log "$GHE_HOSTNAME" 1>&3
+  echo "Restoring Audit logs ..."
+  ghe-restore-audit-log "$GHE_HOSTNAME" 1>&3
 
   echo "Restoring hookshot logs ..."
   ghe-restore-es-hookshot "$GHE_HOSTNAME" 1>&3

script/cibuild

@@ -6,10 +6,7 @@ set -e
 # place with this version at the beginning of each test and many commands have
 # conditional logic based on the remote version. Running the suite against
 # different major versions ensures we're covering these conditional paths.
-REMOTE_VERSIONS="
-  2.14.0
-  2.16.0
-"
+REMOTE_VERSIONS="2.14.0 2.16.0"
 
 # Enable verbose logging of ssh commands
 export GHE_VERBOSE_SSH=true

script/release

@@ -209,6 +209,10 @@ def bump_version(new_version, min_version = nil, path = 'share/github-backup-uti
     content = File.read('test/testlib.sh')
     new_content = content.gsub(/GHE_TEST_REMOTE_VERSION:=[0-9]\.[0-9]+\.0/,"GHE_TEST_REMOTE_VERSION:=#{new_version}")
     File.open('test/testlib.sh', 'w') {|file| file.puts new_content }
+
+    content = File.read('script/cibuild')
+    new_content = content.gsub(/^REMOTE_VERSIONS=.*$/, "REMOTE_VERSIONS=\"#{min_version} #{new_version}\"")
+    File.open('script/cibuild', 'w') {|file| file.puts new_content }
   end
 end
 
@@ -217,7 +221,7 @@ def push_release_branch(version)
     raise "Creating release branch failed:\n\n#{out}"
   end
 
-  unless (out = `git commit --quiet -m 'Bump version: #{version} [ci skip]' debian/changelog share/github-backup-utils/version bin/ghe-host-check test/testlib.sh`)
+  unless (out = `git commit --quiet -m 'Bump version: #{version} [ci skip]' debian/changelog share/github-backup-utils/version bin/ghe-host-check test/testlib.sh script/cibuild`)
     raise "Error committing changelog and version:\n\n#{out}"
   end
 

share/github-backup-utils/ghe-backup-audit-log

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+#/ Usage: ghe-backup-audit-log
+#/ Take a backup of audit logs.
+#/
+#/ Note: This command typically isn't called directly. It's invoked by
+#/ ghe-backup.
+set -e
+
+base_path="$( dirname "${BASH_SOURCE[0]}" )"
+# Bring in the backup configuration
+# shellcheck source=share/github-backup-utils/ghe-backup-config
+. "${base_path}/ghe-backup-config"
+
+# Setup GHE_REMOTE_XXX variables, host and make sure work dir is created
+setup(){
+  # Perform a host-check and establish GHE_REMOTE_XXX variables.
+  ghe_remote_version_required "$host"
+
+  # Set up remote host and root elastic backup directory based on config
+  host="$GHE_HOSTNAME"
+
+  # Make sure root backup dir exists if this is the first run
+  mkdir -p "$GHE_SNAPSHOT_DIR/audit-log"
+}
+
+# Check whether the MySQL backup should be enabled
+# by checking if the audit-log-import directory exists,
+# this makes it backwards-compatible with old snapshots
+mysql_backup_enabled(){
+  ghe-ssh "$host" test -d "$GHE_REMOTE_DATA_USER_DIR/common/audit-log-import"
+}
+
+# Check whether the MySQL import is complete by checking if
+# /data/user/common/audit-log-import/complete exists
+is_import_complete(){
+  ghe-ssh "$host" test -e "$GHE_REMOTE_DATA_USER_DIR/common/audit-log-import/complete"
+}
+
+# Check whether the MySQL import is disabled by verifying if
+# /data/user/common/audit-log-import/skip exists
+is_import_disabled(){
+  ghe-ssh "$host" test -e "$GHE_REMOTE_DATA_USER_DIR/common/audit-log-import/skip"
+}
+
+# Check whether the instance ships an audit log reconciler, if it doesn't
+# we can't dump audit_entries data, only the schema
+is_reconciler_available(){
+  ghe-ssh "$GHE_HOSTNAME" -- "test -e /usr/local/share/enterprise/ghe-auditlog-repair"
+}
+
+# Check whether we only need to back up the audit_entries schema and
+# ignore the actual data.
+#
+# This is the case when:
+#   - The import to MySQL is not complete
+#   - The import is disabled
+#   - The reconciler tool is not available
+skip_mysql_entries(){
+  if ! is_import_complete; then
+    ghe_verbose "audit log import is not complete"
+    return
+  fi
+
+  if is_import_disabled; then
+    ghe_verbose "audit log import is disabled"
+    return
+  fi
+
+  if ! is_reconciler_available; then
+    ghe_verbose "audit log reconciler is not available"
+    return
+  fi
+
+  return 1
+}
+
+# If the import to MySQL is complete, add a flag in the snapshot to indicate so.
+# And also use `ghe-backup-mysql-audit-log` to dump the audit entries.
+backup_mysql(){
+  if skip_mysql_entries; then
+    ghe_verbose "only backing up audit log table schema"
+    "${base_path}/ghe-backup-mysql-audit-log" --schema-only
+    return
+  fi
+
+  "${base_path}/ghe-backup-mysql-audit-log"
+  touch "$GHE_SNAPSHOT_DIR/audit-log/mysql-import-complete"
+}
+
+# Audit log indices in Elasticsearch are backed up when:
+#
+#   - Import is not complete
+#   - Import is disabled
+#   - Reconciler is not available
+#   - GHE_BACKUP_ES_AUDIT_LOGS is not set to 'no'
+es_backup_enabled(){
+  if skip_mysql_entries; then
+    return
+  fi
+
+  [ -z "$GHE_BACKUP_ES_AUDIT_LOGS" ] || [ "$GHE_BACKUP_ES_AUDIT_LOGS" != "no" ]
+}
+
+# Use ghe-backup-es-audit-log to back up Elasticsearch indices
+backup_es(){
+  "${base_path}/ghe-backup-es-audit-log"
+}
+
+backup(){
+  if mysql_backup_enabled; then
+    ghe_verbose "MySQL audit logs backup is enabled"
+    backup_mysql
+  else
+    ghe_verbose "MySQL audit logs backup is disabled"
+  fi
+
+  if es_backup_enabled; then
+    ghe_verbose "Elasticsearch audit logs backup is enabled"
+    backup_es
+  else
+    ghe_verbose "Elasticsearch audit logs backup is disabled"
+  fi
+}
+
+main(){
+  bm_start "$(basename "$0")"
+  setup
+  backup
+  bm_end "$(basename "$0")"
+}
+
+main

share/github-backup-utils/ghe-backup-es-rsync

@@ -27,6 +27,17 @@ fi
 # Make sure root backup dir exists if this is the first run
 mkdir -p "$GHE_SNAPSHOT_DIR/elasticsearch"
 
+# Create exclude file
+exclude_file="$(mktemp)"
+echo elasticsearch.yml >"$exclude_file"
+
+# Exclude audit log indices when configuration says so and import to MySQL is complete
+# as those indices will be rebuilt from MySQL during a restore
+if [ "$GHE_BACKUP_ES_AUDIT_LOGS" = "no" ] && ghe-ssh "$host" test -e "/data/user/common/audit-log-import/complete"; then
+  ghe_verbose "* Excluding Audit Log indices"
+  ghe-ssh "$host" curl -s 'http://localhost:9201/_cat/indices/audit_log?h=uuid' >>$exclude_file 2>&3
+fi
+
 # Verify that the /data/elasticsearch directory exists.
 if ! ghe-ssh "$host" -- "[ -d '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' ]"; then
   ghe_verbose "* The '$GHE_REMOTE_DATA_USER_DIR/elasticsearch' directory doesn't exist."
@@ -47,15 +58,16 @@ ghe-rsync -avz \
   -e "ghe-ssh -p $(ssh_port_part "$host")" \
   --rsync-path="sudo -u elasticsearch rsync" \
   $link_dest \
-  --exclude='elasticsearch.yml' \
+  --exclude-from="$exclude_file" \
   "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
   "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3
 
-# Set up a trap to re-enable flushing on exit
+# Set up a trap to re-enable flushing on exit and remove temp file
 cleanup () {
   ghe_verbose "* Enabling ES index flushing ..."
   echo '{"index":{"translog.disable_flush":false}}' |
   ghe-ssh "$host" -- curl -s -XPUT "localhost:9200/_settings" -d @- >/dev/null
+  ghe-ssh "$host" rm -rf "$exclude_file"
 }
 trap 'cleanup' EXIT
 trap 'exit $?' INT # ^C always terminate
@@ -72,7 +84,7 @@ ghe-rsync -avz \
   -e "ghe-ssh -p $(ssh_port_part "$host")" \
   --rsync-path="sudo -u elasticsearch rsync" \
   $link_dest \
-  --exclude='elasticsearch.yml' \
+  --exclude-from="$exclude_file" \
   "$(ssh_host_part "$host"):$GHE_REMOTE_DATA_USER_DIR/elasticsearch/" \
   "$GHE_SNAPSHOT_DIR/elasticsearch" 1>&3