Merge pull request #551 from github/oakeyc/update-binary

Oakeyc/update binary

Author: jianghao0718

.travis.yml

@@ -10,6 +10,7 @@ matrix:
         - brew install shellcheck
         - brew install jq
         - brew install coreutils
+        - brew install pigz
       script: make test
     - os: linux
       dist: trusty
@@ -26,4 +27,5 @@ matrix:
             - fakeroot
             - jq
             - coreutils
+            - pigz
       script: debuild -uc -us

bin/ghe-backup

@@ -81,6 +81,25 @@ if [ "$(ls -il dest1/testfile | awk '{ print $1 }')" != "$(ls -il dest2/testfile
 fi
 rm -rf src dest1 dest2
 
+# if we should use gitbackups to backup repositories
+should_use_gitbackups_for_repositories(){
+  ghe-ssh "$GHE_HOSTNAME" ghe-config --true "app.github.gitbackups"
+}
+
+# check that the appliance supports using gitbackups for repositories
+can_use_gitbackups_for_repositories(){
+  ghe-ssh "$GHE_HOSTNAME" test -e /data/github/current/bin/ghe-backup-repositories
+}
+
+# Exit early if the appliance is missing script to backup repositories using gitbackups
+if should_use_gitbackups_for_repositories; then
+  if ! can_use_gitbackups_for_repositories; then
+    echo "Error: Configuration setting 'app.github.gitbackups' is enabled but this version of GHES cannot use gitbackups to back up repositories via 'ghe-backup'."
+    echo "Disable configuration setting 'app.github.gitbackups' and re-run 'ghe-backup' to use rsync."
+    exit 1
+  fi
+fi
+
 # To prevent multiple backup runs happening at the same time, we create a
 # in-progress file with the timestamp and pid of the backup process,
 # giving us a form of locking.
@@ -174,10 +193,23 @@ ghe-ssh "$GHE_HOSTNAME" -- 'ghe-export-ssh-host-keys' > ssh-host-keys.tar ||
 failures="$failures ssh-host-keys"
 bm_end "ghe-export-ssh-host-keys"
 
+# if we are going to take a binary backup
+is_binary_backup(){
+  ghe-ssh "$GHE_HOSTNAME" ghe-config --true "mysql.backup.binary"
+}
+
 echo "Backing up MySQL database ..."
 bm_start "ghe-export-mysql"
-echo 'set -o pipefail; ghe-export-mysql | gzip' |
+export_command="ghe-export-mysql"
+if ! is_binary_backup; then
+  # binary backup is already compressed
+  export_command+=" | pigz"
+fi
+echo "set -o pipefail; $export_command" |
 ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > mysql.sql.gz || failures="$failures mysql"
+if is_binary_backup; then
+  echo "NO_ADDITIONAL_COMPRESSION" > mysql-binary-backup-sentinel
+fi
 bm_end "ghe-export-mysql"
 
 echo "Backing up Redis database ..."

share/github-backup-utils/ghe-restore-mysql

@@ -28,18 +28,67 @@ ghe_remote_version_required "$GHE_HOSTNAME"
 # The directory holding the snapshot to restore
 snapshot_dir="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"
 
+# Check if the backup is binary by looking up the sentinel file 
+is_binary_backup(){
+  test -f $snapshot_dir/mysql-binary-backup-sentinel 
+}
+
+# if mysql.backup.binary feature flag is on
+is_binary_backup_feature_on(){
+  ghe-ssh "$GHE_HOSTNAME" ghe-config --true "mysql.backup.binary"
+}
+
+if ghe-ssh "$GHE_HOSTNAME" test -f /etc/github/cluster ; then
+  ghe_mysql_master=$(ghe-ssh "$GHE_HOSTNAME" ghe-config "cluster.mysql-master")
+  if [ -z $ghe_mysql_master ]; then
+    ghe_mysql_master=$GHE_HOSTNAME
+  else
+    port=$(ssh_port_part "$GHE_HOSTNAME")
+    ghe_mysql_master=$ghe_mysql_master${port:+:$port}
+  fi
+else
+  ghe_mysql_master=$GHE_HOSTNAME
+fi
+
+if is_binary_backup_feature_on; then
+  # Feature "mysql.backup.binary" is on, which means new backup scripts are available
+  if is_binary_backup; then
+    # Check if the decompress needed by looking into the sentinel file 
+    # In 2.19.5 we compress the binary backup twice
+    if [ "$(cat $snapshot_dir/mysql-binary-backup-sentinel)" = "NO_ADDITIONAL_COMPRESSION" ]; then
+      IMPORT_MYSQL=ghe-import-mysql-xtrabackup
+      GHE_RESTORE_HOST=$ghe_mysql_master
+    else
+      IMPORT_MYSQL="unpigz | ghe-import-mysql-xtrabackup"
+      GHE_RESTORE_HOST=$ghe_mysql_master
+    fi
+  else
+    IMPORT_MYSQL="unpigz | ghe-import-mysql-mysqldump"
+    GHE_RESTORE_HOST=$GHE_HOSTNAME
+  fi
+else
+  # We do not allow to restore binary backup without "mysql.backup.binary" set 
+  if is_binary_backup; then
+    echo "To restore from a binary backup, you have to set ghe-config \"mysql.backup.binary\" to true" >&2  
+    exit 2
+  else
+    # legacy mode
+    IMPORT_MYSQL="unpigz | ghe-import-mysql-mysqldump"
+    GHE_RESTORE_HOST=$GHE_HOSTNAME
+  fi
+fi
+
 cleanup() {
-  ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz"
+  ghe-ssh "$GHE_RESTORE_HOST" -- "sudo rm -rf $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz"
 }
 trap 'cleanup' INT TERM EXIT
 
-ghe-ssh "$GHE_HOSTNAME" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/tmp'" 1>&3
+ghe-ssh "$GHE_RESTORE_HOST" -- "sudo mkdir -p '$GHE_REMOTE_DATA_USER_DIR/tmp'" 1>&3
 
 # Transfer MySQL data from the snapshot to the GitHub instance.
-cat $snapshot_dir/mysql.sql.gz | ghe-ssh "$GHE_HOSTNAME" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz >/dev/null 2>&1"
+cat $snapshot_dir/mysql.sql.gz | ghe-ssh "$GHE_RESTORE_HOST" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz >/dev/null 2>&1"
 
 # Import the database
-echo "gunzip -cd $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | ghe-import-mysql" | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash 1>&3
-
+echo "cat $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | $IMPORT_MYSQL" | ghe-ssh "$GHE_RESTORE_HOST" -- /bin/bash 1>&3
 
 bm_end "$(basename $0)"

share/github-backup-utils/version

@@ -1 +1 @@
-2.19.2
+2.19.3

test/bin/ghe-import-mysql-mysqldump

@@ -0,0 +1 @@
+ghe-fake-import-command

test/bin/ghe-import-mysql-xtrabackup

@@ -0,0 +1 @@
+ghe-fake-import-command