Merge pull request #165 from github/byodb-client-hooks

Run hooks for mysql backup/restore when external mysql database is configured

Author: omgitsads

backup.config-example

@@ -77,3 +77,11 @@ GHE_NUM_SNAPSHOTS=10
 #
 # WARNING: this feature is in beta.
 #GHE_PARALLEL_MAX_LOAD=50
+
+# When running an external mysql database, run this script to trigger a MySQL backup
+# rather than attempting to backup via backup-utils directly.
+#EXTERNAL_DATABASE_BACKUP_SCRIPT="/bin/false"
+
+# When running an external mysql database, run this script to trigger a MySQL restore
+# rather than attempting to backup via backup-utils directly.
+#EXTERNAL_DATABASE_RESTORE_SCRIPT="/bin/false"

bin/ghe-backup

@@ -180,11 +180,6 @@ ghe-ssh "$GHE_HOSTNAME" -- 'ghe-export-ssh-host-keys' > ssh-host-keys.tar ||
 failures="$failures ssh-host-keys"
 bm_end "ghe-export-ssh-host-keys"
 
-if is_binary_backup_feature_on; then
-  echo "Backing up MySQL database using binary backup strategy ..."
-else
-  echo "Backing up MySQL database using logical backup strategy ..."
-fi
 ghe-backup-mysql || failures="$failures mysql"
 
 commands=("

bin/ghe-restore

@@ -255,16 +255,21 @@ if [ -s "$GHE_RESTORE_SNAPSHOT_PATH/uuid" ] && ! $CLUSTER; then
   ghe-ssh "$GHE_HOSTNAME" -- "sudo rm -rf /data/user/consul/raft"
   fi
 
-if is_binary_backup_feature_on; then
-  appliance_strategy="binary"
-else
-  appliance_strategy="logical"
-fi
+if is_service_external 'mysql' "$GHE_RESTORE_SNAPSHOT_PATH/settings.json"; then
+   appliance_strategy="external"
+   backup_snapshot_strategy="external"
+else 
+  if is_binary_backup_feature_on; then
+    appliance_strategy="binary"
+  else
+    appliance_strategy="logical"
+  fi
 
-if is_binary_backup "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"; then
-  backup_snapshot_strategy="binary"
-else
-  backup_snapshot_strategy="logical"
+  if is_binary_backup "$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"; then
+    backup_snapshot_strategy="binary"
+  else
+    backup_snapshot_strategy="logical"
+  fi
 fi
 
 echo "Restoring MySQL database from ${backup_snapshot_strategy} backup snapshot on an appliance configured for ${appliance_strategy} backups ..."

share/github-backup-utils/ghe-backup-config

@@ -152,6 +152,7 @@ fi
 if [ ${GHE_DATA_DIR:0:1} != "/" ]; then
   GHE_DATA_DIR="$( cd "$GHE_BACKUP_ROOT" && readlink -m "$GHE_DATA_DIR" 2> /dev/null || echo "$GHE_BACKUP_ROOT/$GHE_DATA_DIR" )"
 fi
+export GHE_DATA_DIR
 
 # Assign the Release File path if it hasn't been provided (eg: by test suite)
 : ${GHE_RELEASE_FILE:="/etc/github/enterprise-release"}
@@ -389,4 +390,22 @@ is_binary_backup_feature_on(){
 # Check if the backup is binary by looking up the sentinel file
 is_binary_backup(){
   test -f "$1/mysql-binary-backup-sentinel"
-}
+}
+
+is_service_external(){
+  service=$1
+  config_file=$2
+  case $service in
+    "mysql")
+      if [ -n "$config_file" ]; then
+        enabled=$(GIT_CONFIG="$config_file" git config mysql.external.enabled)
+        [ "$enabled" == "true" ];
+      else
+        ghe-ssh "$GHE_HOSTNAME" ghe-config --true "mysql.external.enabled"
+      fi
+      ;;
+    *)
+      return 1
+      ;;
+    esac
+}

share/github-backup-utils/ghe-backup-mysql

@@ -15,6 +15,26 @@ bm_start "$(basename $0)"
 # Perform a host-check and establish the remote version in GHE_REMOTE_VERSION.
 ghe_remote_version_required "$GHE_HOSTNAME"
 
+if is_service_external 'mysql'; then
+  echo "Backing up external MySQL database using customer-provided script..."
+  if [ -n "$EXTERNAL_DATABASE_BACKUP_SCRIPT" ]; then
+    $EXTERNAL_DATABASE_BACKUP_SCRIPT
+  else
+    echo "Error: EXTERNAL_DATABASE_BACKUP_SCRIPT is not configured. Please configure in backup.config."
+    exit 1
+  fi
+  # Finsh the benchmark and exit early. If the script returns a non-zero 
+  # exit code, it will be caught by`set -e`
+  bm_end "$(basename $0)"
+  exit 0
+fi
+
+if is_binary_backup_feature_on; then
+  echo "Backing up MySQL database using binary backup strategy ..."
+else
+  echo "Backing up MySQL database using logical backup strategy ..."
+fi
+
 export_command="ghe-export-mysql"
 if ! is_binary_backup_feature_on; then
   # binary backup is already compressed

share/github-backup-utils/ghe-backup-settings

@@ -35,6 +35,19 @@ else
   unlink manage-password+
 fi
 
+# Backup external MySQL password if running external MySQL DB.
+if is_service_external 'mysql'; then
+  echo "Transferring external MySQL password..." 1>&3
+  ghe-ssh "$host" -- ghe-config secrets.external.mysql > external-mysql-password+ || (
+    echo "Warning: External MySQL password not set" >&2
+  )
+  if [ -n "$(cat external-mysql-password+)" ]; then
+    mv external-mysql-password+ external-mysql-password
+  else
+    unlink external-mysql-password+
+  fi
+fi
+
 if ghe-ssh "$host" -- "test -f $GHE_REMOTE_DATA_USER_DIR/common/idp.crt"; then
   echo "* Transferring SAML keys ..." 1>&3
   ghe-ssh $host -- sudo tar -C $GHE_REMOTE_DATA_USER_DIR/common/ -cf - "idp.crt saml-sp.p12" > saml-keys.tar

share/github-backup-utils/ghe-restore-mysql

@@ -24,10 +24,25 @@ ghe_remote_version_required "$GHE_HOSTNAME"
 # The snapshot to restore should be set by the ghe-restore command but this lets
 # us run this script directly.
 : ${GHE_RESTORE_SNAPSHOT:=current}
+export GHE_RESTORE_SNAPSHOT
 
 # The directory holding the snapshot to restore
 snapshot_dir="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"
 
+# When customer uses external database, we rely on customer to implement the restore process
+if is_service_external 'mysql' "$snapshot_dir/settings.json"; then
+  if [ -n "$EXTERNAL_DATABASE_RESTORE_SCRIPT" ]; then
+    $EXTERNAL_DATABASE_RESTORE_SCRIPT
+  else
+    echo "Error: EXTERNAL_DATABASE_RESTORE_SCRIPT is not configured. Please configure in backup.config."
+    exit 1
+  fi
+  # Finsh the benchmark and exit early. If the script returns a non-zero 
+  # exit code, it will be caught by`set -e`
+  bm_end "$(basename $0)"
+  exit 0
+fi
+
 ssh_config_file_opt=
 if is_binary_backup_feature_on; then
   # Feature "mysql.backup.binary" is on, which means new backup scripts are available
@@ -84,6 +99,7 @@ ghe-ssh $ssh_config_file_opt "$GHE_RESTORE_HOST" -- "sudo mkdir -p '$GHE_REMOTE_
 # Transfer MySQL data from the snapshot to the GitHub instance.
 cat $snapshot_dir/mysql.sql.gz | ghe-ssh $ssh_config_file_opt "$GHE_RESTORE_HOST" -- "sudo dd of=$GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz >/dev/null 2>&1"
 
+echo "Restore MySQL database ..."
 # Import the database
 echo "cat $GHE_REMOTE_DATA_USER_DIR/tmp/mysql.sql.gz | $IMPORT_MYSQL" | ghe-ssh $ssh_config_file_opt "$GHE_RESTORE_HOST" -- /bin/bash 1>&3
 

share/github-backup-utils/ghe-restore-settings

@@ -28,6 +28,13 @@ GHE_RESTORE_SNAPSHOT_PATH="$GHE_DATA_DIR/$GHE_RESTORE_SNAPSHOT"
 echo "Restoring license ..."
 ghe-ssh "$GHE_HOSTNAME" -- 'ghe-import-license' < "$GHE_RESTORE_SNAPSHOT_PATH/enterprise.ghl" 1>&3
 
+# Restore external MySQL password if running external MySQL DB.
+if [ -f "$GHE_RESTORE_SNAPSHOT_PATH/external-mysql-password" ]; then
+  echo "Restoring external MySQL password ..."
+  echo "ghe-config secrets.external.mysql '$(cat "$GHE_RESTORE_SNAPSHOT_PATH/external-mysql-password")'" |
+  ghe-ssh "$GHE_HOSTNAME" -- /bin/bash
+fi
+
 echo "Restoring settings ..."
 # work around issue importing settings with bad storage mode values
 ( cat "$GHE_RESTORE_SNAPSHOT_PATH/settings.json" && echo ) |

test/test-ghe-backup-config.sh

@@ -184,3 +184,44 @@ begin_test "ghe-backup-config ghe_debug accepts stdin as well as argument"
   unset GHE_VERBOSE_LOG
 )
 end_test
+
+begin_test "ghe-backup-config is_service_external enabled external mysql"
+(
+  set -e
+
+  tmpfile=$(mktemp /tmp/abc-script.XXXXXX)
+  echo "
+[mysql \"external\"]
+  enabled = true
+" > $tmpfile
+  is_service_external 'mysql' $tmpfile
+)
+end_test
+
+begin_test "ghe-backup-config is_service_external disabled external mysql"
+(
+  set -e
+
+  tmpfile=$(mktemp /tmp/abc-script.XXXXXX)
+  echo "
+[mysql \"external\"]
+  enabled = false
+" > $tmpfile
+
+  ! is_service_external 'mysql' $tmpfile
+)
+end_test
+
+begin_test "ghe-backup-config is_service_external unknown service"
+(
+  set -e
+
+  tmpfile=$(mktemp /tmp/abc-script.XXXXXX)
+  echo "
+[mysql \"external\"]
+  enabled = false
+" > $tmpfile
+
+  ! is_service_external 'hubot' $tmpfile
+)
+end_test