Add the ability to do backups to remote systems with rsync

Author: natefoo

README.md

@@ -117,6 +117,11 @@ Role Variables
 - `postgresql_backup_mail_recipient`: User or address that should receive mail
   from the backup scripts.
 
+- `postgresql_backup_remote_rsync_path`: Path to `rsync` on the remote system.
+
+- `postgresql_backup_post_command`: Arbitrary command to run after successful
+  completion of a scheduled backup.
+
 Dependencies
 ------------
 

tasks/backup.yml

@@ -19,7 +19,7 @@
   notify: Reload PostgreSQL
 
 - name: Schedule backups
-  cron: name="PostgreSQL Backup" cron_file=ansible_postgresql_backup user=postgres hour={{ postgresql_backup_hour | default(1) }} minute={{ postgresql_backup_minute | default(0) }} day={{ postgresql_backup_day | default(omit) }} month={{ postgresql_backup_month | default(omit) }} weekday={{ postgresql_backup_weekday | default(omit) }} job={{ postgresql_backup_local_dir }}/bin/scheduled_backup.sh
+  cron: name="PostgreSQL Backup" cron_file=ansible_postgresql_backup user=postgres hour={{ postgresql_backup_hour | default(1) }} minute={{ postgresql_backup_minute | default(0) }} day={{ postgresql_backup_day | default(omit) }} month={{ postgresql_backup_month | default(omit) }} weekday={{ postgresql_backup_weekday | default(omit) }} job="{{ postgresql_backup_local_dir }}/bin/scheduled_backup.sh{{ ' && ' ~ postgresql_backup_post_command if postgresql_backup_post_command is defined else '' }}"
 
 - name: Schedule PostgreSQL working WAL backup
   cron: name="PostgreSQL WAL Backup" cron_file=ansible_postgresql_walbackup user=postgres job={{ postgresql_backup_local_dir }}/bin/backup_working_wal.sh

templates/archive_wal.sh.j2

@@ -17,13 +17,15 @@ mutex="{{ postgresql_backup_local_dir }}/walmutex"
 mailto='{{ postgresql_backup_mail_recipient }}'
 mutex_attempts=50
 
+[ '{{ postgresql_backup_remote_rsync_path | default("None") }}' != 'None' ] && remote_rsync='--rsync-path={{ postgresql_backup_remote_rsync_path }}' || remote_rsync=''
+
 handler()
 {
     command=$@
     out=`$command 2>&1`
     ret=$?
     if [ $ret != 0 ]; then
-        (echo "execuing $command failed with code $ret:" ; echo "$out") | mail -s "$server: WAL backup failed" $mailto
+        (echo "execuing $command failed with code $ret:" ; echo "$out") | mail -s "$server: WAL archive failed" $mailto
         rm -f $mutex
         exit 1
     else
@@ -40,29 +42,32 @@ attempt=0
 while [ -f $mutex ]; do
     attempt=`expr $attempt + 1`
     if [ $attempt -gt $mutex_attempts ]; then
-        echo "archive of WAL $file failed, timed out waiting for mutex from `cat $mutex`" | mail -s "$server: WAL backup failed" $mailto
+        echo "archive of WAL $file failed, timed out waiting for mutex from `cat $mutex`" | mail -s "$server: WAL archive failed" $mailto
         exit 1
     fi
     sleep 5
 done
 
 echo "archive-WAL $$" > $mutex
 
-if [ ! -d $backup_dir ]; then
-    handler mkdir -p $backup_dir
-fi
-
-if [ -f $backup_dir/$file ]; then
-    echo "$backup_dir/$file already exists, overwriting is not allowed" | mail -s "$server: WAL backup failed" $mailto
+# If rsync outputs anything to stdout, the destination already existed, which should not happen
+if [ -n "`handler rsync $remote_rsync --ignore-existing -ptg --info=skip1 $full_file $backup_dir`" ]; then
+    echo "$backup_dir/$file already exists, overwriting is not allowed" | mail -s "$server: WAL archive failed" $mailto
+    rm -f $mutex
     exit 1
 fi
 
-handler rsync -ptg $full_file $backup_dir
+# create an empty directory for --delete
+empty=`handler mktemp -d {{ postgresql_backup_local_dir }}/emptyXXXXXX`
+
+# clear the active directory
+handler rsync $remote_rsync -rptg --delete $empty/ $active_dir
 
-handler rm -f $active_dir/*
+# remove the temp empty dir
+handler rmdir $empty
 
 # debug
-#echo "archive of WAL $file succeeded" | mail -s "$server: WAL backup succeeded" $mailto
+#echo "archive of WAL $file succeeded" | mail -s "$server: WAL archive succeeded" $mailto
 
 # exit normally
 rm -f $mutex

templates/backup_working_wal.sh.j2

@@ -9,11 +9,11 @@ mailto='{{ postgresql_backup_mail_recipient }}'
 
 active=`ls -1rtF $xlog_dir | grep -v '/$' | tail -1`
 
-out=`cp -p $xlog_dir/$active $backup_dir/$active 2>&1`
+out=`scp -p $xlog_dir/$active $backup_dir/$active 2>&1`
 ret=$?
 
 if [ $ret != 0 ]; then
-    (echo "cp failed with code $ret:" ; echo "$out") | mail -s "`hostname`: WAL backup failed" $mailto
+    (echo "scp failed with code $ret:" ; echo "$out") | mail -s "`hostname`: WAL backup failed" $mailto
 fi
 
 exit $ret

templates/scheduled_backup.sh.j2

@@ -10,6 +10,8 @@ mailto='{{ postgresql_backup_mail_recipient }}'
 mutex='{{ postgresql_backup_local_dir }}/nightlymutex'
 rotate='{{ postgresql_backup_rotate | default("True") }}'
 
+[ '{{ postgresql_backup_remote_rsync_path | default("None") }}' != 'None' ] && remote_rsync='--rsync-path={{ postgresql_backup_remote_rsync_path }}' || remote_rsync=''
+
 handler()
 {
     command=$@
@@ -53,6 +55,9 @@ psql_handler()
     fi
 }
 
+[[ "$dir" == *':'* ]] && dir_is_remote=1
+[ "$dir_is_remote" -a $rotate != "False" ] && { echo "error: rotation with remote backup directory is not implemented" | mail -s "$server: Postgres nightly backup failed" $mailto ; exit 1 ; }
+
 [ ! -d `dirname $mutex` ] && handler mkdir -p `dirname $mutex`
 
 while [ -f $mutex ]; do
@@ -63,24 +68,22 @@ start=`date +%s`
 
 echo "nightly $$" > $mutex
 
-# move previous backup
-if [ -d $dir ]; then
-    if [ $rotate = "True" ]; then
+if [ ! "$dir_is_remote" ]; then
+    # move previous backup
+    if [ -d $dir -a $rotate = "True" ]; then
         handler mv $dir {{ postgresql_backup_dir }}/`date -u +%Y%m%dT%H%M%SZ`
-    else
-        handler rm -rf $dir
     fi
 fi
 
-# set up new directory
-handler mkdir -p $dir
-
 # tell postgres we'll be backing up
 psql_handler "SELECT pg_start_backup('$dir');"
 backup_started=1
 
+# ensure the local wal/ directory exists, so rsync creates it for wal archival
+handler mkdir -p $data/wal
+
 # begin the backup
-handler rsync -a --exclude pg_xlog $data/ $dir
+handler rsync $remote_rsync -rptg --delete --delete-delay --exclude pg_xlog $data/ $dir
 
 # tell postgres we're done
 psql_handler "SELECT pg_stop_backup();"