Merge pull request #166 from eoehen/feature/Add-support-for-blobxfer

Add support for blobxfer

Author: tiredofit

.gitattributes

@@ -0,0 +1,2 @@
+# Declare files that will always have LF line endings on checkout.
+*.* text eol=lf

Dockerfile

@@ -68,6 +68,9 @@ RUN set -ex && \
     make && \
     make install && \
     \
+    apk add gcc build-base libressl-dev libffi-dev python3-dev py3-pip && \
+    pip3 install blobxfer && \
+    \
 ### Cleanup
     apk del .db-backup-build-deps && \
     rm -rf /usr/src/* && \

examples/mssql-blobxfer/docker-compose.yml

@@ -0,0 +1,69 @@
+# 
+# Example for Microsoft SQL Server
+# upload with blobxfer to azure storage
+# 
+
+version: '2'
+
+networks:
+  example-mssql-blobxfer-net:
+    name: example-mssql-blobxfer-net
+
+services:
+  example-mssql-s3-db:
+    hostname: example-db-host
+    image: mcr.microsoft.com/mssql/server:2019-latest
+    container_name: example-mssql-s3-db
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:11433:1433"
+    networks:
+      example-mssql-blobxfer-net:
+    volumes:
+      - ./tmp/backups:/tmp/backups    # shared tmp backup directory
+    environment:
+      ACCEPT_EULA: Y
+      MSSQL_SA_PASSWORD: 5hQa0utRFBpIY3yhoIyE
+      MSSQL_PID: Express
+
+  example-mssql-blobxfer-db-backup:
+    container_name: example-mssql-blobxfer-db-backup
+    # if you want to build and use image from current source
+    # execute in terminal --> docker build -t tiredofit/db-backup-mssql-blobxfer .
+    # replace --> image: tiredofit/db-backup-mssql
+    # image: tiredofit/db-backup
+    image: tiredofit/db-backup-mssql-blobxfer
+    links:
+     - example-mssql-s3-db
+    volumes:
+      - ./backups:/backup
+      - ./tmp/backups:/tmp/backups    # shared tmp backup directory
+      #- ./post-script.sh:/assets/custom-scripts/post-script.sh
+    environment:
+      # - DEBUG_MODE=TRUE
+      - DB_TYPE=mssql
+      - DB_HOST=example-db-host
+      # - DB_PORT=1488
+      # - DB_NAME=ALL             # [ALL] not working on sql server.
+      # create database with name `test1` manually first
+      - DB_NAME=test1             # Create this database
+      - DB_USER=sa
+      - DB_PASS=5hQa0utRFBpIY3yhoIyE
+      - DB_DUMP_FREQ=1            # backup every 5 minute
+      # - DB_DUMP_BEGIN=0000      # backup starts immediately
+      - DB_CLEANUP_TIME=3         # clean backups they are older than 60 minutes
+      - ENABLE_CHECKSUM=TRUE
+      - CHECKSUM=SHA1
+      - COMPRESSION=GZ
+      - SPLIT_DB=FALSE
+      - CONTAINER_ENABLE_MONITORING=FALSE
+      # === S3 Blobxfer ===
+      - BACKUP_LOCATION=blobxfer
+      # Add here azure storage account
+      - BLOBXFER_STORAGE_ACCOUNT={TODO Add Storage Name}
+      # Add here azure storage account key
+      - BLOBXFER_STORAGE_ACCOUNT_KEY={TODO Add Key}
+      - BLOBXFER_REMOTE_PATH=docker-db-backup
+    restart: always
+    networks:
+      example-mssql-blobxfer-net:

install/assets/defaults/10-db-backup

@@ -22,3 +22,4 @@ SCRIPT_LOCATION_POST=${SCRIPT_LOCATION_POST:-"/assets/scripts/post/"}
 SIZE_VALUE=${SIZE_VALUE:-"bytes"}
 SPLIT_DB=${SPLIT_DB:-"TRUE"}
 TEMP_LOCATION=${TEMP_LOCATION:-"/tmp/backups"}
+BLOBXFER_REMOTE_PATH=${BLOBXFER_REMOTE_PATH:-"/docker-db-backup"}

install/assets/functions/10-db-backup

@@ -452,10 +452,13 @@ cleanup_old_data() {
     if [ -n "${DB_CLEANUP_TIME}" ]; then
         if [ "${master_exit_code}" != 1 ]; then
             case "${BACKUP_LOCATION,,}" in
-                "file" | "filesystem" )
+                "file" | "filesystem" | "blobxfer" )
                     print_info "Cleaning up old backups"
                     mkdir -p "${DB_DUMP_TARGET}"
                     find "${DB_DUMP_TARGET}"/  -mmin +"${DB_CLEANUP_TIME}" -iname "*" -exec rm {} \;
+
+                    print_info "Cleaning up old backups on S3 storage with blobxfer"
+                    blobxfer upload --mode file --remote-path ${BLOBXFER_REMOTE_PATH} --local-path ${DB_DUMP_TARGET} --delete --delete-only
                 ;;
                 "s3" | "minio" )
                     print_info "Cleaning up old backups"
@@ -631,6 +634,18 @@ move_dbbackup() {
                     silent aws ${PARAM_AWS_ENDPOINT_URL} s3 cp ${TEMP_LOCATION}/*.${checksum_extension} s3://${S3_BUCKET}/${S3_PATH}/ ${s3_ssl} ${s3_ca_cert} ${S3_EXTRA_OPTS}
                 fi
 
+                rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}"
+                rm -rf "${TEMP_LOCATION}"/"${target}"
+            ;;
+            "blobxfer" )
+                print_info "Moving backup to S3 Bucket with blobxfer"
+
+                mkdir -p "${DB_DUMP_TARGET}"
+                mv "${TEMP_LOCATION}"/*."${checksum_extension}" "${DB_DUMP_TARGET}"/
+                mv "${TEMP_LOCATION}"/"${target}" "${DB_DUMP_TARGET}"/"${target}"
+
+                blobxfer upload --mode file --remote-path ${BLOBXFER_REMOTE_PATH} --local-path ${DB_DUMP_TARGET}
+
                 rm -rf "${TEMP_LOCATION}"/*."${checksum_extension}"
                 rm -rf "${TEMP_LOCATION}"/"${target}"
             ;;