Add option to upload archive to IBM COS (#2)

Author: michaellzc

.travis.yml

@@ -1,7 +1,7 @@
 language: bash
 services: docker
 env:
-  secure: AYO9a/roP4bf1YyZHUd4Y7ap4QZqmCK/++1MF8u3RVtIlPcjm8DvN5+7RwvXJeDreWpaK3eewYvP18/ZS5ot04m+y1MF1XgI1HueDqnynCkHtNPxOqjfbkcxCmOvwE3oW5MapbbIV7jcP0ZEAUM3IMHgqLs/E/0QC3sHEG8NaIZbMQ/LH/zuTTX7ZN55b9BURn4jJZWitcmS7NHDPpNJxOWXsYozA7tdnodAzjqXbWyGmDOnC3jBWyZdKEjA/94+sgDGH0XnpEDDuItaNGpPTRCGL3CaLsCdzj05ZD6Sjug2g7b7jpRNgiu5ca+l07IJuImNXkgpE71NOf4CxDLjFU26QEzy4Mx//yU2B1aIM4NELpuDjDgdL1qg3P0xufRH3oGO8pknrQ5JsciCZil2+3iVOZqdXT6F4wueFDjiCOP20iMaHi0NXknsrumdS1K/yDTEb2Z0kj79Yn/+JKpHfP12np1BKMMW2ZQzHcxoQLQNXHXTJCxcQmcuUU7WsAvWaA8Onsga6E+/KoDaHbSruT11OwfddOtIZMAgoSyx0CZVt+SxxUiqbo28u/TEq1ics4zVTzfY/zV88gDGA4lLPx9cDwktwBSpmvFFl6SrB0hsFFJNYN02675XnAwMzftc8mwarxt3vIJg5qrbSpMQGUgwjOsM8EBqSPzbZoUzcog=
+  secure: FUab7+Hp6ev10T7m/zq9MN9B0ReGOdNSE7xrgx9YsA4zDsIwezEnoo1uh1M4RQGnV0OAvclNdNdv5ggGV91ij64ETpKTX6NUev9MO8zdOWw0wMrSi9/AOP0lbKH/M52IQs60zD+fJ9StHqt7DP6krfIjHa0/0YmhZKynQn7dNwI4rriZVOUuveYvVOATA7HGdDDHLqlUN6sUEjLlUpe2WkqmB1EDwgioVGe59slvlIZ9nUKHlRHFIacA2W93RO/o4DuHjjbYJZy1Lw7aWZ3yF3VO3S6tHvuw1Q1daT2mBrnNl3K+SMAJI1BqtfEXtA9Zu8t8Hb42XbGWzvsPwU1nP46InK6+wz15/A49KOFtGwMfVfg5uPon565sQ3vPqxSJ/QxHtGB2GsgfmTlxbuOgY6hKNwh1qWMlWiIaCFFf6++rnDT4ZLVMg3t9mp4tQDqMxASRAiX5X4CFte3oOjeOWAHIxbHa39cQ7wAWpmEptU4RJlyuXhjJp0W4NjuD9KqmgxizYJO8uCUkOfHBGH2BpicqxF6hWnGe0/fJm/JSuQjsGs1SPgcQIvHanrbeAEIIrrkQSnxClxsFxXWLxWTiRei3N/q5wb9/Ew4GfhUV49i8YzgE+wa/AIYO/PWGIaQWjF4sYo59CJaiM7ES/e2NdPcxL+1QDEMuVNsHT7X0/pM=
 addons:
   apt:
     packages:

CHANGELOG.md

@@ -1,74 +1,106 @@
 # Change Log
+
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased][]
 
+## [1.1.0][] - 2019-05-24
+
+### Added
+
+- Added `monsoon-upload-cos` plugin to upload backup files to IBM Cloud Object Storage
+
 ## [1.0.1][] - 2018-04-06
+
 ### Fixed
--   Fixed conflicting package version declarations.
+
+- Fixed conflicting package version declarations.
 
 ## [1.0.0][] - 2018-04-06
+
 ### Added
--   New mandatory parameter: `SOFTLAYER_NETWORK` to choose between
-    `public` and `private` for upload.
+
+- New mandatory parameter: `SOFTLAYER_NETWORK` to choose between
+  `public` and `private` for upload.
 
 ### Updated
--   Updated MySQL, Postgres, and MySQL modules.
--   Updated Sentry Raven from `5.32.0` to `6.6.0`
+
+- Updated MySQL, Postgres, and MySQL modules.
+- Updated Sentry Raven from `5.32.0` to `6.6.0`
 
 ### Fixed
--   Fixed linter errors.
+
+- Fixed linter errors.
 
 ## [0.3.2][] - 2017-12-20
+
 ### Fixed
--   Made plugins fail with non-zero status for backup errors
--   Bumped Python from `2.7.13` to `2.7.14`
--   Switched to modern multi-stage build process
+
+- Made plugins fail with non-zero status for backup errors
+- Bumped Python from `2.7.13` to `2.7.14`
+- Switched to modern multi-stage build process
 
 ## [0.3.1][] - 2017-07-17
+
 ### Fixed
--   Also skip Softlayer upload if missing username or api key
+
+- Also skip Softlayer upload if missing username or api key
 
 ## [0.3.0][] - 2017-07-11
+
 ### Added
--   PostgreSQL support
--   `/backups` Docker volume
+
+- PostgreSQL support
+- `/backups` Docker volume
 
 ### Changed
--   Use `bash` instead of `ash`
--   Local backups go to `/backups`, not `/tmp`
+
+- Use `bash` instead of `ash`
+- Local backups go to `/backups`, not `/tmp`
 
 ### Fixed
--   Apply updates for `db` and `ncurses`
+
+- Apply updates for `db` and `ncurses`
 
 ## [0.2.1][]
+
 ### Fixed
--   Name files `.gz` rather than `.gzip` to avoid `gunzip` issues
+
+- Name files `.gz` rather than `.gzip` to avoid `gunzip` issues
 
 ## [0.2.0][]
+
 ### Added
--   You can skip backing up Mongo, MySQL, or files by not specifying
-    Mongo host, MySQL host, or file paths
--   Softlayer credentials are still required, and Sentry is still optional
+
+- You can skip backing up Mongo, MySQL, or files by not specifying
+  Mongo host, MySQL host, or file paths
+- Softlayer credentials are still required, and Sentry is still optional
 
 ## [0.1.7][]
+
 ### Fixed
--   Wait for backups to complete
+
+- Wait for backups to complete
 
 ## [0.1.6][]
+
 ### Fixed
--   OS package security updates
+
+- OS package security updates
 
 ## 0.1.0
+
 ### Added
--   Mongo, MySQL, and file backup
--   Upload to Softlayer
--   Travis CI for continuous integration
 
-[Unreleased]: https://github.ibm.com/bdu/gamora/compare/1.0.1...HEAD
+- Mongo, MySQL, and file backup
+- Upload to Softlayer
+- Travis CI for continuous integration
+
+[unreleased]: https://github.ibm.com/bdu/gamora/compare/1.1.0...HEAD
+[1.1.0]: https://github.ibm.com/bdu/gamora/compare/1.0.1...1.1.0
 [1.0.1]: https://github.ibm.com/bdu/gamora/compare/1.0.0...1.0.1
 [1.0.0]: https://github.ibm.com/bdu/gamora/compare/0.3.2...1.0.0
 [0.3.2]: https://github.ibm.com/bdu/gamora/compare/0.3.1...0.3.2

docker-compose.yaml

@@ -29,6 +29,14 @@ services:
       - SOFTLAYER_CONTAINER
       - SOFTLAYER_PATH
 
+      # IBM Cloud Object Storage config
+      - IBM_COS_ACCESS_KEY
+      - IBM_COS_SECRET_KEY
+      - IBM_COS_INSTANCE_ID
+      - IBM_COS_ENDPOINT_URL
+      - IBM_COS_BUCKET
+      - IBM_COS_PATH
+
       - BACKUP_LOCAL_PATHS
     links:
       - mysql

gamora/docker-entrypoint.sh

@@ -21,16 +21,19 @@ upload_backup() {
 
   if [[ -z "${SOFTLAYER_PATH:-}" ]]; then
     log "Skipping upload because no Softlayer path"
+    upload_backup_cos "${filename:?}"
     return
   fi
 
   if [[ -z "${SOFTLAYER_USER:-}" ]]; then
     log "Skipping upload because no Softlayer user"
+    upload_backup_cos "${filename:?}"
     return
   fi
 
   if [[ -z "${SOFTLAYER_API_KEY:-}" ]]; then
     log "Skipping upload because no Softlayer api key"
+    upload_backup_cos "${filename:?}"
     return
   fi
 
@@ -49,6 +52,44 @@ upload_backup() {
   log "Done: Uploading backup"
 }
 
+upload_backup_cos() {
+  local filename=$1
+  local remote_path
+
+  if [[ -z "${IBM_COS_INSTANCE_ID:-}" ]]; then
+    log "Skipping upload because no IBM COS service instance id"
+    return
+  fi
+
+  if [[ -z "${IBM_COS_ENDPOINT_URL:-}" ]]; then
+    log "Skipping upload because no IBM COS endpoint url"
+    return
+  fi
+
+  if [[ -z "${IBM_COS_BUCKET:-}" ]]; then
+    log "Skipping upload because no IBM COS bucket name"
+    return
+  fi
+
+  if [[ -z "${IBM_COS_PATH}" ]]; then
+    remote_path="$(date +%Y/%m)"
+  else
+    remote_path="${IBM_COS_PATH:?}/$(date +%Y/%m)"
+  fi
+
+  log "Uploading backup to IBM COS"
+  monsoon "${NOTIFICATION_SETTINGS[@]}" upload cos \
+      --endpoint-url "${IBM_COS_ENDPOINT_URL}" \
+      --instance-id "${IBM_COS_INSTANCE_ID}" \
+      --access-key "${IBM_COS_ACCESS_KEY}" \
+      --secret-key "${IBM_COS_SECRET_KEY}" \
+      "${BACKUP_PATH:?}/${filename:?}" \
+      "${IBM_COS_BUCKET:?}" \
+      "${remote_path:?}" \
+      && rm -f "${BACKUP_PATH:?}/${filename:?}"
+  log "Done: Uploading backup to IBM COS"
+}
+
 back_up_mongo() {
   local filename
 

gamora/requirements.txt

@@ -7,3 +7,4 @@ monsoon-backup-mysql==0.1.7
 monsoon-backup-postgresql==0.1.5
 monsoon-notifier-sentry==0.1.4
 monsoon-upload-softlayer==0.2.0
+monsoon-upload-cos==0.1.0