From 10ca9dc43ddeb61f56d14269e0d504b5f8dcda9c Mon Sep 17 00:00:00 2001 From: Avinash Singhal Date: Thu, 28 Jun 2018 11:58:43 +0530 Subject: [PATCH 1/3] added stripFields option --- bin/dynamo-backup-to-s3 | 15 ++++++++++++--- lib/dynamo-backup.js | 11 +++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/bin/dynamo-backup-to-s3 b/bin/dynamo-backup-to-s3 index d06b72b..f0b30d1 100755 --- a/bin/dynamo-backup-to-s3 +++ b/bin/dynamo-backup-to-s3 @@ -29,6 +29,7 @@ program .option('--aws-key ', 'AWS access key. Will use AWS_ACCESS_KEY_ID env var if --aws-key not set') .option('--aws-secret ', 'AWS secret key. Will use AWS_SECRET_ACCESS_KEY env var if --aws-secret not set') .option('--aws-region ', 'AWS region. Will use AWS_DEFAULT_REGION env var if --aws-region not set') + .option('--strip-fields ', 'Won't include fields in the bakup', list) .parse(process.argv); // run program @@ -46,9 +47,11 @@ var dynamoBackup = new DynamoBackup({ readPercentage: program.readPercentage, stopOnFailure: program.stopOnFailure, base64Binary: program.base64EncodeBinary, + stripFields: program.stripFields, saveDataPipelineFormat: program.saveDataPipelineFormat }); + dynamoBackup.on('error', function(data) { console.log('Error backing up ' + data.tableName); console.log(data.error); @@ -68,7 +71,13 @@ dynamoBackup.on('end-backup', function(tableName) { console.log('Done copying table ' + tableName + '. Took ' + endTime.diff(startTime, 'minutes', true).toFixed(2) + ' minutes'); }); -dynamoBackup.backupAllTables(function() { - console.log('Finished backing up DynamoDB'); +dynamoBackup.backupAllTables(function(err) { + if(err){ + console.log("Error "+ err); + } + else{ + console.log('Finished backing up DynamoDB'); + } process.exit(0); -}); \ No newline at end of file +}); + diff --git a/lib/dynamo-backup.js b/lib/dynamo-backup.js index a81e469..4245e1c 100644 --- a/lib/dynamo-backup.js +++ b/lib/dynamo-backup.js @@ -26,6 +26,7 @@ function DynamoBackup(options) { this.awsSecretKey = options.awsSecretKey; this.awsRegion = options.awsRegion; this.debug = Boolean(options.debug); + this.stripFields = options.stripFields || []; if (this.awsRegion) { params.region = this.awsRegion; @@ -49,6 +50,7 @@ DynamoBackup.prototype.backupTable = function (tableName, backupPath, callback) var self = this; var stream = new ReadableStream(); + console.log("starting backup for "+tableName); if (callback === undefined) { callback = backupPath; backupPath = self._getBackupPath(); @@ -89,7 +91,12 @@ DynamoBackup.prototype.backupTable = function (tableName, backupPath, callback) tableName, function (items) { items.forEach(function (item) { - if (self.base64Binary) { + + self.stripFields.forEach(function(key) { + delete item[key] + }); + + if (self.base64Binary) { _.each(item, function (value, key) { if (value && value.B) { value.B = new Buffer(value.B).toString('base64'); @@ -259,4 +266,4 @@ DynamoBackup.prototype._getDataPipelineAttributeValueKey = function (type) { } }; -module.exports = DynamoBackup; \ No newline at end of file +module.exports = DynamoBackup; From 7934e7e963cc90f5b2c3e1dabb825d56793cae0e Mon Sep 17 00:00:00 2001 From: Avinash Singhal Date: Thu, 28 Jun 2018 12:44:14 +0530 Subject: [PATCH 2/3] added global table option --- bin/dynamo-backup-to-s3 | 4 +++- lib/dynamo-backup.js | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/bin/dynamo-backup-to-s3 b/bin/dynamo-backup-to-s3 index f0b30d1..432c7ba 100755 --- a/bin/dynamo-backup-to-s3 +++ b/bin/dynamo-backup-to-s3 @@ -29,7 +29,8 @@ program .option('--aws-key ', 'AWS access key. Will use AWS_ACCESS_KEY_ID env var if --aws-key not set') .option('--aws-secret ', 'AWS secret key. Will use AWS_SECRET_ACCESS_KEY env var if --aws-secret not set') .option('--aws-region ', 'AWS region. Will use AWS_DEFAULT_REGION env var if --aws-region not set') - .option('--strip-fields ', 'Won't include fields in the bakup', list) + .option('--strip-fields ', 'exclue these fields in the bakup', list) + .option('-g, --global-table', 'specify if global table, will strip aws fields from backup') .parse(process.argv); // run program @@ -48,6 +49,7 @@ var dynamoBackup = new DynamoBackup({ stopOnFailure: program.stopOnFailure, base64Binary: program.base64EncodeBinary, stripFields: program.stripFields, + globalTable: program.globalTable, saveDataPipelineFormat: program.saveDataPipelineFormat }); diff --git a/lib/dynamo-backup.js b/lib/dynamo-backup.js index 4245e1c..6993505 100644 --- a/lib/dynamo-backup.js +++ b/lib/dynamo-backup.js @@ -27,6 +27,12 @@ function DynamoBackup(options) { this.awsRegion = options.awsRegion; this.debug = Boolean(options.debug); this.stripFields = options.stripFields || []; + this.globalTable = options.globalTable || false; + + if(this.globalTable){ + var awsFields = ['aws:rep:updateregion', 'aws:rep:deleting', 'aws:rep:updatetime']; + this.stripFields = _.union(awsFields, this.stripFields); + } if (this.awsRegion) { params.region = this.awsRegion; From 56c0a19812a508d4d3bc67f766cfbeff9d7015c9 Mon Sep 17 00:00:00 2001 From: singhalavi Date: Thu, 28 Jun 2018 14:27:30 +0530 Subject: [PATCH 3/3] Update dynamo-backup.js --- lib/dynamo-backup.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/dynamo-backup.js b/lib/dynamo-backup.js index 6993505..cfdea39 100644 --- a/lib/dynamo-backup.js +++ b/lib/dynamo-backup.js @@ -56,7 +56,6 @@ DynamoBackup.prototype.backupTable = function (tableName, backupPath, callback) var self = this; var stream = new ReadableStream(); - console.log("starting backup for "+tableName); if (callback === undefined) { callback = backupPath; backupPath = self._getBackupPath();