Switch to allow table overwrite during restore. CLI examples added to readme.

Author: sdesalas

README.md

@@ -161,6 +161,8 @@ __Arguments__
 
 It is suitable for restoring large tables without needing to write to disk or use a large amount of memory. Use it on an AWS EC2 instance for best results and to minimise network latency, this should yield restore speeds of around 15min per GB.
 
+Use `--overwrite` if the table already exists. Otherwise it will attempt to generate table on the fly.
+
 Can be run as a command line script or as an npm module. 
 
 # Command line usage
@@ -174,17 +176,54 @@ Can be run as a command line script or as an npm module.
     -V, --version                     output the version number
     -s, --source [path]               Full S3 path to a JSON backup file (Required)
     -t, --table [name]                Name of the Dynamo Table to restore to (Required)
+    -o, --overwrite                   Table already exists, skip auto-create. Default is false.
     -c, --concurrency <requestcount>  Number of concurrent requests & dynamo capacity units. Defaults to 200.
     -pk, --partitionkey [columnname]  Name of Primary Partition Key. If not provided will try determine from backup.
     -sk, --sortkey [columnname]       Name of Secondary Sort Key. Ignored unless --partitionkey is provided.
     -rc, --readcapacity <units>       Read Units for new table (when finished). Default is 5.
-    -wc, --writecapacity <units>      Write Units for new table (when finished). Default is 5.
-    -sf, --stop-on-failure            Stop process when the same batch fails to restore 3 times. Defaults to false.
+    -wc, --writecapacity <units>      Write Units for new table (when finished). Default is --concurrency.
+    -sf, --stop-on-failure            Stop process when the same batch fails to restore multiple times. Defaults to false.
     --aws-key <key>                   AWS access key. Will use AWS_ACCESS_KEY_ID env var if --aws-key not set
     --aws-secret <secret>             AWS secret key. Will use AWS_SECRET_ACCESS_KEY env var if --aws-secret not set
     --aws-region <region>             AWS region. Will use AWS_DEFAULT_REGION env var if --aws-region not set
 ```
 
+## Examples
+
+```
+
+    # Restore over existing table (cmd.exe).
+    > node ./bin/dynamo-restore-from-s3 -t acme-customers -s s3://my-backups/acme-customers.json --overwrite 
+
+    # Restore over existing table (shell).
+    $ ./bin/dynamo-restore-from-s3 -t acme-customers -s s3://my-backups/acme-customers.json --overwrite 
+
+    # Restore over existing table, 1000 concurrent requests. Stop if any batch fails 1000 times.
+    $ ./bin/dynamo-restore-from-s3 -t acme-customers -c 1000 -s s3://my-backups/acme-customers.json --overwrite -sf
+
+    # Restore over existing table, 1000 concurrent requests. When finished, set read capacity to 50 and write capacity to 10 (both needed).
+    $ ./bin/dynamo-restore-from-s3 -t acme-customers -c 1000 -s s3://my-backups/acme-customers.json --overwrite --readcapacity 50 --writecapacity 10
+
+    # Auto-generate table (determine PK from backup). 
+    $ ./bin/dynamo-restore-from-s3 -t acme-customers -s s3://my-backups/acme-customers.json
+
+    # Auto-generate table with partition and sort key.
+    $ ./bin/dynamo-restore-from-s3 -t acme-orders -s s3://my-backups/acme-orders.json -pk customerId -sk createDate 
+
+    # Auto-generate table, defined PK. Concurrency 2000 (~ 2GB backup).
+    $ ./bin/dynamo-restore-from-s3 -t acme-orders -pk orderId -c 2000 -s s3://my-backups/acme-orders.json 
+
+    # Auto-generate table. 2000 write units during restore. When finished set 50 write units and 100 write units (both needed).
+    $ ./bin/dynamo-restore-from-s3 -t acme-orders -c 2000 -s s3://my-backups/acme-orders.json --readcapacity 100 --writecapacity 50
+
+    # Auto-generate table. Concurrency 50 (10 MB backup or less).
+    $ ./bin/dynamo-restore-from-s3 -t acme-orders -c 50 -s s3://my-backups/acme-orders.json 
+
+    # Auto-generate table. Concurrency 50. Stop process if any batch fails 50 times.
+    $ ./bin/dynamo-restore-from-s3 -t acme-orders -c 50 -sf -s s3://my-backups/acme-orders.json 
+
+```
+
 # npm module usage
 
 ## Quick Example
@@ -195,10 +234,8 @@ var DynamoRestore = require('dynamo-backup-to-s3').Restore;
 var restore = new DynamoRestore({
     source: 's3://my-backups/DynamoDB-backup-2016-09-28-15-36-40/acme-customers-prod.json',
     table: 'acme-customers-dev',
+    overwrite: true,
     concurrency: 200, // for large restores use 1 unit per MB as a rule of thumb (ie 1000 for 1GB restore)
-    partitionkey: 'customerId',
-    readcapacity: 1,
-    writecapacity: 1,
     awsAccessKey: /* AWS access key */,
     awsSecretKey: /* AWS secret key */,
     awsRegion: /* AWS region */
@@ -228,11 +265,12 @@ restore.run(function() {
 ```
 var options = {
     source: /* path to json file in s3 bucket, should start with s3://bucketname/... */,
-    table: /* name of dynamo table, created on the fly, MUST NOT EXIST */,
+    table: /* name of dynamo table, will be created on the fly unless overwritten */,
+    overwrite: /* true/false if table already exits (defaults to false) */
     concurrency: /* number of concurrent requests (and dynamo write capacity units) */,
-    partitionkey: /* name of partition key column*/,
+    partitionkey: /* name of partition key column */,
     sortkey: /* name of secondary (sort) key column */ ,
-    readcapacity: /* number of read capacity units */,
+    readcapacity: /* number of read capacity units (when restore finishes) */,
     writecapacity: /* number of write capacity units (when restore finishes) */,
     stopOnFailure: /* true/false should a single failed batch stop the whole restore job? */,
     awsAccessKey: /* AWS access key */,

bin/dynamo-restore-from-s3

@@ -9,11 +9,12 @@ program
     .usage('[options] -s "s3://mybucket/path/to/file.json" -t "new-dynamodb-table"')
     .option('-s, --source [path]', 'Full S3 path to a JSON backup file (Required)')
     .option('-t, --table [name]', 'Name of the Dynamo Table to restore to (Required)')
+    .option('-o, --overwrite', 'Table already exists, skip auto-create. Default is false.')
     .option('-c, --concurrency <requestcount>', 'Number of concurrent requests & dynamo capacity units. Defaults to 200.')
     .option('-pk, --partitionkey [columnname]', 'Name of Primary Partition Key. If not provided will try determine from backup.')
     .option('-sk, --sortkey [columnname]', 'Name of Secondary Sort Key. Ignored unless --partitionkey is provided.')
     .option('-rc, --readcapacity <units>', 'Read Units for new table (when finished). Default is 5.')
-    .option('-wc, --writecapacity <units>', 'Write Units for new table (when finished). Default is 5.')
+    .option('-wc, --writecapacity <units>', 'Write Units for new table (when finished). Default is --concurrency.')
     .option('-sf, --stop-on-failure', 'Stop process when the same batch fails to restore 3 times. Defaults to false.', true)
     .option('--aws-key <key>', 'AWS access key. Will use AWS_ACCESS_KEY_ID env var if --aws-key not set')
     .option('--aws-secret <secret>', 'AWS secret key. Will use AWS_SECRET_ACCESS_KEY env var if --aws-secret not set')
@@ -36,6 +37,7 @@ var dynamoRestore = new DynamoRestore({
     // Main settings
     source: program.source,
     table: program.table,
+    overwrite: !!program.overwrite,
     concurrency: program.concurrency,
     stopOnFailure: !!program.stopOnFailure,
     // New table properties
@@ -64,6 +66,10 @@ dynamoRestore.on('error', function(message) {
     process.exit(-1);
 });
 
+dynamoRestore.on('warning', function(message) {
+    console.log(message);
+});
+
 dynamoRestore.on('start-download', function(streamMeta) {
     var time = runTimes.startDownload = new Date().getTime();
     console.log('Starting download. %s remaining...', translate(streamMeta.ContentLength));

lib/dynamo-restore.js

@@ -19,7 +19,7 @@ function DynamoRestore(options) {
     options.minConcurrency = 1;
     options.maxConcurrency = options.concurrency;
     options.readcapacity = options.readcapacity || 5;
-    options.writecapacity = options.writecapacity || 5;
+    options.writecapacity = options.writecapacity || 0;
     options.stopOnFailure = options.stopOnFailure || false;
     options.awsKey = options.awsKey || process.env.AWS_ACCESS_KEY_ID;
     options.awsSecret = options.awsSecret || process.env.AWS_SECRET_ACCESS_KEY;
@@ -42,18 +42,6 @@ DynamoRestore.prototype.run = function(finishCallback) {
     this._validateS3Backup(this.options);
     this._validateTable(this.options);
     this._startDownload();
-    this.on('finish', (function() {
-        var dynamodb = new AWS.DynamoDB(),
-            options = this.options;
-        // Finish off by updating read/write capacity to end-state
-        dynamodb.updateTable({
-            TableName: options.table,
-            ProvisionedThroughput: {
-                ReadCapacityUnits: options.readcapacity,
-                WriteCapacityUnits: options.writecapacity
-            }
-        }, finishCallback);
-    }).bind(this));
     // Exit program by default if there are no error listeners attached.
     this.on('error', (function(message) {
         if (finishCallback) {
@@ -63,6 +51,23 @@ DynamoRestore.prototype.run = function(finishCallback) {
             throw new Error(message);
         }
     }).bind(this));
+    // Finish off by updating write capacity to end-state (if needed)
+    this.on('finish', (function() {
+        var dynamodb = new AWS.DynamoDB(),
+            options = this.options;
+        // Do we need to update write capacity?
+        if (options.writecapacity) {
+            dynamodb.updateTable({
+                TableName: options.table,
+                ProvisionedThroughput: {
+                    ReadCapacityUnits: options.readcapacity,
+                    WriteCapacityUnits: options.writecapacity
+                }
+            }, finishCallback);
+        } else {
+            finishCallback();
+        }
+    }).bind(this));
 };
 
 DynamoRestore.prototype._validateS3Backup = function(options) {
@@ -96,7 +101,14 @@ DynamoRestore.prototype._checkTableExists = function(error, data) {
         return this.emit('error', 'Fatal Error. Could not connect to AWS DynamoDB engine. Please check your credentials.');
     }
     if (data.TableNames.indexOf(this.options.table) > -1) {
-        return this.emit('error', 'Fatal Error. The destination table already exists! Exiting process..');
+        // Table exists, should we overwrite it??
+        if (this.options.overwrite) {
+            this.emit('warning', util.format('WARN: table [%s] will be overwritten.', options.table));
+            dynamodb.describeTable({ TableName: this.options.table }, this._checkTableReady.bind(this));
+        } else {
+            this.emit('error', 'Fatal Error. The destination table already exists! Exiting process..');
+        }
+        return;
     }
     if (this.options.partitionkey && this.options.partitionkeytype) {
         // Once we know the partition key and data type the rest is a breeze.