Merge pull request #24 from sdesalas/master

Dynamo Restore

Author: dylanlingelbach

README.md

@@ -1,5 +1,7 @@
 # dynamo-backup-to-s3
 
+![dynamo to s3](https://raw.githubusercontent.com/sdesalas/dynamo-backup-to-s3/master/img/dynamo-backup-to-s3.png)
+
 ## Stream DynamoDB backups to S3.
 
 dynamo-backup-to-s3 is a utility to stream DynamoDB data to S3.  Since the data is streamed directly from DynamoDB to S3 it is suitable for copying large tables directly. Tables are copied in parallel.
@@ -148,3 +150,200 @@ __Arguments__
   completed. If no error has occurred, the `callback` should be run without 
   arguments or with an explicit `null` argument.
 * `callback(err)` - A callback which is called when the table has finished backing up, or an error occurs
+
+# dynamo-restore-from-s3
+
+![s3 to dynamo](https://raw.githubusercontent.com/sdesalas/dynamo-backup-to-s3/master/img/dynamo-restore-from-s3.png)
+
+## Restore S3 backups back to Dynamo.
+
+`dynamo-restore-from-s3` is a utility that restores backups in S3 back to dynamo. It streams data down from S3 and throttles the download speed to match the rate of batch writes to Dynamo. 
+
+It is suitable for restoring large tables without needing to write to disk or use a large amount of memory. Use it on an AWS EC2 instance for best results and to minimise network latency, this should yield restore speeds of around 15min per GB.
+
+Use `--overwrite` if the table already exists. Otherwise it will attempt to generate table on the fly.
+
+Can be run as a command line script or as an npm module. 
+
+# Command line usage
+
+```
+  Usage: dynamo-restore-from-s3 [options] -s "s3://mybucket/path/to/file.json" -t "new-dynamodb-table"
+
+  Options:
+
+    -h, --help                        output usage information
+    -V, --version                     output the version number
+    -s, --source [path]               Full S3 path to a JSON backup file (Required)
+    -t, --table [name]                Name of the Dynamo Table to restore to (Required)
+    -o, --overwrite                   Table already exists, skip auto-create. Default is false.
+    -c, --concurrency <requestcount>  Number of concurrent requests & dynamo capacity units. Defaults to 200.
+    -pk, --partitionkey [columnname]  Name of Primary Partition Key. If not provided will try determine from backup.
+    -sk, --sortkey [columnname]       Name of Secondary Sort Key. Ignored unless --partitionkey is provided.
+    -rc, --readcapacity <units>       Read Units for new table (when finished). Default is 5.
+    -wc, --writecapacity <units>      Write Units for new table (when finished). Default is --concurrency.
+    -sf, --stop-on-failure            Stop process when the same batch fails to restore multiple times. Defaults to false.
+    --aws-key <key>                   AWS access key. Will use AWS_ACCESS_KEY_ID env var if --aws-key not set
+    --aws-secret <secret>             AWS secret key. Will use AWS_SECRET_ACCESS_KEY env var if --aws-secret not set
+    --aws-region <region>             AWS region. Will use AWS_DEFAULT_REGION env var if --aws-region not set
+```
+
+## Examples
+
+```
+
+    # Restore over existing table (cmd.exe).
+    > node ./bin/dynamo-restore-from-s3 -t acme-customers -s s3://my-backups/acme-customers.json --overwrite 
+
+    # Restore over existing table (shell).
+    $ ./bin/dynamo-restore-from-s3 -t acme-customers -s s3://my-backups/acme-customers.json --overwrite 
+
+    # Restore over existing table, 1000 concurrent requests. Stop if any batch fails 1000 times.
+    $ ./bin/dynamo-restore-from-s3 -t acme-customers -c 1000 -s s3://my-backups/acme-customers.json --overwrite -sf
+
+    # Restore over existing table, 1000 concurrent requests. When finished, set read capacity to 50 and write capacity to 10 (both needed).
+    $ ./bin/dynamo-restore-from-s3 -t acme-customers -c 1000 -s s3://my-backups/acme-customers.json --overwrite --readcapacity 50 --writecapacity 10
+
+    # Auto-generate table (determine PK from backup). 
+    $ ./bin/dynamo-restore-from-s3 -t acme-customers -s s3://my-backups/acme-customers.json
+
+    # Auto-generate table with partition and sort key.
+    $ ./bin/dynamo-restore-from-s3 -t acme-orders -s s3://my-backups/acme-orders.json -pk customerId -sk createDate 
+
+    # Auto-generate table, defined PK. Concurrency 2000 (~ 2GB backup).
+    $ ./bin/dynamo-restore-from-s3 -t acme-orders -pk orderId -c 2000 -s s3://my-backups/acme-orders.json 
+
+    # Auto-generate table. 2000 write units during restore. When finished set 50 write units and 100 write units (both needed).
+    $ ./bin/dynamo-restore-from-s3 -t acme-orders -c 2000 -s s3://my-backups/acme-orders.json --readcapacity 100 --writecapacity 50
+
+    # Auto-generate table. Concurrency 50 (10 MB backup or less).
+    $ ./bin/dynamo-restore-from-s3 -t acme-orders -c 50 -s s3://my-backups/acme-orders.json 
+
+    # Auto-generate table. Concurrency 50. Stop process if any batch fails 50 times.
+    $ ./bin/dynamo-restore-from-s3 -t acme-orders -c 50 -sf -s s3://my-backups/acme-orders.json 
+
+```
+
+# npm module usage
+
+## Quick Example
+
+```
+var DynamoRestore = require('dynamo-backup-to-s3').Restore;
+
+var restore = new DynamoRestore({
+    source: 's3://my-backups/DynamoDB-backup-2016-09-28-15-36-40/acme-customers-prod.json',
+    table: 'acme-customers-dev',
+    overwrite: true,
+    concurrency: 200, // for large restores use 1 unit per MB as a rule of thumb (ie 1000 for 1GB restore)
+    awsAccessKey: /* AWS access key */,
+    awsSecretKey: /* AWS secret key */,
+    awsRegion: /* AWS region */
+});
+
+restore.on('error', function(message) {
+    console.log(message);
+    process.exit(-1);
+});
+
+restore.on('warning', function(message) {
+    console.log(message);
+});
+
+restore.on('send-batch', function(batches, requests, streamMeta) {
+    console.log('Batch sent. %d in flight. %d Mb remaining to download...', requests, streamMeta.RemainingLength / (1024 * 1024));
+});
+
+restore.run(function() {
+    console.log('Finished restoring DynamoDB table');
+});
+
+```
+
+### Constructor
+
+```
+var options = {
+    source: /* path to json file in s3 bucket, should start with s3://bucketname/... */,
+    table: /* name of dynamo table, will be created on the fly unless overwritten */,
+    overwrite: /* true/false if table already exits (defaults to false) */
+    concurrency: /* number of concurrent requests (and dynamo write capacity units) */,
+    partitionkey: /* name of partition key column */,
+    sortkey: /* name of secondary (sort) key column */ ,
+    readcapacity: /* number of read capacity units (when restore finishes) */,
+    writecapacity: /* number of write capacity units (when restore finishes) */,
+    stopOnFailure: /* true/false should a single failed batch stop the whole restore job? */,
+    awsAccessKey: /* AWS access key */,
+    awsSecretKey: /* AWS secret key */,
+    awsRegion: /* AWS region */
+};
+
+var restore = new DynamoBackup.Restore(options);
+```
+
+## Events
+
+### error
+
+Raised when there is a fatal error restoring a table
+
+__Example__
+```
+restore.on('error', function() {
+    console.log('Error!! + ' + message);
+});
+```
+
+### warning
+
+Raised when there is a warning restoring a table. Normally this will be a failed batch.
+
+__Example__
+```
+restore.on('warning', function() {
+    console.log('Warning!! + ' + message);
+});
+```
+
+### send-batch
+
+Raised whenever a batch is sent to Dynamo. Useful for tracking progress.
+
+__Example__
+```
+restore.on('send-batch', function(batches, requests, streamMeta) {
+    console.log('Batch Sent');
+    console.log('Num cached batches: ', batches); 
+    console.log('Num requests in flight: ', requests);
+    console.log('Stream metadata:, JSON.stringify(streamMeta));
+});
+```
+### finish
+Raised when the restore process is finished.
+
+__Example__
+```
+restore.on('finish', function() {
+    console.log('All done!');
+});
+```
+
+## Functions
+
+### run
+
+Restores table with options as defined in constructor.
+
+__Arguments__
+
+* `callback(err)` - callback to execute when restore job is complete. First argument exists only if there is an error.
+
+__Example__
+```
+restore.run(function(error) {
+    if (error) {
+        return console.log(error);
+    }
+    console.log('All done!');
+});
+```

bin/dynamo-backup-to-s3

@@ -3,7 +3,7 @@
 var program = require('commander'),
     fs = require('fs'),
     moment = require('moment'),
-    DynamoBackup = require('../')
+    DynamoBackup = require('../');
 
 // options
 

bin/dynamo-restore-from-s3

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+
+var program = require('commander'),
+    fs = require('fs'),
+    DynamoRestore = require('../').Restore;
+
+program
+    .version(JSON.parse(fs.readFileSync(__dirname + '/../package.json', 'utf8')).version)
+    .usage('[options] -s "s3://mybucket/path/to/file.json" -t "new-dynamodb-table"')
+    .option('-s, --source [path]', 'Full S3 path to a JSON backup file (Required)')
+    .option('-t, --table [name]', 'Name of the Dynamo Table to restore to (Required)')
+    .option('-o, --overwrite', 'Table already exists, skip auto-create. Default is false.')
+    .option('-c, --concurrency <requestcount>', 'Number of concurrent requests & dynamo capacity units. Defaults to 200.')
+    .option('-pk, --partitionkey [columnname]', 'Name of Primary Partition Key. If not provided will try determine from backup.')
+    .option('-sk, --sortkey [columnname]', 'Name of Secondary Sort Key. Ignored unless --partitionkey is provided.')
+    .option('-rc, --readcapacity <units>', 'Read Units for new table (when finished). Default is 5.')
+    .option('-wc, --writecapacity <units>', 'Write Units for new table (when finished). Default is --concurrency.')
+    .option('-sf, --stop-on-failure', 'Stop process when the same batch fails to restore 3 times. Defaults to false.', true)
+    .option('--aws-key <key>', 'AWS access key. Will use AWS_ACCESS_KEY_ID env var if --aws-key not set')
+    .option('--aws-secret <secret>', 'AWS secret key. Will use AWS_SECRET_ACCESS_KEY env var if --aws-secret not set')
+    .option('--aws-region <region>', 'AWS region. Will use AWS_DEFAULT_REGION env var if --aws-region not set')
+    .parse(process.argv);
+
+// Display help if needed
+if (!program.source || !program.table) {
+    program.outputHelp();
+    process.exit(-1);
+    return;
+}
+
+var runTimes = {
+    start: new Date().getTime()
+};
+
+// Initialize
+var dynamoRestore = new DynamoRestore({
+    // Main settings
+    source: program.source,
+    table: program.table,
+    overwrite: !!program.overwrite,
+    concurrency: program.concurrency,
+    stopOnFailure: !!program.stopOnFailure,
+    // New table properties
+    partitionkey: program.partitionkey,
+    sortkey: program.sortkey,
+    readcapacity: program.readcapacity,
+    writecapacity: program.writecapacity,
+    // Authentication
+    awsKey: program.awsKey,
+    awsSecret: program.awsSecret,
+    awsRegion: program.awsRegion,
+});
+
+function translate(contentLength) {
+    var kb = contentLength / 1024,
+        mb = kb / 1024,
+        gb = mb / 1024;
+    return gb > 5 ? gb.toFixed(0) + ' Gb' :
+        (mb > 5 ? mb.toFixed(0) + 'Mb' :
+        kb.toFixed(0) + 'Kb');
+}
+
+// Define events
+dynamoRestore.on('error', function(message) {
+    console.log(message);
+    process.exit(-1);
+});
+
+dynamoRestore.on('warning', function(message) {
+    console.log(message);
+});
+
+dynamoRestore.on('start-download', function(streamMeta) {
+    var time = runTimes.startDownload = new Date().getTime();
+    console.log('Starting download. %s remaining...', translate(streamMeta.ContentLength));
+});
+
+dynamoRestore.on('send-batch', function(batches, requests, streamMeta) {
+    console.log('Batch sent. %d in flight. %s remaining to download...', requests, translate(streamMeta.RemainingLength));
+});
+
+// Start Process
+dynamoRestore.run(function() {
+    var time = runTimes.end = new Date().getTime(),
+        diff = time - runTimes.start,
+        minutes = Math.floor(diff / (1000 * 60)),
+        seconds = Math.floor((diff % 1000 * 60) / 1000);
+    console.log('Done! Process completed in %s minutes %s seconds.', minutes, seconds);
+    process.exit(0);
+});

img/dynamo-backup-to-s3.png

@@ -1,3 +1,6 @@
 var DynamoBackup = require('./lib/dynamo-backup');
+var DynamoRestore = require('./lib/dynamo-restore');
+
+DynamoBackup.Restore = DynamoRestore;
 
 module.exports = DynamoBackup;