Skip to content

Commit d6a7d21

Browse files
darthtrevinodarthtrevino
committed
Implement v1 branch seeding
1 parent 42bf35e commit d6a7d21

File tree

2 files changed

+185
-60
lines changed

2 files changed

+185
-60
lines changed

‎index.js

Lines changed: 72 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
'use strict';
2-
3-
const _ = require('lodash'),
4-
BbPromise = require('bluebird'),
5-
AWS = require('aws-sdk'),
6-
dynamodbLocal = require('dynamodb-localhost');
2+
const _ = require('lodash');
3+
const BbPromise = require('bluebird');
4+
const AWS = require('aws-sdk');
5+
const dynamodbLocal = require('dynamodb-localhost');
6+
const { writeSeeds, locateSeeds } = require('./src/seeder');
77

88
class ServerlessDynamodbLocal {
99
constructor(serverless, options) {
@@ -18,6 +18,10 @@ class ServerlessDynamodbLocal {
1818
lifecycleEvents: ['migrateHandler'],
1919
usage: 'Creates local DynamoDB tables from the current Serverless configuration'
2020
},
21+
seed: {
22+
lifecycleEvents: ['seedHandler'],
23+
usage: 'Seeds local DynamoDB tables with data'
24+
},
2125
start: {
2226
lifecycleEvents: ['startHandler'],
2327
usage: 'Starts local DynamoDB',
@@ -53,6 +57,10 @@ class ServerlessDynamodbLocal {
5357
migrate: {
5458
shortcut: 'm',
5559
usage: 'After starting dynamodb local, create DynamoDB tables from the current serverless configuration'
60+
},
61+
seed: {
62+
shortcut: 's',
63+
usage: 'After starting and migrating dynamodb local, injects seed data into your tables',
5664
}
5765
}
5866
},
@@ -77,23 +85,23 @@ class ServerlessDynamodbLocal {
7785

7886
this.hooks = {
7987
'dynamodb:migrate:migrateHandler': this.migrateHandler.bind(this),
88+
'dynamodb:migrate:seedHandler': this.seedHandler.bind(this),
8089
'dynamodb:remove:removeHandler': this.removeHandler.bind(this),
8190
'dynamodb:install:installHandler': this.installHandler.bind(this),
8291
'dynamodb:start:startHandler': this.startHandler.bind(this),
83-
'before:offline:start': this.startHandler.bind(this),
92+
'before:offline:start:init': this.startHandler.bind(this),
8493
};
8594
}
8695

8796
dynamodbOptions() {
88-
let self = this;
89-
let config = self.service.custom.dynamodb || {},
90-
port = config.start && config.start.port || 8000,
91-
dynamoOptions = {
92-
endpoint: 'http://localhost:' + port,
93-
region: 'localhost',
94-
accessKeyId: 'MOCK_ACCESS_KEY_ID',
95-
secretAccessKey: 'MOCK_SECRET_ACCESS_KEY'
96-
};
97+
const config = this.service.custom.dynamodb || {};
98+
const port = config.start && config.start.port || 8000;
99+
const dynamoOptions = {
100+
endpoint: 'http://localhost:' + port,
101+
region: 'localhost',
102+
accessKeyId: 'MOCK_ACCESS_KEY_ID',
103+
secretAccessKey: 'MOCK_SECRET_ACCESS_KEY'
104+
};
97105

98106
return {
99107
raw: new AWS.DynamoDB(dynamoOptions),
@@ -102,75 +110,79 @@ class ServerlessDynamodbLocal {
102110
}
103111

104112
migrateHandler() {
105-
let self = this;
106-
107-
return new BbPromise(function (resolve, reject) {
108-
let dynamodb = self.dynamodbOptions();
109-
110-
var tables = self.resourceTables();
113+
const dynamodb = this.dynamodbOptions();
114+
const { tables } = this;
115+
return BbPromise.each(tables, table => this.createTable(dynamodb, table));
116+
}
111117

112-
return BbPromise.each(tables, function (table) {
113-
return self.createTable(dynamodb, table);
114-
}).then(resolve, reject);
118+
seedHandler() {
119+
const { doc: documentClient } = this.dynamodbOptions();
120+
const { seedSources } = this;
121+
return BbPromise.each(seedSources, source => {
122+
if (!source.table) {
123+
throw new Error('seeding source "table" not defined');
124+
}
125+
return locateSeeds(source.sources || [])
126+
.then((seeds) => writeSeeds(documentClient, source.table, seeds));
115127
});
116128
}
117129

118130
removeHandler() {
119-
return new BbPromise(function (resolve) {
120-
dynamodbLocal.remove(resolve);
121-
});
131+
return new BbPromise(resolve => dynamodbLocal.remove(resolve));
122132
}
123133

124134
installHandler() {
125-
let options = this.options;
126-
return new BbPromise(function (resolve) {
127-
dynamodbLocal.install(resolve, options.localPath);
128-
});
135+
const { options } = this;
136+
return new BbPromise((resolve) => dynamodbLocal.install(resolve, options.localPath));
129137
}
130138

131139
startHandler() {
132-
let self = this;
133-
return new BbPromise(function (resolve) {
134-
let config = self.service.custom.dynamodb,
135-
options = _.merge({
136-
sharedDb: self.options.sharedDb || true
137-
},
138-
self.options,
139-
config && config.start
140-
);
141-
if (options.migrate) {
142-
dynamodbLocal.start(options);
143-
console.log(""); // seperator
144-
self.migrateHandler(true);
145-
resolve();
146-
} else {
147-
dynamodbLocal.start(options);
148-
console.log("");
149-
resolve();
150-
}
151-
});
140+
const config = this.service.custom.dynamodb;
141+
const options = _.merge({
142+
sharedDb: this.options.sharedDb || true
143+
},
144+
this.options,
145+
config && config.start
146+
);
147+
148+
dynamodbLocal.start(options);
149+
console.log(""); // separator
150+
151+
return BbPromise.resolve()
152+
.then(() => options.migrate && this.migrateHandler())
153+
.then(() => options.seed && this.seedHandler());
152154
}
153155

154-
resourceTables() {
155-
var resources = this.service.resources.Resources;
156-
return Object.keys(resources).map(function (key) {
156+
/**
157+
* Gets the table definitions
158+
*/
159+
get tables() {
160+
const resources = this.service.resources.Resources;
161+
return Object.keys(resources).map((key) => {
157162
if (resources[key].Type == 'AWS::DynamoDB::Table') {
158163
return resources[key].Properties;
159164
}
160-
}).filter(n => {
161-
return n;
162-
});
165+
}).filter(n => n);
166+
}
167+
168+
/**
169+
* Gets the seeding sources
170+
*/
171+
get seedSources() {
172+
const config = this.service.custom.dynamodb;
173+
return _.get(config, 'start.seeds', []);
163174
}
164175

165176
createTable(dynamodb, migration) {
166-
return new BbPromise(function (resolve) {
167-
dynamodb.raw.createTable(migration, function (err) {
177+
return new BbPromise((resolve, reject) => {
178+
dynamodb.raw.createTable(migration, (err) => {
168179
if (err) {
169180
console.log(err);
181+
reject(err);
170182
} else {
171183
console.log("Table creation completed for table: " + migration.TableName);
184+
resolve(migration);
172185
}
173-
resolve(migration);
174186
});
175187
});
176188
}

‎src/seeder.js

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
const AWS = require('aws-sdk');
2+
const BbPromise = require('bluebird');
3+
const _ = require('lodash');
4+
const path = require('path');
5+
const fs = require('fs');
6+
7+
// DynamoDB has a 25 item limit in batch requests
8+
// https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html
9+
const MAX_MIGRATION_CHUNK = 25;
10+
11+
// TODO: let this be configurable
12+
const MIGRATION_SEED_CONCURRENCY = 5;
13+
14+
/**
15+
* Writes a batch chunk of migration seeds to DynamoDB. DynamoDB has a limit on the number of
16+
* items that may be written in a batch operation.
17+
* @param {DynamoDocumentClient} dynamodb The DynamoDB Document client
18+
* @param {string} tableName The table name being written to
19+
* @param {any[]} seeds The migration seeds being written to the table
20+
*/
21+
function writeSeedBatch(dynamodb, tableName, seeds) {
22+
const params = {
23+
RequestItems: {
24+
[tableName]: seeds.map((seed) => ({
25+
PutRequest: {
26+
Item: seed,
27+
},
28+
})),
29+
},
30+
};
31+
return new BbPromise((resolve, reject) => {
32+
// interval lets us know how much time we've burnt so far. This lets us have a backoff mechanism to try
33+
// again a few times in case the Database resources are in the middle of provisioning.
34+
let interval = 0;
35+
function execute(interval) {
36+
setTimeout(() => dynamodb.batchWrite(params, (err) => {
37+
if (err) {
38+
if (err.code === "ResourceNotFoundException" && interval <= 5000) {
39+
execute(interval + 1000);
40+
} else {
41+
reject(err);
42+
}
43+
} else {
44+
resolve();
45+
}
46+
}), interval);
47+
};
48+
execute(interval);
49+
});
50+
};
51+
52+
/**
53+
* Writes a seed corpus to the given database table
54+
* @param {DocumentClient} dynamodb The DynamoDB document instance
55+
* @param {string} tableName The table name
56+
* @param {any[]} seeds The seed values
57+
*/
58+
function writeSeeds(dynamodb, tableName, seeds) {
59+
if (!dynamodb) {
60+
throw new Error('dynamodb argument must be provided');
61+
}
62+
if (!tableName) {
63+
throw new Error('table name argument must be provided');
64+
}
65+
if (!seeds) {
66+
throw new Error('seeds argument must be provided');
67+
}
68+
69+
if (seeds.length > 0) {
70+
const seedChunks = _.chunk(seeds, MAX_MIGRATION_CHUNK);
71+
return BbPromise.map(
72+
seedChunks,
73+
chunk => writeSeedBatch(dynamodb, tableName, chunk),
74+
{ concurrency: MIGRATION_SEED_CONCURRENCY }
75+
)
76+
.then(() => console.log("Seed running complete for table: " + tableName));
77+
}
78+
}
79+
80+
function fileExists(fileName) {
81+
return new BbPromise((resolve) => {
82+
fs.exists(fileName, (exists) => resolve(exists));
83+
});
84+
}
85+
86+
/**
87+
* Locates seeds given a set of files to scrape
88+
* @param {string[]} sources The filenames to scrape for seeds
89+
*/
90+
function locateSeeds(sources = [], cwd = process.cwd()) {
91+
const locations = sources.map(source => path.join(cwd, source));
92+
return BbPromise.map(locations, (location) => {
93+
return fileExists(location)
94+
.then((exists) => {
95+
if(!exists) {
96+
throw new Error('source file ' + location + ' does not exist');
97+
}
98+
99+
// load the file as JSON
100+
const result = require(location);
101+
102+
// Ensure the output is an array
103+
if (Array.isArray(result)) {
104+
return result;
105+
} else {
106+
return [ result ];
107+
}
108+
});
109+
// Smash the arrays together
110+
}).then(seedArrays => [].concat.apply([], seedArrays));
111+
}
112+
113+
module.exports = { writeSeeds, locateSeeds };

0 commit comments

Comments
 (0)