Skip to content

Commit dc420c8

Browse files
rehrumeshrehrumesh
authored
Merge pull request #77 from darthtrevino/seeding
Implement v1 branch seeding
2 parents 2f83524 + e67c274 commit dc420c8

File tree

3 files changed

+290
-95
lines changed

3 files changed

+290
-95
lines changed

‎README.md

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,8 @@ All CLI options are optional:
5252
--sharedDb -h DynamoDB will use a single database file, instead of using separate files for each credential and region. If you specify -sharedDb, all DynamoDB clients will interact with the same set of tables regardless of their region and credential configuration.
5353
--delayTransientStatuses -t Causes DynamoDB to introduce delays for certain operations. DynamoDB can perform some tasks almost instantaneously, such as create/update/delete operations on tables and indexes; however, the actual DynamoDB service requires more time for these tasks. Setting this parameter helps DynamoDB simulate the behavior of the Amazon DynamoDB web service more closely. (Currently, this parameter introduces delays only for global secondary indexes that are in either CREATING or DELETING status.)
5454
--optimizeDbBeforeStartup -o Optimizes the underlying database tables before starting up DynamoDB on your computer. You must also specify -dbPath when you use this parameter.
55-
--migrate -m After starting DynamoDB local, create DynamoDB tables from the Serverless configuration..
55+
--migrate -m After starting DynamoDB local, create DynamoDB tables from the Serverless configuration.
56+
--seed -s After starting and migrating dynamodb local, injects seed data into your tables. The --seed option determines which data categories to onload.
5657
```
5758

5859
All the above options can be added to serverless.yml to set default configuration: e.g.
@@ -64,10 +65,11 @@ custom:
6465
port: 8000
6566
inMemory: true
6667
migrate: true
68+
seed: true
6769
```
6870

6971
## Migrations: sls dynamodb migrate
70-
### Configurations
72+
### Configuration
7173
In `serverless.yml` add following to execute all the migration upon DynamoDB Local Start
7274
```yml
7375
custom:
@@ -94,6 +96,38 @@ resources:
9496
WriteCapacityUnits: 1
9597
```
9698

99+
## Seeding: sls dynamodb seed
100+
### Configuration
101+
102+
In `serverless.yml` seeding categories are defined under `dynamodb.seed`.
103+
104+
If `dynamodb.start.seed` is true, then seeding is performed after table migrations.
105+
106+
```yml
107+
dynamodb:
108+
start:
109+
seed: true
110+
111+
seed:
112+
domain:
113+
sources:
114+
- table: domain-widgets
115+
sources: [./domainWidgets.json]
116+
- table: domain-fidgets
117+
sources: [./domainFidgets.json]
118+
test:
119+
sources:
120+
- table: users
121+
sources: [./fake-test-users.json]
122+
- table: subscriptions
123+
sources: [./fake-test-subscriptions.json]
124+
```
125+
126+
```bash
127+
> sls dynamodb seed --seed=domain,test
128+
> sls dynamodb start --seed=domain,test
129+
```
130+
97131
## Using DynamoDB Local in your code
98132
You need to add the following parameters to the AWS NODE SDK dynamodb constructor
99133

‎index.js

Lines changed: 124 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -1,73 +1,82 @@
1-
'use strict';
2-
3-
const _ = require('lodash'),
4-
BbPromise = require('bluebird'),
5-
AWS = require('aws-sdk'),
6-
dynamodbLocal = require('dynamodb-localhost');
1+
"use strict";
2+
const _ = require("lodash");
3+
const BbPromise = require("bluebird");
4+
const AWS = require("aws-sdk");
5+
const dynamodbLocal = require("dynamodb-localhost");
6+
const seeder = require("./src/seeder");
77

88
class ServerlessDynamodbLocal {
99
constructor(serverless, options) {
1010
this.serverless = serverless;
1111
this.service = serverless.service;
12+
this.serverlessLog = serverless.cli.log.bind(serverless.cli);
1213
this.config = this.service.custom && this.service.custom.dynamodb || {};
1314
this.options = options;
14-
this.provider = 'aws';
15+
this.provider = "aws";
1516
this.commands = {
1617
dynamodb: {
1718
commands: {
1819
migrate: {
19-
lifecycleEvents: ['migrateHandler'],
20-
usage: 'Creates local DynamoDB tables from the current Serverless configuration'
20+
lifecycleEvents: ["migrateHandler"],
21+
usage: "Creates local DynamoDB tables from the current Serverless configuration"
22+
},
23+
seed: {
24+
lifecycleEvents: ["seedHandler"],
25+
usage: "Seeds local DynamoDB tables with data"
2126
},
2227
start: {
23-
lifecycleEvents: ['startHandler'],
24-
usage: 'Starts local DynamoDB',
28+
lifecycleEvents: ["startHandler"],
29+
usage: "Starts local DynamoDB",
2530
options: {
2631
port: {
27-
shortcut: 'p',
28-
usage: 'The port number that DynamoDB will use to communicate with your application. If you do not specify this option, the default port is 8000'
32+
shortcut: "p",
33+
usage: "The port number that DynamoDB will use to communicate with your application. If you do not specify this option, the default port is 8000"
2934
},
3035
cors: {
31-
shortcut: 'c',
32-
usage: 'Enable CORS support (cross-origin resource sharing) for JavaScript. You must provide a comma-separated "allow" list of specific domains. The default setting for -cors is an asterisk (*), which allows public access.'
36+
shortcut: "c",
37+
usage: "Enable CORS support (cross-origin resource sharing) for JavaScript. You must provide a comma-separated \"allow\" list of specific domains. The default setting for -cors is an asterisk (*), which allows public access."
3338
},
3439
inMemory: {
35-
shortcut: 'i',
36-
usage: 'DynamoDB; will run in memory, instead of using a database file. When you stop DynamoDB;, none of the data will be saved. Note that you cannot specify both -dbPath and -inMemory at once.'
40+
shortcut: "i",
41+
usage: "DynamoDB; will run in memory, instead of using a database file. When you stop DynamoDB;, none of the data will be saved. Note that you cannot specify both -dbPath and -inMemory at once."
3742
},
3843
dbPath: {
39-
shortcut: 'd',
40-
usage: 'The directory where DynamoDB will write its database file. If you do not specify this option, the file will be written to the current directory. Note that you cannot specify both -dbPath and -inMemory at once. For the path, current working directory is <projectroot>/node_modules/serverless-dynamodb-local/dynamob. For example to create <projectroot>/node_modules/serverless-dynamodb-local/dynamob/<mypath> you should specify -d <mypath>/ or --dbPath <mypath>/ with a forwardslash at the end.'
44+
shortcut: "d",
45+
usage: "The directory where DynamoDB will write its database file. If you do not specify this option, the file will be written to the current directory. Note that you cannot specify both -dbPath and -inMemory at once. For the path, current working directory is <projectroot>/node_modules/serverless-dynamodb-local/dynamob. For example to create <projectroot>/node_modules/serverless-dynamodb-local/dynamob/<mypath> you should specify -d <mypath>/ or --dbPath <mypath>/ with a forwardslash at the end."
4146
},
4247
sharedDb: {
43-
shortcut: 'h',
44-
usage: 'DynamoDB will use a single database file, instead of using separate files for each credential and region. If you specify -sharedDb, all DynamoDB clients will interact with the same set of tables regardless of their region and credential configuration.'
48+
shortcut: "h",
49+
usage: "DynamoDB will use a single database file, instead of using separate files for each credential and region. If you specify -sharedDb, all DynamoDB clients will interact with the same set of tables regardless of their region and credential configuration."
4550
},
4651
delayTransientStatuses: {
47-
shortcut: 't',
48-
usage: 'Causes DynamoDB to introduce delays for certain operations. DynamoDB can perform some tasks almost instantaneously, such as create/update/delete operations on tables and indexes; however, the actual DynamoDB service requires more time for these tasks. Setting this parameter helps DynamoDB simulate the behavior of the Amazon DynamoDB web service more closely. (Currently, this parameter introduces delays only for global secondary indexes that are in either CREATING or DELETING status.'
52+
shortcut: "t",
53+
usage: "Causes DynamoDB to introduce delays for certain operations. DynamoDB can perform some tasks almost instantaneously, such as create/update/delete operations on tables and indexes; however, the actual DynamoDB service requires more time for these tasks. Setting this parameter helps DynamoDB simulate the behavior of the Amazon DynamoDB web service more closely. (Currently, this parameter introduces delays only for global secondary indexes that are in either CREATING or DELETING status."
4954
},
5055
optimizeDbBeforeStartup: {
51-
shortcut: 'o',
52-
usage: 'Optimizes the underlying database tables before starting up DynamoDB on your computer. You must also specify -dbPath when you use this parameter.'
56+
shortcut: "o",
57+
usage: "Optimizes the underlying database tables before starting up DynamoDB on your computer. You must also specify -dbPath when you use this parameter."
5358
},
5459
migrate: {
55-
shortcut: 'm',
56-
usage: 'After starting dynamodb local, create DynamoDB tables from the current serverless configuration'
60+
shortcut: "m",
61+
usage: "After starting dynamodb local, create DynamoDB tables from the current serverless configuration."
62+
},
63+
seed: {
64+
shortcut: "s",
65+
usage: "After starting and migrating dynamodb local, injects seed data into your tables. The --seed option determines which data categories to onload.",
5766
}
5867
}
5968
},
6069
remove: {
61-
lifecycleEvents: ['removeHandler'],
62-
usage: 'Removes local DynamoDB'
70+
lifecycleEvents: ["removeHandler"],
71+
usage: "Removes local DynamoDB"
6372
},
6473
install: {
65-
usage: 'Installs local DynamoDB',
66-
lifecycleEvents: ['installHandler'],
74+
usage: "Installs local DynamoDB",
75+
lifecycleEvents: ["installHandler"],
6776
options: {
6877
localPath: {
69-
shortcut: 'x',
70-
usage: 'Local dynamodb install path'
78+
shortcut: "x",
79+
usage: "Local dynamodb install path"
7180
}
7281
}
7382

@@ -77,23 +86,29 @@ class ServerlessDynamodbLocal {
7786
};
7887

7988
this.hooks = {
80-
'dynamodb:migrate:migrateHandler': this.migrateHandler.bind(this),
81-
'dynamodb:remove:removeHandler': this.removeHandler.bind(this),
82-
'dynamodb:install:installHandler': this.installHandler.bind(this),
83-
'dynamodb:start:startHandler': this.startHandler.bind(this),
84-
'before:offline:start': this.startHandler.bind(this),
89+
"dynamodb:migrate:migrateHandler": this.migrateHandler.bind(this),
90+
"dynamodb:migrate:seedHandler": this.seedHandler.bind(this),
91+
"dynamodb:remove:removeHandler": this.removeHandler.bind(this),
92+
"dynamodb:install:installHandler": this.installHandler.bind(this),
93+
"dynamodb:start:startHandler": this.startHandler.bind(this),
94+
"before:offline:start:init": this.startHandler.bind(this),
95+
"before:offline:start:end": this.endHandler.bind(this),
8596
};
8697
}
8798

99+
get port() {
100+
const config = this.config;
101+
const port = _.get(config, "start.port", 8000);
102+
return port;
103+
}
104+
88105
dynamodbOptions() {
89-
let self = this;
90-
let port = self.config.start && self.config.start.port || 8000,
91-
dynamoOptions = {
92-
endpoint: 'http://localhost:' + port,
93-
region: 'localhost',
94-
accessKeyId: 'MOCK_ACCESS_KEY_ID',
95-
secretAccessKey: 'MOCK_SECRET_ACCESS_KEY'
96-
};
106+
const dynamoOptions = {
107+
endpoint: `http://localhost:${this.port}`,
108+
region: "localhost",
109+
accessKeyId: "MOCK_ACCESS_KEY_ID",
110+
secretAccessKey: "MOCK_SECRET_ACCESS_KEY"
111+
};
97112

98113
return {
99114
raw: new AWS.DynamoDB(dynamoOptions),
@@ -102,74 +117,90 @@ class ServerlessDynamodbLocal {
102117
}
103118

104119
migrateHandler() {
105-
let self = this;
106-
107-
return new BbPromise(function (resolve, reject) {
108-
let dynamodb = self.dynamodbOptions();
109-
110-
var tables = self.resourceTables();
120+
const dynamodb = this.dynamodbOptions();
121+
const tables = this.tables;
122+
return BbPromise.each(tables, (table) => this.createTable(dynamodb, table));
123+
}
111124

112-
return BbPromise.each(tables, function (table) {
113-
return self.createTable(dynamodb, table);
114-
}).then(resolve, reject);
125+
seedHandler() {
126+
const documentClient = this.dynamodbOptions().doc;
127+
const seedSources = this.seedSources;
128+
return BbPromise.each(seedSources, (source) => {
129+
if (!source.table) {
130+
throw new Error("seeding source \"table\" property not defined");
131+
}
132+
return seeder.locateSeeds(source.sources || [])
133+
.then((seeds) => seeder.writeSeeds(documentClient, source.table, seeds));
115134
});
116135
}
117136

118137
removeHandler() {
119-
return new BbPromise(function (resolve) {
120-
dynamodbLocal.remove(resolve);
121-
});
138+
return new BbPromise((resolve) => dynamodbLocal.remove(resolve));
122139
}
123140

124141
installHandler() {
125-
let options = this.options;
126-
return new BbPromise(function (resolve) {
127-
dynamodbLocal.install(resolve, options.localPath);
128-
});
142+
const options = this.options;
143+
return new BbPromise((resolve) => dynamodbLocal.install(resolve, options.localPath));
129144
}
130145

131146
startHandler() {
132-
let self = this;
133-
return new BbPromise(function (resolve) {
134-
let options = _.merge({
135-
sharedDb: self.options.sharedDb || true
136-
},
137-
self.options,
138-
self.config && self.config.start
139-
);
140-
if (options.migrate) {
141-
dynamodbLocal.start(options);
142-
console.log(""); // seperator
143-
self.migrateHandler(true);
144-
resolve();
145-
} else {
146-
dynamodbLocal.start(options);
147-
console.log("");
148-
resolve();
149-
}
150-
});
147+
const config = this.config;
148+
const options = _.merge({
149+
sharedDb: this.options.sharedDb || true
150+
},
151+
this.options,
152+
config && config.start
153+
);
154+
155+
dynamodbLocal.start(options);
156+
return BbPromise.resolve()
157+
.then(() => options.migrate && this.migrateHandler())
158+
.then(() => options.seed && this.seedHandler());
151159
}
152160

153-
resourceTables() {
154-
var resources = this.service.resources.Resources;
155-
return Object.keys(resources).map(function (key) {
156-
if (resources[key].Type == 'AWS::DynamoDB::Table') {
161+
endHandler() {
162+
this.serverlessLog('DynamoDB - stopping local database');
163+
dynamodbLocal.stop(this.port);
164+
}
165+
166+
/**
167+
* Gets the table definitions
168+
*/
169+
get tables() {
170+
const resources = this.service.resources.Resources;
171+
return Object.keys(resources).map((key) => {
172+
if (resources[key].Type === "AWS::DynamoDB::Table") {
157173
return resources[key].Properties;
158174
}
159-
}).filter(n => {
160-
return n;
161-
});
175+
}).filter((n) => n);
176+
}
177+
178+
/**
179+
* Gets the seeding sources
180+
*/
181+
get seedSources() {
182+
const config = this.service.custom.dynamodb;
183+
const seedConfig = _.get(config, "seed", {});
184+
const seed = this.options.seed;
185+
if (!seed) {
186+
this.serverlessLog("DynamoDB - No seed categories defined. Skipping data seeding.");
187+
return [];
188+
}
189+
const categories = seed.split(",");
190+
const sourcesByCategory = categories.map((category) => seedConfig[category].sources);
191+
return [].concat.apply([], sourcesByCategory);
162192
}
163193

164194
createTable(dynamodb, migration) {
165-
return new BbPromise(function (resolve) {
166-
dynamodb.raw.createTable(migration, function (err) {
195+
return new BbPromise((resolve, reject) => {
196+
dynamodb.raw.createTable(migration, (err) => {
167197
if (err) {
168-
console.log(err);
198+
this.serverlessLog("DynamoDB - Error - ", err);
199+
reject(err);
169200
} else {
170-
console.log("Table creation completed for table: " + migration.TableName);
201+
this.serverlessLog("DynamoDB - created table " + migration.TableName);
202+
resolve(migration);
171203
}
172-
resolve(migration);
173204
});
174205
});
175206
}

0 commit comments

Comments
 (0)