Skip to content
This repository was archived by the owner on Dec 1, 2022. It is now read-only.

Commit a368c43

Browse files
tmitchel2tmitchel2
committed
Improvements to logging, error logging and build process.
1 parent e0ae9ae commit a368c43

File tree

11 files changed

+228
-119
lines changed

11 files changed

+228
-119
lines changed

‎gulpfile.js

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,21 +7,24 @@ var uglify = require('gulp-uglify');
77
var AWS = require('aws-sdk');
88
var fs = require('fs');
99
var runSequence = require('run-sequence');
10+
var webpack = require('webpack-stream');
1011

1112
// First we need to clean out the dist folder and remove the compiled zip file.
1213
gulp.task('clean', function(cb) {
1314
del('./dist');
1415
cb();
1516
});
1617

17-
gulp.task("webpack", function (cb) {
18-
cb();
18+
gulp.task("webpack", function () {
19+
return gulp.src('src/index.js')
20+
.pipe(webpack( require('./webpack-dev.config.js') ))
21+
.pipe(gulp.dest('build/'));
1922
});
2023

2124
// The js task could be replaced with gulp-coffee as desired.
2225
gulp.task("js", function () {
2326
return gulp
24-
.src("build/lambdaMain.js")
27+
.src("build/index.js")
2528
.pipe(gulp.dest("dist/"));
2629
});
2730

‎package.json

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,21 +4,19 @@
44
"description": "Autoscale DynamoDB using AWS Lambda",
55
"main": "index.js",
66
"scripts": {
7-
"build": "babel src -d lib",
87
"start": "node ./scripts/start.js",
98
"debug": "node-debug ./scripts/start.js",
10-
"build-dev": "webpack --config webpack-dev.config.js --progress --profile --colors",
11-
"build-prod": "webpack --config webpack-prod.config.js --progress --profile --colors",
12-
"dist": "gulp dist"
9+
"build": "gulp dist"
1310
},
1411
"author": "tmitchel2 <[email protected]>",
1512
"license": "MIT",
1613
"dependencies": {
17-
"dotenv": "^2.0.0",
1814
"winston": "^2.2.0"
1915
},
2016
"devDependencies": {
21-
"aws-sdk": "^2.3.5",
17+
"aws-sdk-promise": "0.0.2",
18+
"dotenv": "^2.0.0",
19+
"aws-sdk": "2.2.48",
2220
"babel-polyfill": "^6.7.4",
2321
"measured": "^1.1.0",
2422
"babel": "^6.5.2",
@@ -33,6 +31,7 @@
3331
"gulp-uglify": "^1.5.3",
3432
"gulp-zip": "^3.2.0",
3533
"run-sequence": "^1.1.5",
34+
"webpack-stream": "^3.2.0",
3635
"stats-webpack-plugin": "^0.3.1"
3736
},
3837
"keywords": [

‎scripts/start.js

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
try {
22
var lambda = require('../build/index.js');
33

4+
process.chdir('./build');
5+
46
var context = {
57
succeed: function(data) {
68
try {

‎src/CloudWatch.js

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import AWS from 'aws-sdk';
1+
import AWS from 'aws-sdk-promise';
22
import Global from './global';
33
const {
44
stats,
@@ -11,11 +11,15 @@ export default class CloudWatch {
1111
}
1212

1313
async getMetricStatisticsAsync(params) {
14+
logger.debug('CloudWatch.getMetricStatisticsAsync');
1415
let sw = stats.timer('CloudWatch.getMetricStatisticsAsync').start()
1516
try {
16-
return await this._cw.getMetricStatistics(params).promise();
17-
}
18-
finally {
17+
let res = await this._cw.getMetricStatistics(params).promise();
18+
return res.data;
19+
} catch (ex) {
20+
logger.warn('CloudWatch.getMetricStatisticsAsync failed', JSON.stringify({params}));
21+
throw ex;
22+
} finally {
1923
sw.end();
2024
}
2125
}

‎src/ConfigurableProvisioner.js

Lines changed: 34 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ class ConfigurableProvisioner {
1212

1313
getTableUpdate(tableDescription, tableConsumedCapacityDescription) {
1414
try {
15-
logger.debug('ConfigurableProvisioner.getTableUpdate (start)');
15+
logger.debug('ConfigurableProvisioner.getTableUpdate');
1616

1717
let tableData = {
1818
TableName: tableDescription.Table.TableName,
@@ -44,15 +44,15 @@ class ConfigurableProvisioner {
4444

4545
return result;
4646
} catch (e) {
47-
logger.warn('ConfigurableProvisioner.getTableUpdate (error)');
47+
logger.warn('ConfigurableProvisioner.getTableUpdate failed', JSON.stringify({tableDescription, tableConsumedCapacityDescription}));
4848
logger.error(e);
49-
} finally {
50-
logger.debug('ConfigurableProvisioner.getTableUpdate (complete)');
5149
}
5250
}
5351

5452
getGlobalSecondaryIndexUpdate(tableDescription, tableConsumedCapacityDescription, gsi){
5553
try {
54+
logger.debug('ConfigurableProvisioner.getGlobalSecondaryIndexUpdate');
55+
5656
let gsicc = tableConsumedCapacityDescription.Table.GlobalSecondaryIndexes.find(i => i.IndexName === gsi.IndexName);
5757
let provisionedThroughput = this.getUpdatedProvisionedThroughput({
5858
TableName: tableDescription.Table.TableName,
@@ -72,37 +72,44 @@ class ConfigurableProvisioner {
7272
}
7373
};
7474
} catch (e) {
75-
logger.warn('ConfigurableProvisioner.getGlobalSecondaryIndexUpdate (error)');
75+
logger.warn('ConfigurableProvisioner.getGlobalSecondaryIndexUpdate failed', JSON.stringify({tableDescription, tableConsumedCapacityDescription, gsi}));
7676
throw e;
7777
}
7878
}
7979

8080
getUpdatedProvisionedThroughput(params) {
81-
let newProvisionedThroughput = {
82-
ReadCapacityUnits: params.ProvisionedThroughput.ReadCapacityUnits,
83-
WriteCapacityUnits: params.ProvisionedThroughput.WriteCapacityUnits
84-
};
85-
86-
// Adjust read capacity
87-
if (this.config.readCapacity.increment.isAdjustmentRequired(params)){
88-
newProvisionedThroughput.ReadCapacityUnits = this.config.readCapacity.increment.calculateValue(params);
89-
} else if (this.config.readCapacity.decrement.isAdjustmentRequired(params)) {
90-
newProvisionedThroughput.ReadCapacityUnits = this.config.readCapacity.decrement.calculateValue(params);
91-
}
81+
try {
82+
logger.debug('ConfigurableProvisioner.getUpdatedProvisionedThroughput');
9283

93-
// Adjust write capacity
94-
if (this.config.writeCapacity.increment.isAdjustmentRequired(params)){
95-
newProvisionedThroughput.WriteCapacityUnits = this.config.writeCapacity.increment.calculateValue(params);
96-
} else if (this.config.writeCapacity.decrement.isAdjustmentRequired(params)) {
97-
newProvisionedThroughput.WriteCapacityUnits = this.config.writeCapacity.decrement.calculateValue(params);
98-
}
84+
let newProvisionedThroughput = {
85+
ReadCapacityUnits: params.ProvisionedThroughput.ReadCapacityUnits,
86+
WriteCapacityUnits: params.ProvisionedThroughput.WriteCapacityUnits
87+
};
9988

100-
if (newProvisionedThroughput.ReadCapacityUnits === params.ProvisionedThroughput.ReadCapacityUnits
101-
&& newProvisionedThroughput.WriteCapacityUnits === params.ProvisionedThroughput.WriteCapacityUnits) {
102-
return null;
103-
}
89+
// Adjust read capacity
90+
if (this.config.readCapacity.increment.isAdjustmentRequired(params, this.config.readCapacity.increment.calculateValue)){
91+
newProvisionedThroughput.ReadCapacityUnits = this.config.readCapacity.increment.calculateValue(params);
92+
} else if (this.config.readCapacity.decrement.isAdjustmentRequired(params, this.config.readCapacity.decrement.calculateValue)) {
93+
newProvisionedThroughput.ReadCapacityUnits = this.config.readCapacity.decrement.calculateValue(params);
94+
}
10495

105-
return newProvisionedThroughput;
96+
// Adjust write capacity
97+
if (this.config.writeCapacity.increment.isAdjustmentRequired(params, this.config.writeCapacity.increment.calculateValue)){
98+
newProvisionedThroughput.WriteCapacityUnits = this.config.writeCapacity.increment.calculateValue(params);
99+
} else if (this.config.writeCapacity.decrement.isAdjustmentRequired(params, this.config.writeCapacity.decrement.calculateValue)) {
100+
newProvisionedThroughput.WriteCapacityUnits = this.config.writeCapacity.decrement.calculateValue(params);
101+
}
102+
103+
if (newProvisionedThroughput.ReadCapacityUnits === params.ProvisionedThroughput.ReadCapacityUnits
104+
&& newProvisionedThroughput.WriteCapacityUnits === params.ProvisionedThroughput.WriteCapacityUnits) {
105+
return null;
106+
}
107+
108+
return newProvisionedThroughput;
109+
} catch (e) {
110+
logger.warn('ConfigurableProvisioner.getUpdatedProvisionedThroughput failed', JSON.stringify({params}));
111+
throw e;
112+
}
106113
}
107114
}
108115

‎src/DynamoDB.js

Lines changed: 87 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import AWS from 'aws-sdk';
1+
import AWS from 'aws-sdk-promise';
22
import Global from './global';
33
import CloudWatch from './CloudWatch';
44
const {
@@ -13,36 +13,49 @@ export default class DynamoDB {
1313
}
1414

1515
async listTablesAsync() {
16+
logger.debug('DynamoDB.listTablesAsync');
1617
let sw = stats.timer('DynamoDB.listTablesAsync').start()
1718
try {
18-
return await this._db.listTables().promise();
19-
}
20-
finally {
19+
let res = await this._db.listTables().promise();
20+
return res.data;
21+
} catch (ex) {
22+
logger.warn('DynamoDB.listTablesAsync failed');
23+
throw ex;
24+
} finally {
2125
sw.end();
2226
}
2327
}
2428

2529
async describeTableAsync(params) {
30+
logger.debug('DynamoDB.describeTableAsync');
2631
let sw = stats.timer('DynamoDB.describeTableAsync').start();
2732
try {
28-
return await this._db.describeTable(params).promise();
29-
}
30-
finally {
33+
let res = await this._db.describeTable(params).promise();
34+
return res.data;
35+
} catch (ex) {
36+
logger.warn('DynamoDB.describeTableAsync failed', JSON.stringify({params}));
37+
throw ex;
38+
} finally {
3139
sw.end();
3240
}
3341
}
3442

3543
async updateTableAsync(params) {
44+
logger.debug('DynamoDB.updateTableAsync');
3645
let sw = stats.timer('DynamoDB.updateTableAsync').start();
3746
try {
38-
return this._db.updateTable(params).promise();
39-
}
40-
finally {
47+
let res = this._db.updateTable(params).promise();
48+
return res.data;
49+
} catch (ex) {
50+
logger.warn('DynamoDB.updateTableAsync failed', JSON.stringify({params}));
51+
throw ex;
52+
} finally {
4153
sw.end();
4254
}
4355
}
4456

4557
async describeTableConsumedCapacityAsync(params, periodMinutes) {
58+
logger.debug('DynamoDB.describeTableConsumedCapacityAsync');
4659
let sw = stats.timer('DynamoDB.describeTableConsumedCapacityAsync').start();
4760
try {
4861
// Make all the requests concurrently
@@ -79,38 +92,52 @@ export default class DynamoDB {
7992
GlobalSecondaryIndexes: gsis
8093
}
8194
};
82-
}
83-
finally {
95+
} catch (ex) {
96+
logger.warn('DynamoDB.describeTableConsumedCapacityAsync failed', JSON.stringify({params, periodMinutes}));
97+
throw ex;
98+
} finally {
8499
sw.end();
85100
}
86101
}
87102

88103
getTotalTableProvisionedThroughput(params) {
89-
let ReadCapacityUnits = params.Table.ProvisionedThroughput.ReadCapacityUnits;
90-
let WriteCapacityUnits = params.Table.ProvisionedThroughput.WriteCapacityUnits;
104+
logger.debug('DynamoDB.getTotalTableProvisionedThroughput');
105+
try {
106+
let ReadCapacityUnits = params.Table.ProvisionedThroughput.ReadCapacityUnits;
107+
let WriteCapacityUnits = params.Table.ProvisionedThroughput.WriteCapacityUnits;
91108

92-
if (params.Table.GlobalSecondaryIndexes) {
93-
ReadCapacityUnits += params.Table.GlobalSecondaryIndexes.reduce((prev, curr, i) => prev + curr.ProvisionedThroughput.ReadCapacityUnits, 0);
94-
WriteCapacityUnits += params.Table.GlobalSecondaryIndexes.reduce((prev, curr, i) => prev + curr.ProvisionedThroughput.WriteCapacityUnits, 0);
95-
}
109+
if (params.Table.GlobalSecondaryIndexes) {
110+
ReadCapacityUnits += params.Table.GlobalSecondaryIndexes.reduce((prev, curr, i) => prev + curr.ProvisionedThroughput.ReadCapacityUnits, 0);
111+
WriteCapacityUnits += params.Table.GlobalSecondaryIndexes.reduce((prev, curr, i) => prev + curr.ProvisionedThroughput.WriteCapacityUnits, 0);
112+
}
96113

97-
return {
98-
ReadCapacityUnits,
99-
WriteCapacityUnits
100-
};
114+
return {
115+
ReadCapacityUnits,
116+
WriteCapacityUnits
117+
};
118+
} catch (ex) {
119+
logger.warn('DynamoDB.getTotalTableProvisionedThroughput failed', JSON.stringify({params}));
120+
throw ex;
121+
}
101122
}
102123

103124
getMonthlyEstimatedTableCost(provisionedThroughput) {
104-
const averageHoursPerMonth = 720;
105-
const readCostPerHour = 0.0065;
106-
const readCostUnits = 50;
107-
const writeCostPerHour = 0.0065;
108-
const writeCostUnits = 10;
109-
110-
let readCost = provisionedThroughput.ReadCapacityUnits / readCostUnits * readCostPerHour * averageHoursPerMonth;
111-
let writeCost = provisionedThroughput.WriteCapacityUnits / writeCostUnits * writeCostPerHour * averageHoursPerMonth;
112-
113-
return readCost + writeCost;
125+
logger.debug('DynamoDB.getMonthlyEstimatedTableCost');
126+
try {
127+
const averageHoursPerMonth = 720;
128+
const readCostPerHour = 0.0065;
129+
const readCostUnits = 50;
130+
const writeCostPerHour = 0.0065;
131+
const writeCostUnits = 10;
132+
133+
let readCost = provisionedThroughput.ReadCapacityUnits / readCostUnits * readCostPerHour * averageHoursPerMonth;
134+
let writeCost = provisionedThroughput.WriteCapacityUnits / writeCostUnits * writeCostPerHour * averageHoursPerMonth;
135+
136+
return readCost + writeCost;
137+
} catch (ex) {
138+
logger.warn('DynamoDB.getMonthlyEstimatedTableCost failed', JSON.stringify({provisionedThroughput}));
139+
throw ex;
140+
}
114141
}
115142

116143
getArrayOrDefault(value) {
@@ -122,28 +149,34 @@ export default class DynamoDB {
122149
}
123150

124151
async getConsumedCapacityAsync(isRead, tableName, globalSecondaryIndexName, periodMinutes) {
125-
let EndTime = new Date();
126-
let StartTime = new Date();
127-
StartTime.setTime(EndTime - (60000 * periodMinutes));
128-
let MetricName = isRead ? 'ConsumedReadCapacityUnits' : 'ConsumedWriteCapacityUnits';
129-
let Dimensions = this.getDimensions(tableName, globalSecondaryIndexName);
130-
let params = {
131-
Namespace: 'AWS/DynamoDB',
132-
MetricName,
133-
Dimensions,
134-
StartTime,
135-
EndTime,
136-
Period: (periodMinutes * 60),
137-
Statistics: [ 'Average' ],
138-
Unit: 'Count'
139-
};
140-
141-
let data = await this._cw.getMetricStatisticsAsync(params);
142-
return {
143-
tableName,
144-
globalSecondaryIndexName,
145-
data
146-
};
152+
logger.debug('DynamoDB.getConsumedCapacityAsync');
153+
try {
154+
let EndTime = new Date();
155+
let StartTime = new Date();
156+
StartTime.setTime(EndTime - (60000 * periodMinutes));
157+
let MetricName = isRead ? 'ConsumedReadCapacityUnits' : 'ConsumedWriteCapacityUnits';
158+
let Dimensions = this.getDimensions(tableName, globalSecondaryIndexName);
159+
let params = {
160+
Namespace: 'AWS/DynamoDB',
161+
MetricName,
162+
Dimensions,
163+
StartTime,
164+
EndTime,
165+
Period: (periodMinutes * 60),
166+
Statistics: [ 'Average' ],
167+
Unit: 'Count'
168+
};
169+
170+
let data = await this._cw.getMetricStatisticsAsync(params);
171+
return {
172+
tableName,
173+
globalSecondaryIndexName,
174+
data
175+
};
176+
} catch (ex) {
177+
logger.warn('DynamoDB.getConsumedCapacityAsync failed', JSON.stringify({isRead, tableName, globalSecondaryIndexName, periodMinutes}));
178+
throw ex;
179+
}
147180
}
148181

149182
getDimensions(tableName, globalSecondaryIndexName) {

0 commit comments

Comments
 (0)