Skip to content
This repository was archived by the owner on Dec 1, 2022. It is now read-only.

Commit e741556

Browse files
tmitchel2tmitchel2
committed
Changes for #24 
1 parent 1afd43e commit e741556

File tree

6 files changed

+266
-121
lines changed

6 files changed

+266
-121
lines changed

‎package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
},
2626
"dependencies": {},
2727
"devDependencies": {
28+
"async": "^2.0.0",
2829
"aws-sdk": "2.2.48",
2930
"aws-sdk-promise": "0.0.2",
3031
"babel": "^6.5.2",

‎src/App.js

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
/* @flow */
2+
import Provisioner from './Provisioner';
3+
import Stats from './utils/Stats';
4+
import CostEstimation from './utils/CostEstimation';
5+
import Throughput from './utils/Throughput';
6+
import CapacityCalculator from './CapacityCalculator';
7+
import { json, stats, log, invariant } from './Global';
8+
import type { UpdateTableRequest } from 'aws-sdk-promise';
9+
10+
export default class App {
11+
_provisioner: Provisioner;
12+
_capacityCalculator: CapacityCalculator;
13+
14+
constructor() {
15+
this._provisioner = new Provisioner();
16+
this._capacityCalculator = new CapacityCalculator();
17+
}
18+
19+
async runAsync(event: any, context: any): Promise {
20+
invariant(event != null, 'The argument \'event\' was null');
21+
invariant(context != null, 'The argument \'context\' was null');
22+
23+
let sw = stats.timer('Index.handler').start();
24+
25+
// In local mode the json padding can be overridden
26+
if (event.json && event.json.padding) {
27+
json.padding = event.json.padding;
28+
}
29+
30+
log('Getting table names');
31+
let tableNames = await this._provisioner.getTableNamesAsync();
32+
33+
log('Getting table details');
34+
let tableDetails = await this._getTableDetailsAsync(tableNames);
35+
36+
log('Getting required table update requests');
37+
let tableUpdateRequests = this._getTableUpdateRequests(tableDetails);
38+
39+
if (tableUpdateRequests.length > 0) {
40+
log('Updating tables');
41+
await this._updateTablesAsync(tableUpdateRequests);
42+
log('Updated tables');
43+
} else {
44+
log('No table updates required');
45+
}
46+
47+
sw.end();
48+
this._logMetrics(tableDetails);
49+
50+
// Return an empty response
51+
let response = null;
52+
if (context) {
53+
context.succeed(response);
54+
} else {
55+
return response;
56+
}
57+
}
58+
59+
async _getTableDetailsAsync(tableNames: string[]): Promise<Object[]> {
60+
invariant(tableNames instanceof Array, 'The argument \'tableNames\' was not an array');
61+
62+
let tasks = tableNames.map(name => this._getTableDetailAsync(name));
63+
return await Promise.all(tasks);
64+
}
65+
66+
async _getTableDetailAsync(tableName: string): Promise<Object> {
67+
invariant(typeof tableName === 'string', 'The argument \'tableName\' was not a string');
68+
69+
log('Getting table description', tableName);
70+
let describeTableResponse = await this._provisioner.db
71+
.describeTableAsync({TableName: tableName});
72+
73+
let tableDescription = describeTableResponse.Table;
74+
75+
log('Getting table consumed capacity description', tableName);
76+
let consumedCapacityTableDescription = await this._capacityCalculator
77+
.describeTableConsumedCapacityAsync(tableDescription, 1);
78+
79+
log('Getting table update request', tableName);
80+
let tableUpdateRequest = await this._provisioner.getTableUpdateAsync(tableDescription,
81+
consumedCapacityTableDescription);
82+
83+
// Log the monthlyEstimatedCost
84+
let totalTableProvisionedThroughput = Throughput
85+
.getTotalTableProvisionedThroughput(tableDescription);
86+
87+
let monthlyEstimatedCost = CostEstimation
88+
.getMonthlyEstimatedTableCost(totalTableProvisionedThroughput);
89+
90+
stats
91+
.counter('DynamoDB.monthlyEstimatedCost')
92+
.inc(monthlyEstimatedCost);
93+
94+
let result = {
95+
tableName,
96+
tableDescription,
97+
consumedCapacityTableDescription,
98+
tableUpdateRequest,
99+
totalTableProvisionedThroughput,
100+
monthlyEstimatedCost,
101+
};
102+
103+
return result;
104+
}
105+
106+
async _updateTablesAsync(tableUpdateRequests: UpdateTableRequest[]): Promise {
107+
invariant(tableUpdateRequests instanceof Array,
108+
'The argument \'tableUpdateRequests\' was not an array');
109+
110+
// If we are updating more than 10 tables in a single run
111+
// then we must wait until each one has been completed to
112+
// ensure we do not hit the AWS limit of 10 concurrent updates
113+
let isRateLimitedUpdatingRequired = tableUpdateRequests.length > 10;
114+
await Promise.all(tableUpdateRequests.map(
115+
async req => this._updateTableAsync(req, isRateLimitedUpdatingRequired)
116+
));
117+
}
118+
119+
async _updateTableAsync(tableUpdateRequest: UpdateTableRequest,
120+
isRateLimitedUpdatingRequired: boolean): Promise {
121+
invariant(tableUpdateRequest != null, 'The argument \'tableUpdateRequest\' was null');
122+
invariant(typeof isRateLimitedUpdatingRequired === 'boolean',
123+
'The argument \'isRateLimitedUpdatingRequired\' was not a boolean');
124+
125+
log('Updating table', tableUpdateRequest.TableName);
126+
await this._provisioner.db
127+
.updateTableWithRateLimitAsync(tableUpdateRequest, isRateLimitedUpdatingRequired);
128+
129+
log('Updated table', tableUpdateRequest.TableName);
130+
}
131+
132+
_getTableUpdateRequests(tableDetails: Object[]): UpdateTableRequest[] {
133+
invariant(tableDetails instanceof Array,
134+
'The argument \'tableDetails\' was not an array');
135+
136+
return tableDetails
137+
.filter(({tableUpdateRequest}) => { return tableUpdateRequest != null; })
138+
.map(({tableUpdateRequest}) => tableUpdateRequest);
139+
}
140+
141+
_logMetrics(tableDetails: Object[]) {
142+
invariant(tableDetails instanceof Array,
143+
'The argument \'tableDetails\' was not an array');
144+
145+
// Log stats
146+
let st = new Stats(stats);
147+
let stJSON = st.toJSON();
148+
st.reset();
149+
150+
// Log readable info
151+
let updateRequests = tableDetails.map(i => i.tableUpdateRequest).filter(i => i !== null);
152+
let totalMonthlyEstimatedCost = tableDetails
153+
.reduce((prev, curr) => prev + curr.monthlyEstimatedCost, 0);
154+
let totalProvisionedThroughput = tableDetails.reduce((prev, curr) => {
155+
return {
156+
ReadCapacityUnits: prev.ReadCapacityUnits +
157+
curr.totalTableProvisionedThroughput.ReadCapacityUnits,
158+
WriteCapacityUnits: prev.WriteCapacityUnits +
159+
curr.totalTableProvisionedThroughput.WriteCapacityUnits,
160+
};
161+
}, {ReadCapacityUnits: 0, WriteCapacityUnits: 0});
162+
163+
log(JSON.stringify({
164+
'Index.handler': {
165+
mean: stJSON['Index.handler'].histogram.mean
166+
},
167+
'DynamoDB.listTablesAsync': {
168+
mean: stJSON['DynamoDB.listTablesAsync'].histogram.mean,
169+
},
170+
'DynamoDB.describeTableAsync': {
171+
mean: stJSON['DynamoDB.describeTableAsync'].histogram.mean,
172+
},
173+
'DynamoDB.describeTableConsumedCapacityAsync': {
174+
mean: stJSON['DynamoDB.describeTableConsumedCapacityAsync']
175+
.histogram.mean,
176+
},
177+
'CloudWatch.getMetricStatisticsAsync': {
178+
mean: stJSON['CloudWatch.getMetricStatisticsAsync'].histogram.mean,
179+
},
180+
TableUpdates: {
181+
count: updateRequests.length,
182+
},
183+
TotalProvisionedThroughput: totalProvisionedThroughput,
184+
TotalMonthlyEstimatedCost: totalMonthlyEstimatedCost,
185+
}, null, json.padding));
186+
}
187+
}

‎src/Index.js

Lines changed: 6 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -3,139 +3,26 @@
33
// $FlowIgnore
44
import babelPolyfill from 'babel-polyfill';
55
/* eslint-enable */
6-
import Provisioner from './Provisioner';
76
// $FlowIgnore
87
import dotenv from 'dotenv';
9-
import Stats from './utils/Stats';
10-
import CostEstimation from './utils/CostEstimation';
11-
import Throughput from './utils/Throughput';
12-
import CapacityCalculator from './CapacityCalculator';
13-
import { json, stats, log } from './Global';
8+
import App from './App';
9+
import { log } from './Global';
1410

1511
log('*** LAMBDA INIT ***');
1612
export let handler = async (event: any, context: any) => {
1713
try {
18-
log('*** LAMBDA START ***');
19-
let sw = stats.timer('Index.handler').start();
20-
21-
// In local mode the json padding can be overridden
22-
if (event.json && event.json.padding) {
23-
json.padding = event.json.padding;
24-
}
25-
26-
// Load environment variables
2714
dotenv.config({path: 'config.env'});
2815

29-
let provisioner = new Provisioner();
30-
let capacityCalculator = new CapacityCalculator();
31-
32-
log('Getting table names');
33-
let tableNames = await provisioner.getTableNamesAsync();
34-
let capacityTasks = tableNames
35-
.map(async tableName => {
36-
37-
log('Getting table description', tableName);
38-
let describeTableResponse = await provisioner.db.describeTableAsync({TableName: tableName});
39-
let tableDescription = describeTableResponse.Table;
40-
41-
log('Getting table consumed capacity description', tableName);
42-
let consumedCapacityTableDescription = await capacityCalculator
43-
.describeTableConsumedCapacityAsync(tableDescription, 1);
44-
45-
log('Getting table update request', tableName);
46-
let tableUpdateRequest = await provisioner.getTableUpdateAsync(tableDescription,
47-
consumedCapacityTableDescription);
48-
49-
if (tableUpdateRequest) {
50-
log('Updating table', tableName);
51-
await provisioner.db.updateTableAsync(tableUpdateRequest);
52-
log('Updated table', tableName);
53-
}
54-
55-
// Log the monthlyEstimatedCost
56-
let totalTableProvisionedThroughput = Throughput
57-
.getTotalTableProvisionedThroughput(tableDescription);
58-
59-
let monthlyEstimatedCost = CostEstimation
60-
.getMonthlyEstimatedTableCost(totalTableProvisionedThroughput);
61-
62-
stats
63-
.counter('DynamoDB.monthlyEstimatedCost')
64-
.inc(monthlyEstimatedCost);
65-
66-
return {
67-
tableDescription,
68-
consumedCapacityTableDescription,
69-
tableUpdateRequest,
70-
totalTableProvisionedThroughput,
71-
monthlyEstimatedCost
72-
};
73-
});
74-
75-
let capacityItems = await Promise.all(capacityTasks);
76-
sw.end();
77-
78-
// Log stats
79-
let st = new Stats(stats);
80-
let stJSON = st.toJSON();
81-
st.reset();
82-
83-
// Log readable info
84-
let updateRequests = capacityItems
85-
.map(i => i.tableUpdateRequest)
86-
.filter(i => i !== null);
87-
88-
let totalMonthlyEstimatedCost = capacityItems
89-
.reduce((prev, curr) => prev + curr.monthlyEstimatedCost, 0);
90-
91-
let totalProvisionedThroughput = capacityItems.reduce((prev, curr) => {
92-
return {
93-
ReadCapacityUnits: prev.ReadCapacityUnits +
94-
curr.totalTableProvisionedThroughput.ReadCapacityUnits,
95-
WriteCapacityUnits: prev.WriteCapacityUnits +
96-
curr.totalTableProvisionedThroughput.WriteCapacityUnits,
97-
};
98-
}, {ReadCapacityUnits: 0, WriteCapacityUnits: 0});
99-
100-
log(JSON.stringify({
101-
'Index.handler': {
102-
mean: stJSON['Index.handler'].histogram.mean
103-
},
104-
'DynamoDB.listTablesAsync': {
105-
mean: stJSON['DynamoDB.listTablesAsync'].histogram.mean,
106-
},
107-
'DynamoDB.describeTableAsync': {
108-
mean: stJSON['DynamoDB.describeTableAsync'].histogram.mean,
109-
},
110-
'DynamoDB.describeTableConsumedCapacityAsync': {
111-
mean: stJSON['DynamoDB.describeTableConsumedCapacityAsync']
112-
.histogram.mean,
113-
},
114-
'CloudWatch.getMetricStatisticsAsync': {
115-
mean: stJSON['CloudWatch.getMetricStatisticsAsync'].histogram.mean,
116-
},
117-
TableUpdates: {
118-
count: updateRequests.length,
119-
},
120-
TotalProvisionedThroughput: totalProvisionedThroughput,
121-
TotalMonthlyEstimatedCost: totalMonthlyEstimatedCost,
122-
}, null, json.padding));
123-
124-
// Return an empty response
125-
let response = null;
126-
if (context) {
127-
context.succeed(response);
128-
} else {
129-
return response;
130-
}
131-
16+
let app = new App();
17+
log('*** LAMBDA START ***');
18+
await app.runAsync(event, context);
13219
} catch (e) {
20+
log('*** LAMBDA ERROR ***');
13321
if (context) {
13422
context.fail(e);
13523
} else {
13624
throw e;
13725
}
138-
13926
} finally {
14027
log('*** LAMBDA FINISH ***');
14128
}

0 commit comments

Comments
 (0)