Skip to content

Commit bf0b2d6

Browse files
[ENG-59902] :Refactor the code to use async/promise AWS functions and update the t… (#406)
* Refactor the code to use async/promise AWS functions and update the test cases * update o365 collector package to build the artifact
1 parent 74a54da commit bf0b2d6

File tree

8 files changed

+197
-230
lines changed

8 files changed

+197
-230
lines changed

collectors/o365/.jshintrc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"esversion": 6,
2+
"esversion": 11,
33
"shadow": "outer",
44
"undef": true,
55
"unused": "vars",

collectors/o365/healthcheck.js

Lines changed: 28 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -10,40 +10,42 @@
1010

1111
'use strict';
1212

13-
const o365_mgmnt = require('./lib/o365_mgmnt');
13+
const util = require('util');
1414
const al_health = require('@alertlogic/al-aws-collector-js').Health
1515
const AlLogger = require('@alertlogic/al-aws-collector-js').Logger;
1616

1717
/*
1818
* Checks the subscriptions against the configured content type. Starts the subscriptions if th
1919
*/
2020

21-
function checkO365Subscriptions(callback) {
21+
async function checkO365Subscriptions() {
2222
let collector = this;
23-
return collector.o365_mgmnt_client.listSubscriptions(null)
24-
.then(filterSubscriptions)
25-
.then(filteredStreams => {
26-
if(filteredStreams.length > 0){
27-
AlLogger.info(`O365000101: Starting subscriptions for streams ${filteredStreams.join(', ')}`);
28-
} else{
29-
AlLogger.info(`O365000102: No streams need restarted.`);
30-
}
31-
const streamPromises = filteredStreams.map(stream => collector.o365_mgmnt_client.startSubscription(stream));
32-
return Promise.all(streamPromises);
33-
})
34-
.then(res => callback(null))
35-
.catch(error => {
36-
let errorString;
37-
try{
38-
errorString = JSON.stringify(error);
39-
}
40-
catch (stringifyError){
41-
errorString = error.toJSON ? error.toJSON() :
42-
error.message ? error.message :
43-
util.inspect(error);
44-
}
45-
callback(al_health.errorMsg('O365000103', 'Bad O365 stream status: ' + errorString));
46-
});
23+
try {
24+
const result = await collector.o365_mgmnt_client.listSubscriptions(null);
25+
const filteredStreams = filterSubscriptions(result);
26+
27+
if (filteredStreams.length > 0) {
28+
AlLogger.info(`O365000101: Starting subscriptions for streams ${filteredStreams.join(', ')}`);
29+
} else {
30+
AlLogger.info(`O365000102: No streams need restarted.`);
31+
return;
32+
}
33+
34+
const streamPromises = filteredStreams.map(stream => collector.o365_mgmnt_client.startSubscription(stream));
35+
return await Promise.all(streamPromises);
36+
} catch (error) {
37+
AlLogger.debug(`O365000103: Error in checkO365Subscriptions: ${util.inspect(error, {depth: 5})}`);
38+
let errorString;
39+
try {
40+
errorString = JSON.stringify(error, Object.getOwnPropertyNames(error));
41+
}
42+
catch (stringifyError) {
43+
errorString = error.toJSON ? error.toJSON() :
44+
error.message ? error.message :
45+
util.inspect(error);
46+
}
47+
throw al_health.errorMsg('O365000103', 'Bad O365 stream status: ' + errorString);
48+
}
4749
}
4850

4951
function filterSubscriptions(result){

collectors/o365/index.js

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,11 @@ const O365Collector = require('./o365_collector').O365Collector;
1616
exports.handler = O365Collector.makeHandler(function(event, context) {
1717
debug('input event: ', event);
1818
AlLogger.defaultMeta = {requestId: context.awsRequestId};
19-
O365Collector.load().then(function(creds) {
19+
O365Collector.load().then(async function(creds) {
2020
var o365c = new O365Collector(context, creds);
21-
o365c.handleEvent(event);
21+
await o365c.handleEvent(event);
2222
}).catch(error => {
2323
AlLogger.error(`O365000006 error in creating object ${error}`);
24-
return error;
24+
throw error;
2525
});
2626
});

collectors/o365/lib/o365_mgmnt/o365management.js

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ class O365Management extends msRestAzure.AzureServiceClient {
8080
error.statusCode = status;
8181
error.request = msRest.stripRequest(httpRequest);
8282
error.response = msRest.stripResponse(res);
83-
throw error
83+
throw error;
8484
}
8585
const nextPageUri = headers.get('NextPageUri');
8686
return { parsedBody, nextPageUri };
@@ -220,8 +220,7 @@ class O365Management extends msRestAzure.AzureServiceClient {
220220
return client.sendRequest(httpRequest).then(handler);
221221
}
222222

223-
getPreFormedUrl(uri, options, callback) {
224-
let client = this;
223+
getPreFormedUrl(uri, options) {
225224
let publisherId = this.publisherId;
226225
let requestUrl = uri;
227226

@@ -260,7 +259,7 @@ class O365Management extends msRestAzure.AzureServiceClient {
260259
// Request Handler
261260
const handler = this.requestHandler(httpRequest);
262261
// Send Request
263-
return client.sendRequest(httpRequest).then(handler);
262+
return this.sendRequest(httpRequest).then(handler);
264263
}
265264
}
266265

collectors/o365/o365_collector.js

Lines changed: 28 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ class O365Collector extends PawsCollector {
4040
super(context, creds, packageJson.version, [checkO365Subscriptions], []);
4141
this.o365_mgmnt_client = m_o365mgmnt.getO365ManagmentClient(creds.pawsCreds);
4242
}
43-
pawsInitCollectionState(event, callback) {
43+
async pawsInitCollectionState(event) {
4444
let startTs = process.env.paws_collection_start_ts ?
4545
process.env.paws_collection_start_ts :
4646
moment().toISOString();
@@ -68,23 +68,21 @@ class O365Collector extends PawsCollector {
6868
poll_interval_sec: 1
6969
}
7070
});
71-
// bind the this to checkO365Subscriptions to acces the pawsCreds
72-
const checkSubscriptions = checkO365Subscriptions.bind(this);
73-
return checkSubscriptions((err) => {
74-
return callback(err, initialStates, 1);
75-
});
71+
// bind the this to checkO365Subscriptions to access the pawsCreds
72+
await checkO365Subscriptions.call(this);
73+
return { state: initialStates, nextInvocationTimeout: 1 };
7674
}
7775

78-
pawsGetRegisterParameters(event, callback){
76+
async pawsGetRegisterParameters(event){
7977
const regValues = {
8078
azureTenantId: process.env.paws_collector_param_string_1,
8179
azureStreams: process.env.collector_streams
8280
};
8381

84-
callback(null, regValues);
82+
return regValues;
8583
}
8684

87-
pawsGetLogs(state, callback) {
85+
async pawsGetLogs(state) {
8886
let collector = this;
8987
// This code can remove once exsisting code set collector_streams env variable
9088
if (!process.env.collector_streams) {
@@ -97,7 +95,7 @@ class O365Collector extends PawsCollector {
9795
state.until = nextUntilMoment.toISOString();
9896
state.nextPage = null;
9997
state.poll_interval_sec = nextPollInterval;
100-
return callback(null, [], state, state.poll_interval_sec);
98+
return [[], state, state.poll_interval_sec];
10199
}
102100

103101
if(moment().diff(state.since, 'days', true) > 7){
@@ -151,7 +149,7 @@ class O365Collector extends PawsCollector {
151149
const contentUriFun = ({contentUri}) => collector.o365_mgmnt_client.getPreFormedUrl(contentUri);
152150
const poolLimit = 20;
153151

154-
return this.asyncPoolAll(poolLimit, parsedBody, contentUriFun).then(content => {
152+
return collector.asyncPoolAll(poolLimit, parsedBody, contentUriFun).then(content => {
155153
return {
156154
logs: content.reduce((agg, {parsedBody}) => [...parsedBody, ...agg], []),
157155
nextPage: nextPageUri
@@ -160,43 +158,40 @@ class O365Collector extends PawsCollector {
160158
});
161159

162160
// Now that we have all the content uri promises agregated, we can call them and collect the data
163-
contentPromise.then(({logs, nextPage}) => {
161+
return contentPromise.then(async ({logs, nextPage}) => {
164162
let newState;
165163
if(nextPage === undefined){
166-
newState = this._getNextCollectionState(state);
164+
newState = collector._getNextCollectionState(state);
167165
} else {
168-
newState = this._getNextCollectionStateWithNextPage(state, nextPage);
166+
newState = collector._getNextCollectionStateWithNextPage(state, nextPage);
169167
}
170168

171169
let uniqueLogs = [];
172-
collector.removeDuplicatedItem(logs, "Id", (err, res) => {
173-
if (err) {
174-
return callback(err);
175-
}
176-
else {
177-
if (res.length > 0) {
178-
uniqueLogs = [...res];
179-
}
180-
return callback(null, uniqueLogs, newState, newState.poll_interval_sec);
181-
}
182-
});
170+
const res = await collector.removeDuplicatedItem(logs, "Id");
171+
if (res.length > 0) {
172+
uniqueLogs = [...res];
173+
}
174+
return [uniqueLogs, newState, newState.poll_interval_sec];
183175
}).catch(err => {
184176
// set errorCode to showcase client error on DDMetric
185-
if (typeof err === 'object') {
177+
if (typeof err === 'object' && err !== null) {
186178
err.errorCode = err.code ? err.code : (err.statusCode ? err.statusCode : err.status);
187179
}
188-
let newState = this._handleMSManagementApiError(err, state);
180+
let newState = collector._handleMSManagementApiError(err, state);
189181
if (newState) {
190-
return callback(null, [], newState, newState.poll_interval_sec);
182+
return [[], newState, newState.poll_interval_sec];
191183
}
192184

193185
if (!err.code && err.message) {
194-
const formatedError = this._formatErrorMessage(err);
195-
AlLogger.error(`O365000003 Error in collection: ${err.message}`);
196-
return callback(formatedError);
186+
const formatedError = collector._formatErrorMessage(err);
187+
AlLogger.error(`O365000003 Error in collection: ${formatedError.message || err.message}`);
188+
throw formatedError;
189+
} else if (typeof err === 'string') {
190+
// if error is a string, wrap it in Error
191+
throw new Error(err);
197192
} else {
198-
// if error is string or don't have err.code in error object, then return complete error object/string.
199-
return callback(err);
193+
// if error is already an Error object or object with details, throw as is
194+
throw err;
200195
}
201196
});
202197
}

collectors/o365/package.json

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "o365-collector",
3-
"version": "1.2.74",
3+
"version": "1.3.0",
44
"description": "Alert Logic AWS based O365 Log Collector",
55
"repository": {},
66
"private": true,
@@ -9,24 +9,24 @@
99
"test": "JUNIT_REPORT_PATH=./test/report.xml nyc --reporter=text mocha --colors"
1010
},
1111
"devDependencies": {
12-
"@aws-sdk/client-cloudformation": "^3.777.0",
13-
"@aws-sdk/client-cloudwatch": "^3.777.0",
14-
"@aws-sdk/client-dynamodb": "^3.777.0",
15-
"@aws-sdk/client-kms": "^3.777.0",
16-
"@aws-sdk/client-lambda": "^3.777.0",
17-
"@aws-sdk/client-s3": "^3.779.0",
18-
"@aws-sdk/client-sqs": "^3.777.0",
19-
"@aws-sdk/client-ssm": "^3.777.0",
12+
"@aws-sdk/client-cloudformation": "^3.991.0",
13+
"@aws-sdk/client-cloudwatch": "^3.991.0",
14+
"@aws-sdk/client-dynamodb": "^3.991.0",
15+
"@aws-sdk/client-kms": "^3.991.0",
16+
"@aws-sdk/client-lambda": "^3.991.0",
17+
"@aws-sdk/client-s3": "^3.991.0",
18+
"@aws-sdk/client-sqs": "^3.991.0",
19+
"@aws-sdk/client-ssm": "^3.991.0",
2020
"jshint": "^2.13.6",
2121
"mocha": "^10.7.3",
2222
"nyc": "^17.1.0",
2323
"rewire": "^9.0.1",
24-
"sinon": "^20.0.0"
24+
"sinon": "^21.0.1"
2525
},
2626
"dependencies": {
27-
"@alertlogic/al-aws-collector-js": "4.2.0",
27+
"@alertlogic/al-aws-collector-js": "4.2.1",
2828
"@alertlogic/al-collector-js": "3.0.20",
29-
"@alertlogic/paws-collector": "2.2.11",
29+
"@alertlogic/paws-collector": "2.3.0",
3030
"@azure/ms-rest-azure-js": "2.1.0",
3131
"@azure/ms-rest-js": "2.7.0",
3232
"@azure/ms-rest-nodeauth": "3.1.1",
@@ -37,7 +37,7 @@
3737
"tiny-async-pool": "^2.1.0"
3838
},
3939
"overrides": {
40-
"axios": "^1.11.0",
40+
"axios": "^1.13.5",
4141
"diff": "^8.0.3"
4242
},
4343
"author": "Alert Logic Inc."

0 commit comments

Comments
 (0)