Skip to content

Commit 6a021c2

Browse files
lukaszczerpak-cloudinarylukaszczerpak-cloudinary
authored
fix: WPT API calls optimizations (DELO-5576) (#141)
* chore: update * fix: skip checking test status and fetch results straightaway * chore: testing
1 parent f1d8a23 commit 6a021c2

File tree

7 files changed

+143
-121
lines changed

7 files changed

+143
-121
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,6 @@ npm-debug.log
1616
!.elasticbeanstalk/*.global.yml
1717
.env
1818

19+
# mise
20+
mise.toml
21+
.mise.local.toml

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "web-speed-test-server",
3-
"version": "1.3.5",
3+
"version": "1.3.6-beta",
44
"private": true,
55
"scripts": {
66
"test": "mocha",

‎routes/wpt.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ const wtp = (app) => {
5454
const quality = req.query.quality;
5555
let rollBarMsg = {testId: testId, thirdPartyErrorCode: "", file: path.basename((__filename))};
5656
logger.info("Fetch WPT test results", rollBarMsg, req);
57-
apiCaller.checkTestStatus(testId, quality, (error, result) => {
57+
apiCaller.getTestResults(testId, quality, (error, result) => {
5858
routeCallback(error, result, res, rollBarMsg)
5959
});
6060
});

‎test/resources/test2.json‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
{
2+
"data": {
3+
"average": {
4+
"firstView": []
5+
},
6+
"bwDown": 0,
7+
"bwUp": 0,
8+
"completed": 1758191003,
9+
"connectivity": "Native",
10+
"fvonly": true,
11+
"id": "250918_YiDc5V_5A0",
12+
"latency": 0,
13+
"location": "SLC_US_02:Chrome",
14+
"median": [],
15+
"mobile": 0,
16+
"plr": "0",
17+
"runs": {
18+
"1": {
19+
"firstView": []
20+
}
21+
},
22+
"shaperLimit": 0,
23+
"standardDeviation": {
24+
"firstView": []
25+
},
26+
"successfulFVRuns": 0,
27+
"summary": "https://www.webpagetest.org/results.php?test=250918_YiDc5V_5A0",
28+
"testRuns": 1,
29+
"testUrl": "https://s.codepen.io/maxro/debug/5b4dde20bc49345de60920ffd1ce8af1",
30+
"url": "https://s.codepen.io/maxro/debug/5b4dde20bc49345de60920ffd1ce8af1"
31+
},
32+
"statusCode": 200,
33+
"statusText": "Test Complete",
34+
"webPagetestVersion": "21.07"
35+
}

‎test/wptTests.js‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,5 +45,10 @@ describe('Parse WPT result', () => {
4545
assert.isString(results.metaData.screenShot, 'There should be a screenshot');
4646
assert.isObject(results.metaData.viewportSize, 'ViewportSize is not an object');
4747
})
48+
it('No firstView data', () => {
49+
let resultJson = JSON.parse(fs.readFileSync('./test/resources/test2.json'));
50+
let results = wtpParser.parseTestResults(resultJson);
51+
assert.equal(results.status, 'not_ready', 'Data not ready is expected');
52+
})
4853

4954
});

‎wtp/apiCaller.js‎

Lines changed: 92 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -14,59 +14,71 @@ const cloudinaryCaller = require('../cloudinary/apiCaller');
1414
const {truncateString} = require('../util/strings');
1515
const RESULTS_URL = 'https://www.webpagetest.org/jsonResult.php';
1616
const RUN_TEST_URL = 'https://www.webpagetest.org/runtest.php';
17-
const GET_TEST_STATUS = 'https://www.webpagetest.org/testStatus.php';
1817
const locationSelector = require('./locationSelector');
1918
const apiKeys = require('./apiKey');
2019

2120
const getTestResults = async (testId, quality, cb) => {
22-
let options = {
23-
method: "GET",
24-
url: RESULTS_URL,
25-
searchParams: {test: testId},
26-
headers: { 'User-Agent': 'WebSpeedTest', 'X-WPT-API-KEY': apiKeys.getRandom() }
27-
};
28-
let response;
29-
let rollBarMsg = {};
30-
try {
31-
response = await got(options)
32-
logger.info("Fetched WPT test results");
33-
const {statusCode, body} = response;
34-
let resBody = JSON.parse(body);
35-
rollBarMsg = {testId: resBody.data.id, analyzedUrl: resBody.data.testUrl, thirdPartyErrorCode: "", file: path.basename((__filename))};
36-
if (statusCode !== 200) {
37-
cb({status: 'error', message: 'WTP returned bad status with testId ' + testId, error: response.statusCode, logLevel: LOG_LEVEL_ERROR}, null, response, rollBarMsg);
38-
return;
21+
let options = {
22+
method: "GET",
23+
url: RESULTS_URL,
24+
searchParams: {test: testId},
25+
headers: {'User-Agent': 'WebSpeedTest', 'X-WPT-API-KEY': apiKeys.getRandom()}
26+
};
27+
let response;
28+
let rollBarMsg = {};
29+
try {
30+
response = await got(options)
31+
logger.info("Fetched WPT test results");
32+
const {statusCode, body} = response;
33+
let resBody = JSON.parse(body);
34+
rollBarMsg = {testId: resBody.data.id, analyzedUrl: resBody.data.testUrl, thirdPartyErrorCode: "", file: path.basename((__filename))};
35+
if (statusCode !== 200) {
36+
cb({status: 'error', message: 'WTP returned bad status with testId ' + testId, error: response.statusCode, logLevel: LOG_LEVEL_ERROR}, null, response, rollBarMsg);
37+
return;
38+
}
39+
rollBarMsg.thirdPartyErrorCode = resBody.statusCode;
40+
if (resBody.statusCode > 400) {
41+
rollBarMsg.thirdPartyErrorCode = resBody.statusCode;
42+
cb({status: 'error', message: 'WTP returned bad status with testId ' + testId, error: resBody}, null, response, rollBarMsg);
43+
return;
44+
}
45+
if (resBody.statusCode >= 100 && resBody.statusCode < 200) {
46+
cb(null, {status: 'success', message: 'test not finished', code: 150}, null, null);
47+
return;
48+
}
49+
if (!body) {
50+
cb({status: 'error', message: 'WTP returned empty body with testId ' + testId, error: 'empty body', logLevel: LOG_LEVEL_WARNING}, null, response, rollBarMsg);
51+
return;
52+
}
53+
if (typeof resBody.data.statusCode !== 'undefined') {
54+
cb({status: 'error', message: resBody.data.statusText + 'testId ' + testId, error: resBody, logLevel: LOG_LEVEL_WARNING}, null, response, rollBarMsg);
55+
return;
56+
}
57+
let wtpRes = resultParser.parseTestResults(resBody);
58+
if (!wtpRes) {
59+
cb({status: 'error', message: 'WTP results are missing data with testId ' + testId, error: resBody, logLevel: LOG_LEVEL_ERROR}, null, response, rollBarMsg);
60+
return;
61+
} else if (wtpRes.status === 'not_ready') {
62+
cb(null, {status: 'success', message: 'data not ready yet', code: 150}, null, null);
63+
return;
64+
} else if (wtpRes.status === 'error') {
65+
cb(wtpRes);
66+
return;
67+
} else {
68+
cloudinaryCaller(wtpRes.imageList, wtpRes.dpr, wtpRes.metaData, quality, cb, rollBarMsg);
69+
}
70+
} catch (e) {
71+
cb({status: 'error', message: 'Error calling WTP with testId ' + testId, error: e, logLevel: LOG_LEVEL_ERROR}, null, response, rollBarMsg);
72+
return;
3973
}
40-
if (!body) {
41-
cb({status: 'error', message: 'WTP returned empty body with testId ' + testId, error: 'empty body', logLevel: LOG_LEVEL_WARNING}, null, response, rollBarMsg);
42-
return;
43-
}
44-
if (typeof resBody.data.statusCode !== 'undefined') {
45-
cb({status: 'error', message: resBody.data.statusText + 'testId ' + testId, error: resBody, logLevel: LOG_LEVEL_WARNING}, null, response, rollBarMsg);
46-
return;
47-
}
48-
let wtpRes = resultParser.parseTestResults(resBody);
49-
if (!wtpRes) {
50-
cb({status: 'error', message: 'WTP results are missing data with testId ' + testId, error: resBody, logLevel: LOG_LEVEL_ERROR}, null, response, rollBarMsg);
51-
return;
52-
} else if(wtpRes.status === 'error') {
53-
cb(wtpRes);
54-
return;
55-
} else {
56-
cloudinaryCaller(wtpRes.imageList, wtpRes.dpr, wtpRes.metaData, quality, cb, rollBarMsg);
57-
}
58-
} catch (e) {
59-
cb({status: 'error', message: 'Error calling WTP with testId ' + testId, error: e, logLevel: LOG_LEVEL_ERROR}, null, response, rollBarMsg);
60-
return;
61-
}
6274
};
6375

6476
const runWtpTest = async (url, mobile, cb) => {
65-
//logger.info('Running new test ' + url);
66-
let options = {
67-
method: "POST",
68-
url: RUN_TEST_URL,
69-
searchParams: {
77+
//logger.info('Running new test ' + url);
78+
let options = {
79+
method: "POST",
80+
url: RUN_TEST_URL,
81+
searchParams: {
7082
url: url,
7183
f: "json",
7284
width: config.get('wtp.viewportWidth'),
@@ -76,82 +88,43 @@ const runWtpTest = async (url, mobile, cb) => {
7688
mobile: (mobile) ? 1 : 0,
7789
fvonly: 1, // first view only
7890
timeline: 1 // workaround for WPT sometimes hanging on getComputedStyle()
79-
},
80-
headers: { 'User-Agent': 'WebSpeedTest', 'X-WPT-API-KEY': apiKeys.getRandom() },
81-
throwHttpErrors: false
82-
};
83-
let response;
84-
let rollBarMsg = {testId: "", analyzedUrl: url, thirdPartyErrorCode: "", thirdPartyErrorMsg: "", file: path.basename((__filename))};
85-
try {
86-
response = await got(options);
87-
const {statusCode, body} = response;
88-
if (statusCode !== 200) {
89-
rollBarMsg.thirdPartyErrorCode = response.statusCode;
90-
rollBarMsg.thirdPartyErrorBody = body && truncateString(body, 1000) || "";
91-
cb({status: 'error', message: 'WTP returned bad status with url ' + url, error: response.statusMessage, logLevel: LOG_LEVEL_ERROR}, null, response, rollBarMsg);
92-
return;
93-
}
94-
if (!body) {
95-
cb({status: 'error', message: 'WTP returned empty body with url ' + url, error: 'empty body'}, null, response, rollBarMsg);
96-
return;
97-
}
98-
let bodyJson = JSON.parse(body);
99-
rollBarMsg.testId = (typeof bodyJson.data !== 'undefined' && typeof bodyJson.data.testId !== 'undefined') ?
100-
(bodyJson.data.testId) :
101-
"N/A";
102-
logger.info("Started WPT test", {"testId": rollBarMsg.testId});
103-
let testId = resultParser.parseTestResponse(bodyJson, rollBarMsg);
104-
if (typeof testId === 'object') {
105-
cb(null, testId);
106-
return;
107-
}
108-
cb(null, {status: "success", data: {testId}});
109-
} catch (error) {
110-
cb({status: 'error', message: 'Error calling WTP with url ' + url, error: error}, null, response, rollBarMsg);
111-
return;
112-
}
113-
};
114-
115-
const checkTestStatus = async (testId, quality, cb) => {
116-
let options = {
117-
method: "GET",
118-
url: GET_TEST_STATUS,
119-
searchParams: {test: testId, f: "json"},
120-
'headers': { 'User-Agent': 'WebSpeedTest', 'X-WPT-API-KEY': apiKeys.getRandom() }
121-
};
122-
let response;
123-
let rollBarMsg = {};
124-
try {
125-
response = await got(options);
126-
logger.info("Fetched WPT test status");
127-
const {statusCode, body} = response;
128-
let bodyJson = JSON.parse(body);
129-
rollBarMsg = {testId: testId, thirdPartyErrorCode: "", file: path.basename((__filename))};
130-
if (statusCode !== 200) {
131-
cb({status: 'error', message: 'WTP returned bad status testId ' + testId , error: response.statusCode}, null, response, rollBarMsg);
132-
return;
133-
}
134-
//logger.debug('Test status code ' + bodyJson.statusCode, rollBarMsg);
135-
rollBarMsg.thirdPartyErrorCode = bodyJson.statusCode;
136-
if (bodyJson.statusCode > 400) {
137-
rollBarMsg.thirdPartyErrorCode = bodyJson.statusCode;
138-
cb({status: 'error', message: 'WTP returned bad status with testId ' + testId, error: bodyJson}, null, response, rollBarMsg);
139-
return;
140-
}
141-
if (bodyJson.statusCode === 200 || bodyJson.statusCode === 400) {
142-
getTestResults(testId, quality, cb);
143-
}
144-
if (bodyJson.statusCode >= 100 && bodyJson.statusCode < 200) {
145-
cb(null, {status: 'success', message: 'test not finished', code: 150}, null, null);
91+
},
92+
headers: {'User-Agent': 'WebSpeedTest', 'X-WPT-API-KEY': apiKeys.getRandom()},
93+
throwHttpErrors: false
94+
};
95+
let response;
96+
let rollBarMsg = {testId: "", analyzedUrl: url, thirdPartyErrorCode: "", thirdPartyErrorMsg: "", file: path.basename((__filename))};
97+
try {
98+
response = await got(options);
99+
const {statusCode, body} = response;
100+
if (statusCode !== 200) {
101+
rollBarMsg.thirdPartyErrorCode = response.statusCode;
102+
rollBarMsg.thirdPartyErrorBody = body && truncateString(body, 1000) || "";
103+
cb({status: 'error', message: 'WTP returned bad status with url ' + url, error: response.statusMessage, logLevel: LOG_LEVEL_ERROR}, null, response, rollBarMsg);
104+
return;
105+
}
106+
if (!body) {
107+
cb({status: 'error', message: 'WTP returned empty body with url ' + url, error: 'empty body'}, null, response, rollBarMsg);
108+
return;
109+
}
110+
let bodyJson = JSON.parse(body);
111+
rollBarMsg.testId = (typeof bodyJson.data !== 'undefined' && typeof bodyJson.data.testId !== 'undefined') ?
112+
(bodyJson.data.testId) :
113+
"N/A";
114+
logger.info("Started WPT test", {"testId": rollBarMsg.testId});
115+
let testId = resultParser.parseTestResponse(bodyJson, rollBarMsg);
116+
if (typeof testId === 'object') {
117+
cb(null, testId);
118+
return;
119+
}
120+
cb(null, {status: "success", data: {testId}});
121+
} catch (error) {
122+
cb({status: 'error', message: 'Error calling WTP with url ' + url, error: error}, null, response, rollBarMsg);
123+
return;
146124
}
147-
} catch (error) {
148-
cb({status: 'error', message: 'Error checking WTP status with testId ' + testId, error: error}, null, response, rollBarMsg);
149-
return;
150-
}
151125
};
152126

153127
module.exports = {
154-
getTestResults,
155-
runWtpTest,
156-
checkTestStatus
128+
getTestResults,
129+
runWtpTest
157130
};

‎wtp/wtpResultsParser.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,12 @@ const path = require('path');
1515
const parseTestResults = (testJson) => {
1616
let rollBarMsg = {testId: testJson.data.id, analyzedUrl: testJson.data.testUrl, thirdPartyErrorCode: "", file: path.basename((__filename))};
1717
try {
18+
// check if data is available
19+
if (Array.isArray(testJson.data.median) && testJson.data.median.length == 0) {
20+
logger.warn("Test results not ready yet", rollBarMsg);
21+
return {status: 'not_ready', message: 'data_not_ready'};
22+
}
23+
1824
let browserName = _.get(testJson, 'data.location', 'somePlace:N/A').split(':')[1];
1925
if ('firefox' === browserName.toLowerCase()) {
2026
logger.warn("Test run with firefox that is not supported", rollBarMsg);

0 commit comments

Comments
 (0)