diff --git a/.gitignore b/.gitignore index ad4241a..075ccf9 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ npm-debug.log !.elasticbeanstalk/*.global.yml .env +# mise +mise.toml +.mise.local.toml diff --git a/package.json b/package.json index cda87f1..d8d9019 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "web-speed-test-server", - "version": "1.3.5", + "version": "1.3.6-beta", "private": true, "scripts": { "test": "mocha", diff --git a/routes/wpt.js b/routes/wpt.js index 085aa0a..799eb78 100644 --- a/routes/wpt.js +++ b/routes/wpt.js @@ -54,7 +54,7 @@ const wtp = (app) => { const quality = req.query.quality; let rollBarMsg = {testId: testId, thirdPartyErrorCode: "", file: path.basename((__filename))}; logger.info("Fetch WPT test results", rollBarMsg, req); - apiCaller.checkTestStatus(testId, quality, (error, result) => { + apiCaller.getTestResults(testId, quality, (error, result) => { routeCallback(error, result, res, rollBarMsg) }); }); diff --git a/test/resources/test2.json b/test/resources/test2.json new file mode 100644 index 0000000..165889b --- /dev/null +++ b/test/resources/test2.json @@ -0,0 +1,35 @@ +{ + "data": { + "average": { + "firstView": [] + }, + "bwDown": 0, + "bwUp": 0, + "completed": 1758191003, + "connectivity": "Native", + "fvonly": true, + "id": "250918_YiDc5V_5A0", + "latency": 0, + "location": "SLC_US_02:Chrome", + "median": [], + "mobile": 0, + "plr": "0", + "runs": { + "1": { + "firstView": [] + } + }, + "shaperLimit": 0, + "standardDeviation": { + "firstView": [] + }, + "successfulFVRuns": 0, + "summary": "https://www.webpagetest.org/results.php?test=250918_YiDc5V_5A0", + "testRuns": 1, + "testUrl": "https://s.codepen.io/maxro/debug/5b4dde20bc49345de60920ffd1ce8af1", + "url": "https://s.codepen.io/maxro/debug/5b4dde20bc49345de60920ffd1ce8af1" + }, + "statusCode": 200, + "statusText": "Test Complete", + "webPagetestVersion": "21.07" +} \ No newline at end of file diff --git a/test/wptTests.js b/test/wptTests.js index 671a1d0..3dc6d51 100644 --- a/test/wptTests.js +++ b/test/wptTests.js @@ -45,5 +45,10 @@ describe('Parse WPT result', () => { assert.isString(results.metaData.screenShot, 'There should be a screenshot'); assert.isObject(results.metaData.viewportSize, 'ViewportSize is not an object'); }) + it('No firstView data', () => { + let resultJson = JSON.parse(fs.readFileSync('./test/resources/test2.json')); + let results = wtpParser.parseTestResults(resultJson); + assert.equal(results.status, 'not_ready', 'Data not ready is expected'); + }) }); \ No newline at end of file diff --git a/wtp/apiCaller.js b/wtp/apiCaller.js index f492dc6..38c661c 100644 --- a/wtp/apiCaller.js +++ b/wtp/apiCaller.js @@ -14,59 +14,71 @@ const cloudinaryCaller = require('../cloudinary/apiCaller'); const {truncateString} = require('../util/strings'); const RESULTS_URL = 'https://www.webpagetest.org/jsonResult.php'; const RUN_TEST_URL = 'https://www.webpagetest.org/runtest.php'; -const GET_TEST_STATUS = 'https://www.webpagetest.org/testStatus.php'; const locationSelector = require('./locationSelector'); const apiKeys = require('./apiKey'); const getTestResults = async (testId, quality, cb) => { - let options = { - method: "GET", - url: RESULTS_URL, - searchParams: {test: testId}, - headers: { 'User-Agent': 'WebSpeedTest', 'X-WPT-API-KEY': apiKeys.getRandom() } - }; - let response; - let rollBarMsg = {}; - try { - response = await got(options) - logger.info("Fetched WPT test results"); - const {statusCode, body} = response; - let resBody = JSON.parse(body); - rollBarMsg = {testId: resBody.data.id, analyzedUrl: resBody.data.testUrl, thirdPartyErrorCode: "", file: path.basename((__filename))}; - if (statusCode !== 200) { - cb({status: 'error', message: 'WTP returned bad status with testId ' + testId, error: response.statusCode, logLevel: LOG_LEVEL_ERROR}, null, response, rollBarMsg); - return; + let options = { + method: "GET", + url: RESULTS_URL, + searchParams: {test: testId}, + headers: {'User-Agent': 'WebSpeedTest', 'X-WPT-API-KEY': apiKeys.getRandom()} + }; + let response; + let rollBarMsg = {}; + try { + response = await got(options) + logger.info("Fetched WPT test results"); + const {statusCode, body} = response; + let resBody = JSON.parse(body); + rollBarMsg = {testId: resBody.data.id, analyzedUrl: resBody.data.testUrl, thirdPartyErrorCode: "", file: path.basename((__filename))}; + if (statusCode !== 200) { + cb({status: 'error', message: 'WTP returned bad status with testId ' + testId, error: response.statusCode, logLevel: LOG_LEVEL_ERROR}, null, response, rollBarMsg); + return; + } + rollBarMsg.thirdPartyErrorCode = resBody.statusCode; + if (resBody.statusCode > 400) { + rollBarMsg.thirdPartyErrorCode = resBody.statusCode; + cb({status: 'error', message: 'WTP returned bad status with testId ' + testId, error: resBody}, null, response, rollBarMsg); + return; + } + if (resBody.statusCode >= 100 && resBody.statusCode < 200) { + cb(null, {status: 'success', message: 'test not finished', code: 150}, null, null); + return; + } + if (!body) { + cb({status: 'error', message: 'WTP returned empty body with testId ' + testId, error: 'empty body', logLevel: LOG_LEVEL_WARNING}, null, response, rollBarMsg); + return; + } + if (typeof resBody.data.statusCode !== 'undefined') { + cb({status: 'error', message: resBody.data.statusText + 'testId ' + testId, error: resBody, logLevel: LOG_LEVEL_WARNING}, null, response, rollBarMsg); + return; + } + let wtpRes = resultParser.parseTestResults(resBody); + if (!wtpRes) { + cb({status: 'error', message: 'WTP results are missing data with testId ' + testId, error: resBody, logLevel: LOG_LEVEL_ERROR}, null, response, rollBarMsg); + return; + } else if (wtpRes.status === 'not_ready') { + cb(null, {status: 'success', message: 'data not ready yet', code: 150}, null, null); + return; + } else if (wtpRes.status === 'error') { + cb(wtpRes); + return; + } else { + cloudinaryCaller(wtpRes.imageList, wtpRes.dpr, wtpRes.metaData, quality, cb, rollBarMsg); + } + } catch (e) { + cb({status: 'error', message: 'Error calling WTP with testId ' + testId, error: e, logLevel: LOG_LEVEL_ERROR}, null, response, rollBarMsg); + return; } - if (!body) { - cb({status: 'error', message: 'WTP returned empty body with testId ' + testId, error: 'empty body', logLevel: LOG_LEVEL_WARNING}, null, response, rollBarMsg); - return; - } - if (typeof resBody.data.statusCode !== 'undefined') { - cb({status: 'error', message: resBody.data.statusText + 'testId ' + testId, error: resBody, logLevel: LOG_LEVEL_WARNING}, null, response, rollBarMsg); - return; - } - let wtpRes = resultParser.parseTestResults(resBody); - if (!wtpRes) { - cb({status: 'error', message: 'WTP results are missing data with testId ' + testId, error: resBody, logLevel: LOG_LEVEL_ERROR}, null, response, rollBarMsg); - return; - } else if(wtpRes.status === 'error') { - cb(wtpRes); - return; - } else { - cloudinaryCaller(wtpRes.imageList, wtpRes.dpr, wtpRes.metaData, quality, cb, rollBarMsg); - } - } catch (e) { - cb({status: 'error', message: 'Error calling WTP with testId ' + testId, error: e, logLevel: LOG_LEVEL_ERROR}, null, response, rollBarMsg); - return; - } }; const runWtpTest = async (url, mobile, cb) => { - //logger.info('Running new test ' + url); - let options = { - method: "POST", - url: RUN_TEST_URL, - searchParams: { + //logger.info('Running new test ' + url); + let options = { + method: "POST", + url: RUN_TEST_URL, + searchParams: { url: url, f: "json", width: config.get('wtp.viewportWidth'), @@ -76,82 +88,43 @@ const runWtpTest = async (url, mobile, cb) => { mobile: (mobile) ? 1 : 0, fvonly: 1, // first view only timeline: 1 // workaround for WPT sometimes hanging on getComputedStyle() - }, - headers: { 'User-Agent': 'WebSpeedTest', 'X-WPT-API-KEY': apiKeys.getRandom() }, - throwHttpErrors: false - }; - let response; - let rollBarMsg = {testId: "", analyzedUrl: url, thirdPartyErrorCode: "", thirdPartyErrorMsg: "", file: path.basename((__filename))}; - try { - response = await got(options); - const {statusCode, body} = response; - if (statusCode !== 200) { - rollBarMsg.thirdPartyErrorCode = response.statusCode; - rollBarMsg.thirdPartyErrorBody = body && truncateString(body, 1000) || ""; - cb({status: 'error', message: 'WTP returned bad status with url ' + url, error: response.statusMessage, logLevel: LOG_LEVEL_ERROR}, null, response, rollBarMsg); - return; - } - if (!body) { - cb({status: 'error', message: 'WTP returned empty body with url ' + url, error: 'empty body'}, null, response, rollBarMsg); - return; - } - let bodyJson = JSON.parse(body); - rollBarMsg.testId = (typeof bodyJson.data !== 'undefined' && typeof bodyJson.data.testId !== 'undefined') ? - (bodyJson.data.testId) : - "N/A"; - logger.info("Started WPT test", {"testId": rollBarMsg.testId}); - let testId = resultParser.parseTestResponse(bodyJson, rollBarMsg); - if (typeof testId === 'object') { - cb(null, testId); - return; - } - cb(null, {status: "success", data: {testId}}); - } catch (error) { - cb({status: 'error', message: 'Error calling WTP with url ' + url, error: error}, null, response, rollBarMsg); - return; - } -}; - -const checkTestStatus = async (testId, quality, cb) => { - let options = { - method: "GET", - url: GET_TEST_STATUS, - searchParams: {test: testId, f: "json"}, - 'headers': { 'User-Agent': 'WebSpeedTest', 'X-WPT-API-KEY': apiKeys.getRandom() } - }; - let response; - let rollBarMsg = {}; - try { - response = await got(options); - logger.info("Fetched WPT test status"); - const {statusCode, body} = response; - let bodyJson = JSON.parse(body); - rollBarMsg = {testId: testId, thirdPartyErrorCode: "", file: path.basename((__filename))}; - if (statusCode !== 200) { - cb({status: 'error', message: 'WTP returned bad status testId ' + testId , error: response.statusCode}, null, response, rollBarMsg); - return; - } - //logger.debug('Test status code ' + bodyJson.statusCode, rollBarMsg); - rollBarMsg.thirdPartyErrorCode = bodyJson.statusCode; - if (bodyJson.statusCode > 400) { - rollBarMsg.thirdPartyErrorCode = bodyJson.statusCode; - cb({status: 'error', message: 'WTP returned bad status with testId ' + testId, error: bodyJson}, null, response, rollBarMsg); - return; - } - if (bodyJson.statusCode === 200 || bodyJson.statusCode === 400) { - getTestResults(testId, quality, cb); - } - if (bodyJson.statusCode >= 100 && bodyJson.statusCode < 200) { - cb(null, {status: 'success', message: 'test not finished', code: 150}, null, null); + }, + headers: {'User-Agent': 'WebSpeedTest', 'X-WPT-API-KEY': apiKeys.getRandom()}, + throwHttpErrors: false + }; + let response; + let rollBarMsg = {testId: "", analyzedUrl: url, thirdPartyErrorCode: "", thirdPartyErrorMsg: "", file: path.basename((__filename))}; + try { + response = await got(options); + const {statusCode, body} = response; + if (statusCode !== 200) { + rollBarMsg.thirdPartyErrorCode = response.statusCode; + rollBarMsg.thirdPartyErrorBody = body && truncateString(body, 1000) || ""; + cb({status: 'error', message: 'WTP returned bad status with url ' + url, error: response.statusMessage, logLevel: LOG_LEVEL_ERROR}, null, response, rollBarMsg); + return; + } + if (!body) { + cb({status: 'error', message: 'WTP returned empty body with url ' + url, error: 'empty body'}, null, response, rollBarMsg); + return; + } + let bodyJson = JSON.parse(body); + rollBarMsg.testId = (typeof bodyJson.data !== 'undefined' && typeof bodyJson.data.testId !== 'undefined') ? + (bodyJson.data.testId) : + "N/A"; + logger.info("Started WPT test", {"testId": rollBarMsg.testId}); + let testId = resultParser.parseTestResponse(bodyJson, rollBarMsg); + if (typeof testId === 'object') { + cb(null, testId); + return; + } + cb(null, {status: "success", data: {testId}}); + } catch (error) { + cb({status: 'error', message: 'Error calling WTP with url ' + url, error: error}, null, response, rollBarMsg); + return; } - } catch (error) { - cb({status: 'error', message: 'Error checking WTP status with testId ' + testId, error: error}, null, response, rollBarMsg); - return; - } }; module.exports = { - getTestResults, - runWtpTest, - checkTestStatus + getTestResults, + runWtpTest }; diff --git a/wtp/wtpResultsParser.js b/wtp/wtpResultsParser.js index 5af5cf6..16f01c3 100644 --- a/wtp/wtpResultsParser.js +++ b/wtp/wtpResultsParser.js @@ -15,6 +15,12 @@ const path = require('path'); const parseTestResults = (testJson) => { let rollBarMsg = {testId: testJson.data.id, analyzedUrl: testJson.data.testUrl, thirdPartyErrorCode: "", file: path.basename((__filename))}; try { + // check if data is available + if (Array.isArray(testJson.data.median) && testJson.data.median.length == 0) { + logger.warn("Test results not ready yet", rollBarMsg); + return {status: 'not_ready', message: 'data_not_ready'}; + } + let browserName = _.get(testJson, 'data.location', 'somePlace:N/A').split(':')[1]; if ('firefox' === browserName.toLowerCase()) { logger.warn("Test run with firefox that is not supported", rollBarMsg);