From b3d1e0b46c9955ccddcc4fdb6957c5dd95a4ca4b Mon Sep 17 00:00:00 2001 From: Kristinn Vikar Date: Wed, 1 Sep 2021 16:07:12 +0000 Subject: [PATCH 1/6] Majorly simplify and fix Earthquake API --- endpoints/earthquake/index.js | 54 +++++++++++------------------------ 1 file changed, 17 insertions(+), 37 deletions(-) diff --git a/endpoints/earthquake/index.js b/endpoints/earthquake/index.js index 4184d607..cfe22107 100644 --- a/endpoints/earthquake/index.js +++ b/endpoints/earthquake/index.js @@ -38,7 +38,6 @@ function getEarthquakes(callback, params) { */ function parseJavaScriptVariable(body) { // Work with empty string if scraping fails. - let res let jsonString = '' // Create a cheerio object from response body. const $ = cheerio.load(body) @@ -49,42 +48,23 @@ function parseJavaScriptVariable(body) { jsonString = $(elem).html().match(/(VI\.quakeInfo = )(.+);/)[2] } }) + + jsonObject = new Function("return " + jsonString)() + console.log(jsonObject) - // Convert the variable to JavaScript Object Notation and fix seperators in values. - const resString = jsonString.replace(/(:\'[-0-9]\d*)(,)(\d*)/g, '$1.$3') - - // Create a Regular expression and change date representation. - // eslint-disable-next-line max-len - const regexDate = /(\'t\':)new Date\((([0-9.,-]+),([0-9.,-]+),([0-9.,-]+),([0-9.,-]+),([0-9.,-]+),([0-9.,-]+))\)(,\'a\')/g - const dateReplace = (match, p1, p2, p3, p4, p5, p6, p7, p8, p9) => { - const parsedDate = new Date( - parseInt(p3, 10), - parseInt(p4.split('-')[0], 10) - 1, parseInt(p5, 10), - parseInt(p6, 10), - parseInt(p7, 10), - parseInt(p8, 10) - ) - return `${p1}\\${parsedDate.toISOString()}\\${p9}` - } - - try { - // Create semi-final JSON string. - res = JSON.parse(resString.replace(regexDate, dateReplace).replace(/\'/g, '"')) - } catch (ex) { - return JSON.parse([{ error: 'Error parsing source.' }]) - } // rename fields to match current specs const resFields = [] - res.forEach((element) => { - const tmpRow = {} - tmpRow.timestamp = element.t - tmpRow.latitude = element.lat - tmpRow.longitude = element.lon - tmpRow.depth = element.dep - tmpRow.size = element.s - tmpRow.quality = element.q - tmpRow.humanReadableLocation = `${element.dL} km ${element.dD} af ${element.dR}` - resFields.push(tmpRow) + jsonObject.forEach((element) => { + const row = { + timestamp: element.t, + latitude: element.lat, + longitude: element.lon, + depth: element.dep, + size: element.s, + quality: element.q, + humanReadableLocation: `${element.dL} km ${element.dD} af ${element.dR}` + } + resFields.push(row) }) return resFields @@ -154,9 +134,9 @@ function parseList(body) { } /* - * Hraun table parse + * Hraun table parse, currently broken */ -app.get('/earthquake/is', (req, res) => { +app.get('/earthquake/is/sec', (req, res) => { getEarthquakes((error, body) => { if (error) { return res.status(500).json({ error: error.toString() }) @@ -169,7 +149,7 @@ app.get('/earthquake/is', (req, res) => { /* * Main vedur.is website (JS variable included in the source) (secondary source of information). */ -app.get('/earthquake/is/sec', (req, res) => { +app.get('/earthquake/is', (req, res) => { getEarthquakes((error, body) => { if (error) { return res.json({ error: error.toString() }) From 4551d50efc2951b85dcf7d1e24f130bede1f146d Mon Sep 17 00:00:00 2001 From: Kristinn Vikar Date: Wed, 1 Sep 2021 16:09:26 +0000 Subject: [PATCH 2/6] Forgot to remove a debug print --- endpoints/earthquake/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/endpoints/earthquake/index.js b/endpoints/earthquake/index.js index cfe22107..8140eeb0 100644 --- a/endpoints/earthquake/index.js +++ b/endpoints/earthquake/index.js @@ -50,7 +50,6 @@ function parseJavaScriptVariable(body) { }) jsonObject = new Function("return " + jsonString)() - console.log(jsonObject) // rename fields to match current specs const resFields = [] From f16bfab029953092acffcf2357886681edfd02c3 Mon Sep 17 00:00:00 2001 From: Kristinn Vikar Date: Wed, 1 Sep 2021 16:11:17 +0000 Subject: [PATCH 3/6] add documentation and fix previous documentation --- endpoints/earthquake/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/endpoints/earthquake/index.js b/endpoints/earthquake/index.js index 8140eeb0..42f881f0 100644 --- a/endpoints/earthquake/index.js +++ b/endpoints/earthquake/index.js @@ -49,6 +49,7 @@ function parseJavaScriptVariable(body) { } }) + // Create a function that returns the JSON object (handles the date stuff for us) jsonObject = new Function("return " + jsonString)() // rename fields to match current specs @@ -146,7 +147,7 @@ app.get('/earthquake/is/sec', (req, res) => { }) /* - * Main vedur.is website (JS variable included in the source) (secondary source of information). + * Main vedur.is website (JS variable included in the source). */ app.get('/earthquake/is', (req, res) => { getEarthquakes((error, body) => { From f5d6c8af2e42f1de0b4e748119189fe2bba28031 Mon Sep 17 00:00:00 2001 From: Kristinn Vikar Date: Wed, 1 Sep 2021 16:18:45 +0000 Subject: [PATCH 4/6] Fix ESLint complaints --- endpoints/earthquake/index.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/endpoints/earthquake/index.js b/endpoints/earthquake/index.js index 42f881f0..659735b7 100644 --- a/endpoints/earthquake/index.js +++ b/endpoints/earthquake/index.js @@ -48,9 +48,11 @@ function parseJavaScriptVariable(body) { jsonString = $(elem).html().match(/(VI\.quakeInfo = )(.+);/)[2] } }) - + // Create a function that returns the JSON object (handles the date stuff for us) - jsonObject = new Function("return " + jsonString)() + // Disable ESLint because we have to evaluate the json string + // eslint-disable-next-line eqeqeq + var jsonObject = new Function('return ' + jsonString)() // rename fields to match current specs const resFields = [] From 50b8cf5a057eed84e2ec1719d6b2aa38dbe7f497 Mon Sep 17 00:00:00 2001 From: Kristinn Vikar Date: Wed, 1 Sep 2021 16:30:48 +0000 Subject: [PATCH 5/6] Fix more ESLint errors --- endpoints/earthquake/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/endpoints/earthquake/index.js b/endpoints/earthquake/index.js index 659735b7..32d16802 100644 --- a/endpoints/earthquake/index.js +++ b/endpoints/earthquake/index.js @@ -51,8 +51,8 @@ function parseJavaScriptVariable(body) { // Create a function that returns the JSON object (handles the date stuff for us) // Disable ESLint because we have to evaluate the json string - // eslint-disable-next-line eqeqeq - var jsonObject = new Function('return ' + jsonString)() + // eslint-disable-next-line no-new-func + const jsonObject = new Function('return ' + jsonString)() // rename fields to match current specs const resFields = [] From 5049ca73d7f83ed9dc7fd76bc46b1aa36d884ad6 Mon Sep 17 00:00:00 2001 From: Kristinn Vikar Date: Thu, 23 Sep 2021 20:08:36 +0000 Subject: [PATCH 6/6] Fix results in earthquake API being strings intead of floats. --- endpoints/earthquake/index.js | 336 +++++++++++++++++----------------- 1 file changed, 168 insertions(+), 168 deletions(-) diff --git a/endpoints/earthquake/index.js b/endpoints/earthquake/index.js index 32d16802..7b987e89 100644 --- a/endpoints/earthquake/index.js +++ b/endpoints/earthquake/index.js @@ -1,168 +1,168 @@ -/* eslint-disable no-prototype-builtins */ -/* eslint-disable no-restricted-syntax */ -/* eslint-disable no-useless-escape */ -/* eslint-disable prefer-destructuring */ -const request = require('request') -const cheerio = require('cheerio') -const helpers = require('apis-helpers') -const app = require('../../server') - -const browser = helpers.browser - -/* - This function only handles the request part and calls callback with - the body - */ -function getEarthquakes(callback, params) { - // eslint-disable-next-line eqeqeq - const reqParams = !params ? { - url: 'http://hraun.vedur.is/ja/skjalftar/skjlisti.html', - headers: { 'User-Agent': browser() }, - // needed for some reason.. defaulting to ISO-8859-1 - encoding: 'binary', - } : params - - request(reqParams, (error, res, body) => { - if (error || res.statusCode !== 200) { - return callback(new Error('Could not retrieve the data from the data source')) - } - - return callback(error, body) - }) -} - -/* - * This function traverses the DOM, and looks for a JS variable that is included - * in the source of vedur.is. - * Note that it is synchronous. - */ -function parseJavaScriptVariable(body) { - // Work with empty string if scraping fails. - let jsonString = '' - // Create a cheerio object from response body. - const $ = cheerio.load(body) - - // Find the variable inside one of the