Skip to content

v2 refactor cm 20210104 #347

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

v2 refactor cm 20210104 #347

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 26 additions & 27 deletions .eslintrc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,29 +1,28 @@
{
"env": {
"node": true,
"commonjs": true,
"es6": true
},
"parserOptions": {
"ecmaVersion": 9
},
"rules": {
"max-len": [1, { "code": 120, "tabWidth": 4, "ignoreUrls": true, "ignoreComments": true, "ignoreStrings": true }],
"semi": [2, "never"],
"quotes": [2, "single"],
"comma-dangle": [2, "never"],
"object-curly-spacing": [2, "always"],
"indent": [2, 2, { "SwitchCase": 1 }],
"capitalized-comments": [2, "never"],
"spaced-comment": [2, "always"],
"multiline-comment-style": [2, "starred-block"],
"no-multiple-empty-lines": [2, { "max": 2 }],
"no-mixed-spaces-and-tabs": 2,
"no-trailing-spaces": 2,
"no-var": 2,
"no-redeclare": 2,
"no-const-assign": 2,
"no-self-assign": 2,
"no-unused-vars": [2, { "args": "none" }]
}
"env": {
"node": true,
"commonjs": true,
"es6": true
},
"parserOptions": {
"ecmaVersion": 9
},
"rules": {
"max-len": [1, { "code": 120, "tabWidth": 4, "ignoreUrls": true, "ignoreComments": true, "ignoreStrings": true }],
"semi": [2, "never"],
"quotes": [2, "single"],
"comma-dangle": [2, "never"],
"object-curly-spacing": [2, "always"],
"indent": [2, 2, { "SwitchCase": 1 }],
"spaced-comment": [2, "always"],
"multiline-comment-style": [2, "starred-block"],
"no-multiple-empty-lines": [2, { "max": 2 }],
"no-mixed-spaces-and-tabs": 2,
"no-trailing-spaces": 2,
"no-var": 2,
"no-redeclare": 2,
"no-const-assign": 2,
"no-self-assign": 2,
"no-unused-vars": [2, { "args": "none" }]
}
}
3 changes: 1 addition & 2 deletions lib/bankHolidayChecker.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,12 @@

'use strict'

const moment = require('moment')
const bankHolidays = require('./../conf/bankHolidays.json')

function bankHolidayChecker() {
let found
try {
const today = moment().format('YYYY-MM-DD')
const today = new Date().toLocaleDateString('hu-HU')
const year = today.match(/[0-9]{4}/)[0]
found = bankHolidays[year].includes(today)
} catch (e) {
Expand Down
6 changes: 3 additions & 3 deletions lib/bankHolidayChecker.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ const bankHolidayChecker = require('./../lib/bankHolidayChecker')

jest.mock('moment', () => () => ({ format: () => '2019-12-24' }))

describe('bank holiday checker', function() {
test('should recognize christmas', function() {
let bankHoliday = bankHolidayChecker.bankHolidayChecker()
describe('bank holiday checker', () => {
test('should recognize christmas', () => {
const bankHoliday = bankHolidayChecker.bankHolidayChecker()
expect(bankHoliday).toBe(true)
})
})
7 changes: 2 additions & 5 deletions lib/dateCatcher.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
*/

const moment = require('moment')
const todayDotSeparated = require('./../src/date').date.todayDotSeparated
const { todayDotSeparated } = require('./../src/date').date

/*
* @param {boolean} interval: optional argument defaults to false
Expand Down Expand Up @@ -59,10 +59,7 @@ async function dateCatcher(textContent, interval = false) {
const month = actualDateStrings[1] // e.g. July => július
const day = actualDateStrings[2]

let parsedDate = moment([year, 0, day])
.month(month)
.locale('hu')
.format('YYYY MMMM DD')
let parsedDate = moment([year, 0, day]).month(month).locale('hu').format('YYYY MMMM DD')
parsedDate = moment(parsedDate, 'YYYY MMMM DD').format('w')

let currentWeek = moment(todayDotSeparated, 'YYYY-MM-DD').format('w')
Expand Down
23 changes: 12 additions & 11 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,25 @@
{
"license": "MIT",
"main": "./src/dailyMenuScraper.js",
"main": "./src/index.js",
"dependencies": {
"cors": "^2.8.5",
"express": "^4.17.1",
"moment": "^2.24.0",
"mongodb": "^3.3.4",
"puppeteer": "^5.0.0",
"request": "^2.88.0"
"moment": "^2.29.1",
"mongodb": "^3.6.2",
"puppeteer": "^5.3.1",
"request": "^2.88.0",
"node-fetch": "^2.6.1"
},
"devDependencies": {
"codecov": "^3.7.1",
"eslint": "^7.0.0",
"jest": "^26.0.1",
"nock": "^13.0.2"
"codecov": "^3.8.0",
"eslint": "^7.11.0",
"jest": "^26.5.3",
"nock": "^13.0.4"
},
"scripts": {
"start": "node ./src/server.js",
"scrape": "node ./src/dailyMenuScraper.js",
"scrape-debug": "node ./src/dailyMenuScraper.js --debug",
"scrape": "node ./src/index.js",
"scrape-debug": "node ./src/index.js --debug",
"test": "jest --verbose --runInBand --detectOpenHandles",
"lint": "eslint"
},
Expand Down
6 changes: 3 additions & 3 deletions scrapers/bank3.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,9 @@ const objectDecider = require('./../lib/objectDecider')
const priceCatcher = require('./../lib/priceCatcher')
const priceCompareToDb = require('./../lib/priceCompareToDb')
const stringValueCleaner = require('./../lib/stringValueCleaner')
const browserWSEndpoint = require('./../src/dailyMenuScraper').browserWSEndpoint
const finalJSON = require('./../src/dailyMenuScraper').finalJSON
const finalMongoJSON = require('./../src/dailyMenuScraper').finalMongoJSON
const browserWSEndpoint = require('./../src/index').browserWSEndpoint
const finalJSON = require('./../src/index').finalJSON
const finalMongoJSON = require('./../src/index').finalMongoJSON
const RestaurantMenuOutput = require('./../src/restaurantMenuClasses').RestaurantMenuOutput
const RestaurantMenuDb = require('./../src/restaurantMenuClasses').RestaurantMenuDb

Expand Down
Empty file added scrapers/bank3_2.js
View file
Open in desktop
Empty file.
6 changes: 3 additions & 3 deletions scrapers/bodza.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ const puppeteer = require('puppeteer')
const priceCatcher = require('./../lib/priceCatcher')
const objectDecider = require('./../lib/objectDecider')
const priceCompareToDb = require('./../lib/priceCompareToDb')
const browserWSEndpoint = require('./../src/dailyMenuScraper').browserWSEndpoint
const browserWSEndpoint = require('./../src/index').browserWSEndpoint
const todayDotSeparated = require('./../src/date').date.todayDotSeparated
const finalJSON = require('./../src/dailyMenuScraper').finalJSON
const finalMongoJSON = require('./../src/dailyMenuScraper').finalMongoJSON
const finalJSON = require('./../src/index').finalJSON
const finalMongoJSON = require('./../src/index').finalMongoJSON
const RestaurantMenuOutput = require('./../src/restaurantMenuClasses').RestaurantMenuOutput
const RestaurantMenuDb = require('./../src/restaurantMenuClasses').RestaurantMenuDb

Expand Down
22 changes: 22 additions & 0 deletions scrapers/domScraper.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
const puppeteer = require('puppeteer')
const { RestaurantMenuOutput } = require('./../src/restaurantMenuClasses')
const { RestaurantMenuDb } = require('./../src/restaurantMenuClasses')
const { browserWSEndpoint } = require('./../src/index')
const { finalJSON } = require('./../src/index')
const { finalMongoJSON } = require('./../src/index')

const scraper = async () => {
const browser = await puppeteer.connect({ browserWSEndpoint })
const page = await browser.newPage()

// abort all images, source: https://github.com/GoogleChrome/puppeteer/blob/master/examples/block-images.js
await page.setRequestInterception(true)
page.on('request', request => {
if (request.resourceType() === 'image') {
request.abort()
} else {
request.continue()
}
})
}
module.exports.scraper = scraper
6 changes: 3 additions & 3 deletions scrapers/fruccola.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@
const puppeteer = require('puppeteer')
const objectDecider = require('./../lib/objectDecider')
const priceCompareToDb = require('./../lib/priceCompareToDb')
const browserWSEndpoint = require('./../src/dailyMenuScraper').browserWSEndpoint
const finalJSON = require('./../src/dailyMenuScraper').finalJSON
const finalMongoJSON = require('./../src/dailyMenuScraper').finalMongoJSON
const browserWSEndpoint = require('./../src/index').browserWSEndpoint
const finalJSON = require('./../src/index').finalJSON
const finalMongoJSON = require('./../src/index').finalMongoJSON
const RestaurantMenuOutput = require('./../src/restaurantMenuClasses').RestaurantMenuOutput
const RestaurantMenuDb = require('./../src/restaurantMenuClasses').RestaurantMenuDb

Expand Down
6 changes: 3 additions & 3 deletions scrapers/i55.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ const objectDecider = require('./../lib/objectDecider')
const priceCatcher = require('./../lib/priceCatcher')
const priceCompareToDb = require('./../lib/priceCompareToDb')
const stringValueCleaner = require('./../lib/stringValueCleaner')
const browserWSEndpoint = require('./../src/dailyMenuScraper').browserWSEndpoint
const finalJSON = require('./../src/dailyMenuScraper').finalJSON
const finalMongoJSON = require('./../src/dailyMenuScraper').finalMongoJSON
const browserWSEndpoint = require('./../src/index').browserWSEndpoint
const finalJSON = require('./../src/index').finalJSON
const finalMongoJSON = require('./../src/index').finalMongoJSON
const RestaurantMenuOutput = require('./../src/restaurantMenuClasses').RestaurantMenuOutput
const RestaurantMenuDb = require('./../src/restaurantMenuClasses').RestaurantMenuDb

Expand Down
6 changes: 3 additions & 3 deletions scrapers/kamra.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ const puppeteer = require('puppeteer')
const objectDecider = require('./../lib/objectDecider')
const priceCatcher = require('./../lib/priceCatcher')
const priceCompareToDb = require('./../lib/priceCompareToDb')
const browserWSEndpoint = require('./../src/dailyMenuScraper').browserWSEndpoint
const finalJSON = require('./../src/dailyMenuScraper').finalJSON
const finalMongoJSON = require('./../src/dailyMenuScraper').finalMongoJSON
const browserWSEndpoint = require('./../src/index').browserWSEndpoint
const finalJSON = require('./../src/index').finalJSON
const finalMongoJSON = require('./../src/index').finalMongoJSON
const RestaurantMenuOutput = require('./../src/restaurantMenuClasses').RestaurantMenuOutput
const RestaurantMenuDb = require('./../src/restaurantMenuClasses').RestaurantMenuDb

Expand Down
4 changes: 2 additions & 2 deletions scrapers/karcsi.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ const ocrSpaceApiSimple = require('./../lib/ocrSpaceApiSimple')
const stringValueCleaner = require('./../lib/stringValueCleaner')
const priceCompareToDb = require('./../lib/priceCompareToDb')
const today = require('./../src/date').date.today
const finalJSON = require('./../src/dailyMenuScraper').finalJSON
const finalMongoJSON = require('./../src/dailyMenuScraper').finalMongoJSON
const finalJSON = require('./../src/index').finalJSON
const finalMongoJSON = require('./../src/index').finalMongoJSON
const RestaurantMenuOutput = require('./../src/restaurantMenuClasses').RestaurantMenuOutput
const RestaurantMenuDb = require('./../src/restaurantMenuClasses').RestaurantMenuDb

Expand Down
27 changes: 6 additions & 21 deletions scrapers/ketszerecsen.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ const puppeteer = require('puppeteer')
const objectDecider = require('./../lib/objectDecider')
const priceCatcher = require('./../lib/priceCatcher')
const priceCompareToDb = require('./../lib/priceCompareToDb')
const browserWSEndpoint = require('./../src/dailyMenuScraper').browserWSEndpoint
const browserWSEndpoint = require('./../src/index').browserWSEndpoint
const today = require('./../src/date').date.today
const finalJSON = require('./../src/dailyMenuScraper').finalJSON
const finalMongoJSON = require('./../src/dailyMenuScraper').finalMongoJSON
const finalJSON = require('./../src/index').finalJSON
const finalMongoJSON = require('./../src/index').finalMongoJSON
const RestaurantMenuOutput = require('./../src/restaurantMenuClasses').RestaurantMenuOutput
const RestaurantMenuDb = require('./../src/restaurantMenuClasses').RestaurantMenuDb

Expand Down Expand Up @@ -64,8 +64,7 @@ async function scraper() {
let paramColor = '#000000'
let paramTitleString = 'Két Szerecsen Bisztro'
let paramUrl = 'https://ketszerecsen.hu/#daily'
let paramIcon =
'https://images.deliveryhero.io/image/netpincer/caterer/sh-9a3e84d0-2e42-11e2-9d48-7a92eabdcf20/logo.png'
let paramIcon = 'https://images.deliveryhero.io/image/netpincer/caterer/sh-9a3e84d0-2e42-11e2-9d48-7a92eabdcf20/logo.png'
let paramValueString
let paramPriceString
let paramPriceCurrency
Expand All @@ -77,22 +76,8 @@ async function scraper() {
let mongoObj = null

// @ KETSZERECSEN selectors [1: first course, 2: main course]
let ketszerecsenArray1 = [
null,
'p:nth-child(4)',
'p:nth-child(7)',
'p:nth-child(10)',
'p:nth-child(13)',
'p:nth-child(16)'
]
let ketszerecsenArray2 = [
null,
'p:nth-child(5)',
'p:nth-child(8)',
'p:nth-child(11)',
'p:nth-child(14)',
'p:nth-child(17)'
]
let ketszerecsenArray1 = [null, 'p:nth-child(4)', 'p:nth-child(7)', 'p:nth-child(10)', 'p:nth-child(13)', 'p:nth-child(16)']
let ketszerecsenArray2 = [null, 'p:nth-child(5)', 'p:nth-child(8)', 'p:nth-child(11)', 'p:nth-child(14)', 'p:nth-child(17)']

try {
await page.goto(paramUrl, { waitUntil: 'networkidle2' })
Expand Down
6 changes: 3 additions & 3 deletions scrapers/korhely.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ const objectDecider = require('./../lib/objectDecider')
const priceCatcher = require('./../lib/priceCatcher')
const priceCompareToDb = require('./../lib/priceCompareToDb')
const stringValueCleaner = require('./../lib/stringValueCleaner')
const browserWSEndpoint = require('./../src/dailyMenuScraper').browserWSEndpoint
const finalJSON = require('./../src/dailyMenuScraper').finalJSON
const finalMongoJSON = require('./../src/dailyMenuScraper').finalMongoJSON
const browserWSEndpoint = require('./../src/index').browserWSEndpoint
const finalJSON = require('./../src/index').finalJSON
const finalMongoJSON = require('./../src/index').finalMongoJSON
const RestaurantMenuOutput = require('./../src/restaurantMenuClasses').RestaurantMenuOutput
const RestaurantMenuDb = require('./../src/restaurantMenuClasses').RestaurantMenuDb

Expand Down
6 changes: 3 additions & 3 deletions scrapers/mozsar.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,10 @@ const objectDecider = require('./../lib/objectDecider')
const priceCatcher = require('./../lib/priceCatcher')
const priceCompareToDb = require('./../lib/priceCompareToDb')
const stringValueCleaner = require('./../lib/stringValueCleaner')
const browserWSEndpoint = require('./../src/dailyMenuScraper').browserWSEndpoint
const browserWSEndpoint = require('./../src/index').browserWSEndpoint
const today = require('./../src/date').date.today
const finalJSON = require('./../src/dailyMenuScraper').finalJSON
const finalMongoJSON = require('./../src/dailyMenuScraper').finalMongoJSON
const finalJSON = require('./../src/index').finalJSON
const finalMongoJSON = require('./../src/index').finalMongoJSON
const RestaurantMenuOutput = require('./../src/restaurantMenuClasses').RestaurantMenuOutput
const RestaurantMenuDb = require('./../src/restaurantMenuClasses').RestaurantMenuDb

Expand Down
6 changes: 3 additions & 3 deletions scrapers/nokedli.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ const puppeteer = require('puppeteer')
const objectDecider = require('./../lib/objectDecider')
const ocrSpaceApiSimple = require('./../lib/ocrSpaceApiSimple')
const stringValueCleaner = require('./../lib/stringValueCleaner')
const browserWSEndpoint = require('./../src/dailyMenuScraper').browserWSEndpoint
const browserWSEndpoint = require('./../src/index').browserWSEndpoint
const today = require('./../src/date').date.today
const finalJSON = require('./../src/dailyMenuScraper').finalJSON
const finalMongoJSON = require('./../src/dailyMenuScraper').finalMongoJSON
const finalJSON = require('./../src/index').finalJSON
const finalMongoJSON = require('./../src/index').finalMongoJSON
const RestaurantMenuOutput = require('./../src/restaurantMenuClasses').RestaurantMenuOutput
const RestaurantMenuDb = require('./../src/restaurantMenuClasses').RestaurantMenuDb

Expand Down
6 changes: 3 additions & 3 deletions scrapers/ocrFacebookImage.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@ const ocrSpaceApiSimple = require('./../lib/ocrSpaceApiSimple')
const priceCatcher = require('./../lib/priceCatcher')
const priceCompareToDb = require('./../lib/priceCompareToDb')
const stringValueCleaner = require('./../lib/stringValueCleaner')
const browserWSEndpoint = require('./../src/dailyMenuScraper').browserWSEndpoint
const browserWSEndpoint = require('./../src/index').browserWSEndpoint
const today = require('./../src/date').date.today
const finalJSON = require('./../src/dailyMenuScraper').finalJSON
const finalMongoJSON = require('./../src/dailyMenuScraper').finalMongoJSON
const finalJSON = require('./../src/index').finalJSON
const finalMongoJSON = require('./../src/index').finalMongoJSON
const RestaurantMenuOutput = require('./../src/restaurantMenuClasses').RestaurantMenuOutput
const RestaurantMenuDb = require('./../src/restaurantMenuClasses').RestaurantMenuDb

Expand Down
6 changes: 3 additions & 3 deletions scrapers/roza.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ const puppeteer = require('puppeteer')
const objectDecider = require('./../lib/objectDecider')
const priceCatcher = require('./../lib/priceCatcher')
const stringValueCleaner = require('./../lib/stringValueCleaner')
const browserWSEndpoint = require('./../src/dailyMenuScraper').browserWSEndpoint
const finalJSON = require('./../src/dailyMenuScraper').finalJSON
const finalMongoJSON = require('./../src/dailyMenuScraper').finalMongoJSON
const browserWSEndpoint = require('./../src/index').browserWSEndpoint
const finalJSON = require('./../src/index').finalJSON
const finalMongoJSON = require('./../src/index').finalMongoJSON
const RestaurantMenuOutput = require('./../src/restaurantMenuClasses').RestaurantMenuOutput
const RestaurantMenuDb = require('./../src/restaurantMenuClasses').RestaurantMenuDb

Expand Down
6 changes: 3 additions & 3 deletions scrapers/suppe.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@
const puppeteer = require('puppeteer')
const objectDecider = require('./../lib/objectDecider')
const priceCompareToDb = require('./../lib/priceCompareToDb')
const browserWSEndpoint = require('./../src/dailyMenuScraper').browserWSEndpoint
const browserWSEndpoint = require('./../src/index').browserWSEndpoint
const today = require('./../src/date').date.today
const dayNames = require('./../src/date').date.dayNames
const finalJSON = require('./../src/dailyMenuScraper').finalJSON
const finalMongoJSON = require('./../src/dailyMenuScraper').finalMongoJSON
const finalJSON = require('./../src/index').finalJSON
const finalMongoJSON = require('./../src/index').finalMongoJSON
const RestaurantMenuOutput = require('./../src/restaurantMenuClasses').RestaurantMenuOutput
const RestaurantMenuDb = require('./../src/restaurantMenuClasses').RestaurantMenuDb

Expand Down
6 changes: 3 additions & 3 deletions scrapers/vian.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ const puppeteer = require('puppeteer')
const objectDecider = require('./../lib/objectDecider')
const priceCatcher = require('./../lib/priceCatcher')
const priceCompareToDb = require('./../lib/priceCompareToDb')
const browserWSEndpoint = require('./../src/dailyMenuScraper').browserWSEndpoint
const browserWSEndpoint = require('./../src/index').browserWSEndpoint
const today = require('./../src/date').date.today
const finalJSON = require('./../src/dailyMenuScraper').finalJSON
const finalMongoJSON = require('./../src/dailyMenuScraper').finalMongoJSON
const finalJSON = require('./../src/index').finalJSON
const finalMongoJSON = require('./../src/index').finalMongoJSON
const RestaurantMenuOutput = require('./../src/restaurantMenuClasses').RestaurantMenuOutput
const RestaurantMenuDb = require('./../src/restaurantMenuClasses').RestaurantMenuDb

Expand Down
Loading