Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
"top-sites": "~1.1.220"
},
"devDependencies": {
"@browserless/test": "latest",
"@commitlint/cli": "latest",
"@commitlint/config-conventional": "latest",
"@ksmithut/prettier-standard": "latest",
Expand Down
20 changes: 14 additions & 6 deletions test/encoding.js
Original file line number Diff line number Diff line change
@@ -1,30 +1,38 @@
'use strict'

const { getBrowserContext } = require('@browserless/test')
const test = require('ava')

const { runFixtureServer, initBrowserless } = require('./helpers')
const { runFixtureServer } = require('./helpers')
const getHTML = require('..')

const getBrowserless = initBrowserless(test)

;[false, true].forEach(prerender => {
const mode = prerender ? 'prerender' : 'fetch'

test(`${mode} » Shift-JIS`, async t => {
const url = await runFixtureServer(t, '51242_54045.html')
const { html } = await getHTML(url, { prerender, getBrowserless })
const { html } = await getHTML(url, {
prerender,
getBrowserless: () => getBrowserContext(t)
})
t.true(html.includes('或る日の小せん'))
})

test(`${mode} » Windows-1250`, async t => {
const url = await runFixtureServer(t, 'rp.pl.html')
const { html } = await getHTML(url, { prerender, getBrowserless })
const { html } = await getHTML(url, {
prerender,
getBrowserless: () => getBrowserContext(t)
})
t.true(html.includes('majątków'))
})

test(`${mode} » UTF-8`, async t => {
const url = await runFixtureServer(t, 'utf8.with.meta.html')
const { html } = await getHTML(url, { prerender, getBrowserless })
const { html } = await getHTML(url, {
prerender,
getBrowserless: () => getBrowserContext(t)
})
t.true(html.includes('日本語'))
})
})
8 changes: 0 additions & 8 deletions test/helpers.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
'use strict'

const { default: listen } = require('async-listen')
const createBrowserless = require('browserless')
const dateRegex = require('regex-iso-date')
const { createServer } = require('http')
const pretty = require('pretty')
Expand All @@ -18,12 +17,6 @@ const closeServer = server =>
const fixture = name =>
fs.readFileSync(path.join(__dirname, '/fixtures/', name))

const initBrowserless = test => {
const browserlessFactory = createBrowserless()
test.after.always(browserlessFactory.close)
return () => browserlessFactory.createContext()
}

const runServer = async (t, fn) => {
const server = createServer(fn)
const url = await listen(server)
Expand All @@ -42,7 +35,6 @@ const prettyHtml = html =>

module.exports = {
createHeaders,
initBrowserless,
prettyHtml,
runFixtureServer,
runServer
Expand Down
54 changes: 35 additions & 19 deletions test/index.js
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
'use strict'

const { getBrowserContext } = require('@browserless/test')
const PCancelable = require('p-cancelable')
const cheerio = require('cheerio')
const test = require('ava')

const { initBrowserless, runServer, prettyHtml } = require('./helpers')
const { runServer, prettyHtml } = require('./helpers')
const getHTML = require('..')

const getBrowserless = initBrowserless(test)

const wait = async (promise, prop) => {
const res = await promise
return prop ? res[prop] : res
Expand Down Expand Up @@ -37,17 +36,23 @@ test('promise is cancelable', async t => {
test('reachable URL', async t => {
const url = 'https://example.com'
const [prerenderDisabled, prerenderEnabled] = await Promise.all([
getHTML(url, { prerender: false, getBrowserless }),
getHTML(url, {
prerender: false,
getBrowserless: () => getBrowserContext(t)
}),
getHTML(url, {
prerender: true,
getBrowserless,
getBrowserless: () => getBrowserContext(t),
puppeteerOpts: { adblock: false }
})
])

t.is(
await wait(
getHTML(url, { prerender: false, getBrowserless }),
getHTML(url, {
prerender: false,
getBrowserless: () => getBrowserContext(t)
}),
'statusCode'
),
200
Expand All @@ -56,7 +61,7 @@ test('reachable URL', async t => {
await wait(
getHTML(url, {
prerender: true,
getBrowserless,
getBrowserless: () => getBrowserContext(t),
puppeteerOpts: { adblock: false }
}),
'statusCode'
Expand All @@ -82,12 +87,12 @@ test('timeout URL', async t => {
const [prerenderDisabled, prerenderEnabled] = await Promise.all([
getHTML(url, {
prerender: false,
getBrowserless,
getBrowserless: () => getBrowserContext(t),
gotOpts: { timeout: 1000 }
}),
getHTML(url, {
prerender: true,
getBrowserless,
getBrowserless: () => getBrowserContext(t),
puppeteerOpts: { timeout: 2000, adblock: false }
})
])
Expand All @@ -102,10 +107,13 @@ test('unreachable URL', async t => {
const url = 'https://notexisturl.dev'

const [prerenderDisabled, prerenderEnabled] = await Promise.all([
getHTML(url, { prerender: false, getBrowserless }),
getHTML(url, {
prerender: false,
getBrowserless: () => getBrowserContext(t)
}),
getHTML(url, {
prerender: true,
getBrowserless,
getBrowserless: () => getBrowserContext(t),
puppeteerOpts: { adblock: false }
})
])
Expand All @@ -120,7 +128,7 @@ test('from audio URL', async t => {
const targetUrl =
'https://audiodemos.github.io/vctk_set0/embedadapt_100sample.wav'
const { url, stats, html } = await getHTML(targetUrl, {
getBrowserless,
getBrowserless: () => getBrowserContext(t),
prerender: false
})

Expand All @@ -131,7 +139,9 @@ test('from audio URL', async t => {

test('from image URL', async t => {
const targetUrl = 'https://kikobeats.com/images/avatar.jpg'
const { url, stats, html } = await getHTML(targetUrl, { getBrowserless })
const { url, stats, html } = await getHTML(targetUrl, {
getBrowserless: () => getBrowserContext(t)
})

t.is(stats.mode, 'fetch')
t.is(url, targetUrl)
Expand All @@ -144,15 +154,19 @@ test('from image URL', async t => {

test('from SVG image URL', async t => {
const targetUrl = 'https://cdn.microlink.io/file-examples/sample.svg'
const { stats } = await getHTML(targetUrl, { getBrowserless })
const { stats } = await getHTML(targetUrl, {
getBrowserless: () => getBrowserContext(t)
})
t.true(stats.timing < 3000)
t.is(stats.mode, 'fetch')
})

test('from big image URL', async t => {
const targetUrl =
'https://static.jutarnji.hr/images/live-multimedia/binary/2016/6/17/10/iStock_82744687_XXLARGE.jpg'
const { stats } = await getHTML(targetUrl, { getBrowserless })
const { stats } = await getHTML(targetUrl, {
getBrowserless: () => getBrowserContext(t)
})
t.true(stats.timing < 3000)
t.is(stats.mode, 'fetch')
})
Expand All @@ -162,7 +176,7 @@ test('from URL with no content type', async t => {
res.end('<!doctype html><title>.</title>')
})
const { stats } = await getHTML(targetUrl, {
getBrowserless,
getBrowserless: () => getBrowserContext(t),
prerender: false
})
t.is(stats.mode, 'fetch')
Expand All @@ -171,7 +185,9 @@ test('from URL with no content type', async t => {
test('from image URL that returns HTML markup', async t => {
const targetUrl =
'https://www.europapress.es/chance/gente/%7B%7BrutaFoto%7D%7D%7B%7Bfechor%7D%7D_%7B%7BanchoFoto%7D%7D_%7B%7BaltoFoto%7D%7D%7B%7BversionFoto%7D%7D.jpg'
const { stats } = await getHTML(targetUrl, { getBrowserless })
const { stats } = await getHTML(targetUrl, {
getBrowserless: () => getBrowserContext(t)
})
t.true(stats.timing < 3000)
t.is(stats.mode, 'fetch')
})
Expand All @@ -180,7 +196,7 @@ test('from video URL', async t => {
const targetUrl = 'https://cdn.microlink.io/file-examples/sample.mp4'
const { url, stats, html } = await getHTML(targetUrl, {
prerender: false,
getBrowserless
getBrowserless: () => getBrowserContext(t)
})

t.is(stats.mode, 'fetch')
Expand All @@ -192,7 +208,7 @@ test('from bad SSL URL', async t => {
const targetUrl = 'https://self-signed.badssl.com/'
const { url, stats, html } = await getHTML(targetUrl, {
prerender: false,
getBrowserless,
getBrowserless: () => getBrowserContext(t),
gotOpts: {
https: { rejectUnauthorized: false }
}
Expand Down
17 changes: 10 additions & 7 deletions test/mode.js
Original file line number Diff line number Diff line change
@@ -1,28 +1,31 @@
'use strict'

const { getBrowserContext } = require('@browserless/test')
const test = require('ava')

const getHTML = require('../src')
const { initBrowserless } = require('./helpers')

const getBrowserless = initBrowserless(test)

test('`{ prerender: true }`', async t => {
const url = 'https://example.com'
const { stats } = await getHTML(url, { getBrowserless })
const { stats } = await getHTML(url, {
getBrowserless: () => getBrowserContext(t)
})
t.is(stats.mode, 'prerender')
})

test('`{ prerender: false }`', async t => {
const url = 'https://example.com'
const { stats } = await getHTML(url, { prerender: false, getBrowserless })
const { stats } = await getHTML(url, {
prerender: false,
getBrowserless: () => getBrowserContext(t)
})
t.is(stats.mode, 'fetch')
})

test("`{ prerender: 'auto' }`", async t => {
const url = 'https://google.com'
const { stats } = await getHTML(url, {
getBrowserless,
getBrowserless: () => getBrowserContext(t),
puppeteerOpts: { adblock: false }
})
t.is(stats.mode, 'fetch')
Expand All @@ -33,7 +36,7 @@ test.skip('prerender error fallback into fetch mode', async t => {
'https://www.sportsnet.ca/hockey/nhl/leafs-john-tavares-return-new-york-hope-positive/'
const { stats, html } = await getHTML(url, {
prerender: true,
getBrowserless,
getBrowserless: () => getBrowserContext(t),
puppeteerOpts: { adblock: false }
})
t.true(!!html)
Expand Down
11 changes: 5 additions & 6 deletions test/pdf.js
Original file line number Diff line number Diff line change
@@ -1,21 +1,20 @@
'use strict'

const { getBrowserContext } = require('@browserless/test')
const cheerio = require('cheerio')
const test = require('ava')

const { initBrowserless, prettyHtml } = require('./helpers')
const { prettyHtml } = require('./helpers')
const getHTML = require('..')

const getBrowserless = initBrowserless(test)

const PDF_OVER_TRESHOLD = 'https://cdn.microlink.io/file-examples/sample.pdf'
const PDF_UNDER_TRESHOLD = 'https://pdfobject.com/pdf/sample.pdf'

test('disable if `mutool` is not installed', async t => {
const targetUrl = 'https://cdn.microlink.io/file-examples/sample.pdf'
const { url, stats, html } = await getHTML(targetUrl, {
mutool: false,
getBrowserless
getBrowserless: () => getBrowserContext(t)
})

const $ = cheerio.load(html)
Expand All @@ -29,7 +28,7 @@ test('disable if `mutool` is not installed', async t => {
test('turn PDF into HTML markup over the treshold', async t => {
const targetUrl = PDF_OVER_TRESHOLD
const { url, stats, html } = await getHTML(targetUrl, {
getBrowserless
getBrowserless: () => getBrowserContext(t)
})

const $ = cheerio.load(html)
Expand All @@ -44,7 +43,7 @@ test('turn PDF into HTML markup over the treshold', async t => {
test('turn PDF into HTML markup under the treshold', async t => {
const targetUrl = PDF_UNDER_TRESHOLD
const { url, stats, html } = await getHTML(targetUrl, {
getBrowserless
getBrowserless: () => getBrowserContext(t)
})
const $ = cheerio.load(html)
t.is(url, targetUrl)
Expand Down
6 changes: 2 additions & 4 deletions test/redirects.js
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
'use strict'

const { getBrowserContext } = require('@browserless/test')
const test = require('ava')

const { initBrowserless } = require('./helpers')
const getHTML = require('..')

const getBrowserless = initBrowserless(test)

;[true, false].forEach(prerender => {
const mode = prerender ? 'prerender' : 'fetch'

Expand All @@ -16,7 +14,7 @@ const getBrowserless = initBrowserless(test)

const { redirects } = await getHTML(targetUrl, {
prerender,
getBrowserless
getBrowserless: () => getBrowserContext(t)
})

t.deepEqual(redirects, [
Expand Down
9 changes: 4 additions & 5 deletions test/url.js
Original file line number Diff line number Diff line change
@@ -1,20 +1,19 @@
'use strict'

const { getBrowserContext } = require('@browserless/test')
const test = require('ava')

const { initBrowserless, runServer, prettyHtml } = require('./helpers')
const { runServer, prettyHtml } = require('./helpers')
const getHTML = require('..')

const getBrowserless = initBrowserless(test)

;[false, true].forEach(prerender => {
const mode = prerender ? 'prerender' : 'fetch'
test(`${mode} » as string`, async t => {
const url = await runServer(t, (_, res) =>
res.end('<!doctype html><title>.</title>')
)
const { html } = await getHTML(url.toString(), {
getBrowserless,
getBrowserless: () => getBrowserContext(t),
prerender,
puppeteerOpts: { adblock: false, animations: true }
})
Expand All @@ -38,7 +37,7 @@ const getBrowserless = initBrowserless(test)
res.end('<!doctype html><title>.</title>')
)
const { html } = await getHTML(url, {
getBrowserless,
getBrowserless: () => getBrowserContext(t),
prerender,
puppeteerOpts: { adblock: false, animations: true }
})
Expand Down