'), null)
+ })
+
+ it('extract title only', async () => {
+ const html = readFileSync('./test-data/regular-article.html', 'utf8')
+ const result = extractTitleWithReadability(html)
+ assert.equal(result, 'Article title here - ArticleParser')
+ })
+
+ it('extract title from page without title', async () => {
+ const html = readFileSync('./test-data/html-no-title.html', 'utf8')
+ const result = extractTitleWithReadability(html)
+ assert.equal(result, null)
+ })
-test('test extractTitleWithReadability', async () => {
- const html = readFileSync('./test-data/regular-article.html', 'utf8')
- const result = extractTitleWithReadability(html)
- expect(result).toBe('Article title here - ArticleParser')
+ it('extract title from non-string', async () => {
+ const result = extractTitleWithReadability({})
+ assert.equal(result, null)
+ })
})
diff --git a/src/utils/extractWithSelector.test.js b/src/utils/extractWithSelector.test.js
deleted file mode 100644
index da46051b..00000000
--- a/src/utils/extractWithSelector.test.js
+++ /dev/null
@@ -1,20 +0,0 @@
-// extractWithSelector.test
-/* eslint-env jest */
-
-import { readFileSync } from 'fs'
-
-import { isString } from 'bellajs'
-
-import extractWithSelector from './extractWithSelector.js'
-
-test('test extractWithSelector a bad input', () => {
- const result = extractWithSelector(null)
- expect(result).toBe(null)
-})
-
-test('test extractWithSelector from good html content', async () => {
- const html = readFileSync('./test-data/regular-article.html', 'utf8')
- const result = extractWithSelector(html, 'article', ['.ads-section'])
- expect(isString(result)).toBe(true)
- expect(result.length > 200).toBe(true)
-})
diff --git a/src/utils/findDate.js b/src/utils/findDate.js
new file mode 100644
index 00000000..3a666e02
--- /dev/null
+++ b/src/utils/findDate.js
@@ -0,0 +1,57 @@
+
+/**
+ * Convert date format to YYYY-MM-DD
+ *
+ * @param {string} dateString
+ * @returns {string} YYYY-MM-DD
+ */
+function convertDateFormat (dateString) {
+ const parts = dateString.split('/')
+ if (parts.length !== 3) return dateString
+
+ let year, month, day
+
+ if (parseInt(parts[0]) > 12) {
+ [day, month, year] = parts
+ } else {
+ [month, day, year] = parts
+ }
+
+ year = year.length === 2 ? '20' + year : year
+ return `${year}-${month.padStart(2, '0')}-${day.padStart(2, '0')}T00:00:00`
+}
+
+/**
+ * Look for the publication date in the body of the content.
+ *
+ * @param {Document} document - The HTML Document
+ * @returns {string} The date string
+ */
+export default function (doc) {
+ const datePatterns = [
+ /\d{4}-\d{2}-\d{2}/,
+ /\d{1,2}\/\d{1,2}\/\d{2,4}/,
+ ]
+
+ const findDate = (element) => {
+ for (const pattern of datePatterns) {
+ const match = element.textContent.match(pattern)
+ if (match) return convertDateFormat(match[0])
+ }
+ return ''
+ }
+
+ const priorityElements = doc.querySelectorAll('time, [datetime], [itemprop~=datePublished], [itemprop~=dateCreated]')
+ for (const el of priorityElements) {
+ const date = el.getAttribute('datetime') || el.getAttribute('content') || findDate(el)
+ if (date) return date
+ }
+
+ const secondaryElements = doc.querySelectorAll('p, span, div')
+ for (const el of secondaryElements) {
+ const date = findDate(el)
+ if (date) return date
+ }
+
+ return ''
+}
diff --git a/src/utils/findRulesByUrl.test.js b/src/utils/findRulesByUrl.test.js
deleted file mode 100644
index 4c5ccb66..00000000
--- a/src/utils/findRulesByUrl.test.js
+++ /dev/null
@@ -1,45 +0,0 @@
-// findRulesByUrl.test
-/* eslint-env jest */
-
-import { isFunction } from 'bellajs'
-
-import findRulesByUrl from './findRulesByUrl.js'
-
-describe('test findRulesByUrl()', () => {
- const entries = [
- {
- urls: [{}, ''],
- expectation: {}
- },
- {
- urls: [1209, 'https://vietnamnet.vn/path/to/article'],
- expectation: (result, expect) => {
- expect(result).toBeTruthy()
- expect(result).toEqual(expect.objectContaining({ selector: '#ArticleContent' }))
- expect(result.selector).toEqual('#ArticleContent')
- }
- },
- {
- urls: ['https://vnn.vn/path/to/article'],
- expectation: (result, expect) => {
- expect(result).toBeTruthy()
- expect(result).toEqual(expect.objectContaining({ selector: '#ArticleContent' }))
- expect(result.selector).toEqual('#ArticleContent')
- }
- }
- ]
- entries.forEach((entry) => {
- const {
- urls,
- expectation
- } = entry
- test('check if findRulesByUrl() works correctly', () => {
- const result = findRulesByUrl(urls)
- if (isFunction(expectation)) {
- expectation(result, expect)
- } else {
- expect(result).toEqual(expectation)
- }
- })
- })
-})
diff --git a/src/utils/getHostname.test.js b/src/utils/getHostname.test.js
deleted file mode 100644
index fb58d47a..00000000
--- a/src/utils/getHostname.test.js
+++ /dev/null
@@ -1,39 +0,0 @@
-// getHostname.test
-/* eslint-env jest */
-
-import getHostname from './getHostname.js'
-
-describe('test getHostname()', () => {
- const entries = [
- {
- url: '',
- expected: ''
- },
- {
- url: {},
- expected: ''
- },
- {
- url: 'https://www.some.where/article/abc-xyz',
- expected: 'some.where'
- },
- {
- url: 'https://www.alpha.some.where/blog/authors/article/abc-xyz',
- expected: 'alpha.some.where'
- },
- {
- url: 'https://10.1.1.5:1888/article/abc-xyz',
- expected: '10.1.1.5'
- }
- ]
- entries.forEach((entry) => {
- const {
- url,
- expected
- } = entry
- test(`absolutifyUrl("${url}") must become "${expected}"`, () => {
- const result = getHostname(url)
- expect(result).toEqual(expected)
- })
- })
-})
diff --git a/src/utils/getTimeToRead.js b/src/utils/getTimeToRead.js
index ccd448cd..7d8cef37 100644
--- a/src/utils/getTimeToRead.js
+++ b/src/utils/getTimeToRead.js
@@ -1,12 +1,7 @@
// utils -> getTimeToRead
-import {
- getParserOptions
-} from '../config.js'
-
-export default (text) => {
+export default (text, wordsPerMinute) => {
const words = text.trim().split(/\s+/g).length
- const { wordsPerMinute } = getParserOptions()
const minToRead = words / wordsPerMinute
const secToRead = Math.ceil(minToRead * 60)
return secToRead
diff --git a/src/utils/html.js b/src/utils/html.js
new file mode 100644
index 00000000..25b23ba4
--- /dev/null
+++ b/src/utils/html.js
@@ -0,0 +1,50 @@
+// utils -> html
+
+import { DOMParser } from 'linkedom'
+import sanitize from 'sanitize-html'
+import { pipe } from '@ndaidong/bellajs'
+
+import { getSanitizeHtmlOptions } from '../config.js'
+
+export const purify = (html) => {
+ return sanitize(html, {
+ allowedTags: false,
+ allowedAttributes: false,
+ allowVulnerableTags: true,
+ })
+}
+
+const WS_REGEXP = /^[\s\f\n\r\t\u1680\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff\x09\x0a\x0b\x0c\x0d\x20\xa0]+$/ // eslint-disable-line
+
+const stripMultiLinebreaks = (str) => {
+ return str.replace(/(\r\n|\n|\u2424){2,}/g, '\n').split('\n').map((line) => {
+ return WS_REGEXP.test(line) ? line.trim() : line
+ }).filter((line) => {
+ return line.length > 0
+ }).join('\n')
+}
+
+const stripMultispaces = (str) => {
+ return str.replace(WS_REGEXP, ' ').trim()
+}
+
+export const getCharset = (html) => {
+ const doc = new DOMParser().parseFromString(html, 'text/html')
+ const m = doc.querySelector('meta[charset]') || null
+ let charset = m ? m.getAttribute('charset') : ''
+ if (!charset) {
+ const h = doc.querySelector('meta[http-equiv="content-type"]') || null
+ charset = h ? h.getAttribute('content')?.split(';')[1]?.replace('charset=', '')?.trim() : ''
+ }
+ return charset?.toLowerCase() || 'utf8'
+}
+
+export const cleanify = (inputHtml) => {
+ const doc = new DOMParser().parseFromString(inputHtml, 'text/html')
+ const html = doc.documentElement.innerHTML
+ return pipe(
+ input => sanitize(input, getSanitizeHtmlOptions()),
+ input => stripMultiLinebreaks(input),
+ input => stripMultispaces(input)
+ )(html)
+}
diff --git a/src/utils/html.test.js b/src/utils/html.test.js
new file mode 100644
index 00000000..00fc2263
--- /dev/null
+++ b/src/utils/html.test.js
@@ -0,0 +1,23 @@
+// html.test
+import { describe, it } from 'node:test'
+import assert from 'node:assert'
+
+import { readFileSync } from 'node:fs'
+
+import { isString } from '@ndaidong/bellajs'
+
+import {
+ cleanify
+} from './html.js'
+
+describe('test cleanify() method', () => {
+ it('check if unwanted elements/attributes removed', () => {
+ const html = readFileSync('./test-data/regular-article.html', 'utf8')
+ assert.ok(html.includes('
'))
+ const result = cleanify(html)
+ assert.ok(isString(result))
+ assert.equal(result.includes('
'), false)
+ })
+})
diff --git a/src/utils/isHTMLString.test.js b/src/utils/isHTMLString.test.js
deleted file mode 100644
index 74a74838..00000000
--- a/src/utils/isHTMLString.test.js
+++ /dev/null
@@ -1,42 +0,0 @@
-// isHTMLString.test
-/* eslint-env jest */
-
-import {
- readFileSync
-} from 'fs'
-
-import isHTMLString from './isHTMLString.js'
-
-test('test isHTMLString(bad input)', () => {
- const result = isHTMLString({})
- expect(result).toBe(false)
-})
-
-test('test isHTMLString(regular string)', () => {
- const result = isHTMLString('This is just a string, not HTML')
- expect(result).toBe(false)
-})
-
-test('test isHTMLString(bad-format HTML)', () => {
- const result = isHTMLString('
Hello world')
- expect(result).toBe(false)
-})
-
-test('test isHTMLString(well-format HTML)', () => {
- const result = isHTMLString('
Hello world
')
- expect(result).toBe(true)
-})
-
-test('test isHTMLString(example HTML page)', () => {
- const files = [
- 'regular-article.html',
- 'html-no-title.html',
- 'html-article-no-source.html',
- 'html-too-short-article.html'
- ]
- files.forEach((file) => {
- const html = readFileSync(`./test-data/${file}`, 'utf8')
- const result = isHTMLString(html)
- expect(result).toBe(true)
- })
-})
diff --git a/src/utils/isValidUrl.js b/src/utils/isValidUrl.js
index 5e22a45e..d1e642c2 100644
--- a/src/utils/isValidUrl.js
+++ b/src/utils/isValidUrl.js
@@ -4,7 +4,7 @@ export default (url = '') => {
try {
const ourl = new URL(url)
return ourl !== null && ourl.protocol.startsWith('http')
- } catch (err) {
+ } catch {
return false
}
}
diff --git a/src/utils/isValidUrl.test.js b/src/utils/isValidUrl.test.js
deleted file mode 100644
index 67e30b5e..00000000
--- a/src/utils/isValidUrl.test.js
+++ /dev/null
@@ -1,47 +0,0 @@
-// isValidUrl.test
-/* eslint-env jest */
-
-import isValidUrl from './isValidUrl.js'
-
-describe('test isValidUrl()', () => {
- const cases = [
- {
- url: 'https://www.23hq.com',
- expected: true
- },
- {
- url: 'https://secure.actblue.com',
- expected: true
- },
- {
- url: 'https://docs.microsoft.com/en-us/azure/iot-edge/quickstart?view=iotedge-2018-06',
- expected: true
- },
- {
- url: 'http://192.168.1.199:8081/example/page',
- expected: true
- },
- {
- url: 'ftp://192.168.1.199:8081/example/page',
- expected: false
- },
- {
- url: '',
- expected: false
- },
- {
- url: null,
- expected: false
- },
- {
- url: { a: 'x' },
- expected: false
- }
- ]
- cases.forEach(({ url, expected }) => {
- test(`isValidUrl("${url}") must return "${expected}"`, () => {
- const result = isValidUrl(url)
- expect(result).toEqual(expected)
- })
- })
-})
diff --git a/src/utils/linker.js b/src/utils/linker.js
new file mode 100644
index 00000000..3c1a70f0
--- /dev/null
+++ b/src/utils/linker.js
@@ -0,0 +1,133 @@
+// utils -> linker
+
+import { DOMParser } from 'linkedom'
+
+import { findBestMatch } from './similarity.js'
+
+export const isValid = (url = '') => {
+ try {
+ const ourl = new URL(url)
+ return ourl !== null && ourl.protocol.startsWith('http')
+ } catch {
+ return false
+ }
+}
+
+export const chooseBestUrl = (candidates = [], title = '') => {
+ const ranking = findBestMatch(title, candidates)
+ return ranking.bestMatch.target
+}
+
+export const absolutify = (fullUrl = '', relativeUrl = '') => {
+ try {
+ const result = new URL(relativeUrl, fullUrl)
+ return result.toString()
+ } catch {
+ return ''
+ }
+}
+
+const blacklistKeys = [
+ 'CNDID',
+ '__twitter_impression',
+ '_hsenc',
+ '_openstat',
+ 'action_object_map',
+ 'action_ref_map',
+ 'action_type_map',
+ 'amp',
+ 'fb_action_ids',
+ 'fb_action_types',
+ 'fb_ref',
+ 'fb_source',
+ 'fbclid',
+ 'ga_campaign',
+ 'ga_content',
+ 'ga_medium',
+ 'ga_place',
+ 'ga_source',
+ 'ga_term',
+ 'gs_l',
+ 'hmb_campaign',
+ 'hmb_medium',
+ 'hmb_source',
+ 'mbid',
+ 'mc_cid',
+ 'mc_eid',
+ 'mkt_tok',
+ 'referrer',
+ 'spJobID',
+ 'spMailingID',
+ 'spReportId',
+ 'spUserID',
+ 'utm_brand',
+ 'utm_campaign',
+ 'utm_cid',
+ 'utm_content',
+ 'utm_int',
+ 'utm_mailing',
+ 'utm_medium',
+ 'utm_name',
+ 'utm_place',
+ 'utm_pubreferrer',
+ 'utm_reader',
+ 'utm_social',
+ 'utm_source',
+ 'utm_swu',
+ 'utm_term',
+ 'utm_userid',
+ 'utm_viz_id',
+ 'wt_mc_o',
+ 'yclid',
+ 'WT.mc_id',
+ 'WT.mc_ev',
+ 'WT.srch',
+ 'pk_source',
+ 'pk_medium',
+ 'pk_campaign',
+]
+
+export const purify = (url) => {
+ try {
+ const pureUrl = new URL(url)
+
+ blacklistKeys.forEach((key) => {
+ pureUrl.searchParams.delete(key)
+ })
+
+ return pureUrl.toString().replace(pureUrl.hash, '')
+ } catch {
+ return null
+ }
+}
+
+/**
+ * @param inputHtml {string}
+ * @param url {string}
+ * @returns article {string}
+ */
+export const normalize = (html, url) => {
+ const doc = new DOMParser().parseFromString(html, 'text/html')
+
+ Array.from(doc.getElementsByTagName('a')).forEach((element) => {
+ const href = element.getAttribute('href')
+ if (href) {
+ element.setAttribute('href', absolutify(url, href))
+ element.setAttribute('target', '_blank')
+ }
+ })
+
+ Array.from(doc.getElementsByTagName('img')).forEach((element) => {
+ const src = element.getAttribute('data-src') ?? element.getAttribute('src')
+ if (src) {
+ element.setAttribute('src', absolutify(url, src))
+ }
+ })
+
+ return Array.from(doc.childNodes).map(element => element.outerHTML).join('')
+}
+
+export const getDomain = (url) => {
+ const host = (new URL(url)).host
+ return host.replace('www.', '')
+}
diff --git a/src/utils/linker.test.js b/src/utils/linker.test.js
new file mode 100644
index 00000000..6ed89d2e
--- /dev/null
+++ b/src/utils/linker.test.js
@@ -0,0 +1,182 @@
+// linker.test
+import { describe, it } from 'node:test'
+import assert from 'node:assert'
+
+import { readFileSync } from 'node:fs'
+
+import { isString } from '@ndaidong/bellajs'
+
+import {
+ chooseBestUrl,
+ isValid as isValidUrl,
+ purify as purifyUrl,
+ normalize as normalizeUrls,
+ absolutify as absolutifyUrl
+} from './linker.js'
+
+describe('test isValidUrl()', () => {
+ const cases = [
+ {
+ url: 'https://www.23hq.com',
+ expected: true,
+ },
+ {
+ url: 'https://secure.actblue.com',
+ expected: true,
+ },
+ {
+ url: 'https://docs.microsoft.com/en-us/azure/iot-edge/quickstart?view=iotedge-2018-06',
+ expected: true,
+ },
+ {
+ url: 'http://192.168.1.199:8081/example/page',
+ expected: true,
+ },
+ {
+ url: 'ftp://192.168.1.199:8081/example/page',
+ expected: false,
+ },
+ {
+ url: '',
+ expected: false,
+ },
+ {
+ url: null,
+ expected: false,
+ },
+ {
+ url: { a: 'x' },
+ expected: false,
+ },
+ ]
+ cases.forEach(({ url, expected }) => {
+ it(`isValidUrl("${url}") must return "${expected}"`, () => {
+ const result = isValidUrl(url)
+ assert.equal(result, expected)
+ })
+ })
+})
+
+describe('test normalizeUrls()', () => {
+ it('test adding absolute URLs to all links', () => {
+ const bestUrl = 'https://test-url.com/burritos-for-life'
+ const html = readFileSync('./test-data/regular-article.html', 'utf8')
+ const result = normalizeUrls(html, bestUrl)
+ assert.ok(isString(result))
+ assert.equal(result.includes('
watermelon '), false)
+ assert.equal(result.includes('
watermelon '), true)
+ })
+ it('test adding target=_blank to all links', () => {
+ const bestUrl = 'https://test-url.com/burritos-for-life'
+ const html = readFileSync('./test-data/regular-article.html', 'utf8')
+ const result = normalizeUrls(html, bestUrl)
+ assert.ok(isString(result))
+ assert.equal(result.includes('
rational peach '), false)
+ assert.equal(result.includes('
rational peach '), true)
+ })
+})
+
+describe('test purifyUrl()', () => {
+ const entries = [
+ {
+ url: '',
+ expected: null,
+ },
+ {
+ url: {},
+ expected: null,
+ },
+ {
+ url: 'https://some.where/article/abc-xyz',
+ expected: 'https://some.where/article/abc-xyz',
+ },
+ {
+ url: 'https://some.where/article/abc-xyz#name,bob',
+ expected: 'https://some.where/article/abc-xyz',
+ },
+ {
+ url: 'https://some.where/article/abc-xyz?utm_source=news4&utm_medium=email&utm_campaign=spring-summer',
+ expected: 'https://some.where/article/abc-xyz',
+ },
+ {
+ url: 'https://some.where/article/abc-xyz?q=3&utm_source=news4&utm_medium=email&utm_campaign=spring-summer',
+ expected: 'https://some.where/article/abc-xyz?q=3',
+ },
+ {
+ url: 'https://some.where/article/abc-xyz?pk_source=news4&pk_medium=email&pk_campaign=spring-summer',
+ expected: 'https://some.where/article/abc-xyz',
+ },
+ {
+ url: 'https://some.where/article/abc-xyz?q=3&pk_source=news4&pk_medium=email&pk_campaign=spring-summer',
+ expected: 'https://some.where/article/abc-xyz?q=3',
+ },
+ ]
+ entries.forEach((entry) => {
+ const {
+ url,
+ expected,
+ } = entry
+ it(`purifyUrl("${url}") must become "${expected}"`, () => {
+ const result = purifyUrl(url)
+ assert.equal(result, expected)
+ })
+ })
+})
+
+describe('test absolutifyUrl()', () => {
+ const entries = [
+ {
+ full: '',
+ expected: '',
+ },
+ {
+ relative: {},
+ expected: '',
+ },
+ {
+ full: 'https://some.where/article/abc-xyz',
+ relative: 'category/page.html',
+ expected: 'https://some.where/article/category/page.html',
+ },
+ {
+ full: 'https://some.where/article/abc-xyz',
+ relative: '../category/page.html',
+ expected: 'https://some.where/category/page.html',
+ },
+ {
+ full: 'https://some.where/blog/authors/article/abc-xyz',
+ relative: '/category/page.html',
+ expected: 'https://some.where/category/page.html',
+ },
+ {
+ full: 'https://some.where/article/abc-xyz',
+ expected: 'https://some.where/article/abc-xyz',
+ },
+ ]
+ entries.forEach((entry) => {
+ const {
+ full,
+ relative,
+ expected,
+ } = entry
+ it(`absolutifyUrl("${full}", "${relative}") must become "${expected}"`, () => {
+ const result = absolutifyUrl(full, relative)
+ assert.equal(result, expected)
+ })
+ })
+})
+
+describe('test chooseBestUrl()', () => {
+ it('test chooseBestUrl an actual case', () => {
+ const title = 'Google đã ra giá mua Fitbit'
+ const urls = [
+ 'https://alpha.xyz/tin-tuc-kinh-doanh/-/view_content/content/2965950/google-da-ra-gia-mua-fitbit',
+ 'https://alpha.xyz/tin-tuc-kinh-doanh/view/2965950/907893219797',
+ 'https://alpha.xyz/tin-tuc-kinh-doanh/google-da-ra-gia-mua-fitbit',
+ 'https://a.xyz/read/google-da-ra-gia-mua-fitbit',
+ 'https://a.xyz/read/2965950/907893219797',
+ ]
+ const result = chooseBestUrl(urls, title)
+ assert.equal(result, urls[3])
+ })
+})
diff --git a/src/utils/logger.js b/src/utils/logger.js
deleted file mode 100644
index d87f5b68..00000000
--- a/src/utils/logger.js
+++ /dev/null
@@ -1,15 +0,0 @@
-// utils / logger
-
-import debug from 'debug'
-
-const name = 'article-parser'
-
-export const info = debug(`${name}:info`)
-export const error = debug(`${name}:error`)
-export const warning = debug(`${name}:warning`)
-
-export default {
- info: debug(`${name}:info`),
- error: debug(`${name}:error`),
- warning: debug(`${name}:warning`)
-}
diff --git a/src/utils/normalizeUrls.test.js b/src/utils/normalizeUrls.test.js
deleted file mode 100644
index c587b4e8..00000000
--- a/src/utils/normalizeUrls.test.js
+++ /dev/null
@@ -1,41 +0,0 @@
-// normalizeUrls.test
-/* eslint-env jest */
-
-import { readFileSync } from 'fs'
-
-import { isString } from 'bellajs'
-
-import normalizeUrls from './normalizeUrls.js'
-
-describe('test normalizeUrls()', () => {
- test('test adding absolute URLs to all links', () => {
- const bestUrl = 'https://test-url.com/burritos-for-life'
- const html = readFileSync('./test-data/regular-article.html', 'utf8')
- const result = normalizeUrls(html, bestUrl)
- expect(isString(result)).toBe(true)
- expect(result).toEqual(
- expect.not.stringContaining('
watermelon ')
- )
- expect(result).toEqual(
- expect.stringContaining(
- '
watermelon '
- )
- )
- })
- test('test adding target=_blank to all links', () => {
- const bestUrl = 'https://test-url.com/burritos-for-life'
- const html = readFileSync('./test-data/regular-article.html', 'utf8')
- const result = normalizeUrls(html, bestUrl)
- expect(isString(result)).toBe(true)
- expect(result).toEqual(
- expect.not.stringContaining(
- '
rational peach '
- )
- )
- expect(result).toEqual(
- expect.stringContaining(
- '
rational peach '
- )
- )
- })
-})
diff --git a/src/utils/parseFromHtml.js b/src/utils/parseFromHtml.js
index 9f153b82..406d0777 100644
--- a/src/utils/parseFromHtml.js
+++ b/src/utils/parseFromHtml.js
@@ -1,48 +1,38 @@
// utils -> parseFromHtml
-import { stripTags, truncate, unique } from 'bellajs'
+import { stripTags, truncate, unique, pipe } from '@ndaidong/bellajs'
-import sanitize from 'sanitize-html'
+import { purify, cleanify } from './html.js'
-import isValidUrl from './isValidUrl.js'
-import purifyUrl from './purifyUrl.js'
-import absolutifyUrl from './absolutifyUrl.js'
-import chooseBestUrl from './chooseBestUrl.js'
-import getHostname from './getHostname.js'
+import {
+ isValid as isValidUrl,
+ purify as purifyUrl,
+ absolutify as absolutifyUrl,
+ normalize as normalizeUrls,
+ chooseBestUrl,
+ getDomain
+} from './linker.js'
-import findRulesByUrl from './findRulesByUrl.js'
-import cleanAndMinifyHtml from './cleanAndMinifyHtml.js'
import extractMetaData from './extractMetaData.js'
-import extractJsonLd from './extractJsonLd.js'
+
import extractWithReadability, {
extractTitleWithReadability
} from './extractWithReadability.js'
-import extractWithSelector from './extractWithSelector.js'
-import getTimeToRead from './getTimeToRead.js'
-import normalizeUrls from './normalizeUrls.js'
-import stripUnwantedTags from './stripUnwantedTags.js'
-import transformHtml from './transformHtml.js'
-import logger from './logger.js'
+import { execPreParser, execPostParser } from './transformation.js'
-import { getParserOptions } from '../config.js'
+import getTimeToRead from './getTimeToRead.js'
-const cleanify = html => {
- return sanitize(html, {
- allowedTags: false,
- allowedAttributes: false
- })
+const summarize = (desc, txt, threshold, maxlen) => { // eslint-disable-line
+ return desc.length > threshold
+ ? desc
+ : truncate(txt, maxlen).replace(/\n/g, ' ')
}
-const summarize = (desc, txt, threshold, maxlen) => {
- return desc.length < threshold
- ? truncate(txt, maxlen).replace(/\n/g, ' ')
- : desc
-}
+export default async (inputHtml, inputUrl = '', parserOptions = {}) => {
+ const pureHtml = purify(inputHtml)
+ const meta = extractMetaData(pureHtml)
-export default async (inputHtml, inputUrl = '') => {
- const html = cleanify(inputHtml)
- const meta = extractMetaData(html)
let title = meta.title
const {
@@ -53,23 +43,23 @@ export default async (inputHtml, inputUrl = '') => {
description: metaDesc,
image: metaImg,
author,
- source,
- published
+ published,
+ favicon: metaFav,
+ type,
} = meta
const {
- descriptionLengthThreshold,
- descriptionTruncateLen,
- contentLengthThreshold
- } = getParserOptions()
+ wordsPerMinute = 300,
+ descriptionTruncateLen = 210,
+ descriptionLengthThreshold = 180,
+ contentLengthThreshold = 200,
+ } = parserOptions
// gather title
if (!title) {
- logger.info('Could not detect article title from meta!')
- title = extractTitleWithReadability(html, inputUrl)
+ title = extractTitleWithReadability(pureHtml, inputUrl)
}
if (!title) {
- logger.info('Could not detect article title!')
return null
}
@@ -81,41 +71,38 @@ export default async (inputHtml, inputUrl = '') => {
)
if (!links.length) {
- logger.info('Could not detect article link!')
return null
}
- // choose the best url
+ // choose the best url, which one looks like title the most
const bestUrl = chooseBestUrl(links, title)
- // get defined selector
- const {
- selector = null,
- unwanted = [],
- transform = null
- } = findRulesByUrl(links)
-
- // find article content
- const mainContentSelected = extractWithSelector(html, selector)
-
- const mainContent = stripUnwantedTags(mainContentSelected ?? html, unwanted)
-
- const mainContentAbsoluteUrls = normalizeUrls(mainContent, bestUrl)
-
- const transformedContent = transformHtml(mainContentAbsoluteUrls, transform)
+ const fns = pipe(
+ (input) => {
+ return normalizeUrls(input, bestUrl)
+ },
+ (input) => {
+ return execPreParser(input, links)
+ },
+ (input) => {
+ return extractWithReadability(input, bestUrl)
+ },
+ (input) => {
+ return input ? execPostParser(input, links) : null
+ },
+ (input) => {
+ return input ? cleanify(input) : null
+ }
+ )
- const content = extractWithReadability(transformedContent, bestUrl)
+ const content = fns(inputHtml)
if (!content) {
- logger.info('Could not detect article content!')
return null
}
- const normalizedContent = cleanAndMinifyHtml(content)
-
- const textContent = stripTags(normalizedContent)
+ const textContent = stripTags(content)
if (textContent.length < contentLengthThreshold) {
- logger.info('Main article is too short!')
return null
}
@@ -127,22 +114,7 @@ export default async (inputHtml, inputUrl = '') => {
)
const image = metaImg ? absolutifyUrl(bestUrl, metaImg) : ''
-
- let { publisher, ...jsonLData } = extractJsonLd(html, inputUrl)
- let authors = jsonLData.author
- if (!authors.length && author) {
- const trimStr = (str) => str.trimStart().trimEnd()
- authors = author.split(/,|and|& /).filter((a) => !!trimStr(a)).map((a) => ({ name: trimStr(a) }))
- }
-
- const hostName = getHostname(bestUrl)
-
- if (!publisher) {
- publisher = {
- name: source || hostName,
- url: hostName
- }
- }
+ const favicon = metaFav ? absolutifyUrl(bestUrl, metaFav) : ''
return {
url: bestUrl,
@@ -150,11 +122,12 @@ export default async (inputHtml, inputUrl = '') => {
description,
links,
image,
- content: normalizedContent,
- author: authors,
- source: source || hostName,
- publisher,
+ content,
+ author,
+ favicon,
+ source: getDomain(bestUrl),
published,
- ttr: getTimeToRead(textContent)
+ ttr: getTimeToRead(textContent, wordsPerMinute),
+ type,
}
}
diff --git a/src/utils/parseFromHtml.test.js b/src/utils/parseFromHtml.test.js
index c6eac666..e9965038 100644
--- a/src/utils/parseFromHtml.test.js
+++ b/src/utils/parseFromHtml.test.js
@@ -1,127 +1,133 @@
// parseFromHtml.test
-/* eslint-env jest */
+import { describe, it } from 'node:test'
+import assert from 'node:assert'
-import { readFileSync } from 'fs'
+import { readFileSync } from 'node:fs'
-import { isFunction } from 'bellajs'
+import { isFunction } from '@ndaidong/bellajs'
-import {
- addQueryRules
-} from '../config.js'
+import { extractFromHtml as parseFromHtml } from '../main.js'
+import { addTransformations } from './transformation.js'
-import parseFromHtml from './parseFromHtml.js'
+const expDesc = [
+ 'Navigation here Few can name a rational peach that isn\'t a conscientious goldfish!',
+ 'One cannot separate snakes from plucky pomegranates?',
+ 'Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs.',
+].join(' ')
-describe('test parseFromHtml()', () => {
- const cases = [
- {
- input: {
- desc: 'a bad input',
- html: {}
- },
- expectation: null
+const cases = [
+ {
+ input: {
+ desc: 'a webpage with no title',
+ html: readFileSync('./test-data/html-no-title.html', 'utf8'),
},
- {
- input: {
- desc: 'a webpage with no title',
- html: readFileSync('./test-data/html-no-title.html', 'utf8')
- },
- expectation: null
+ expectation: null,
+ },
+ {
+ input: {
+ desc: 'a webpage without link',
+ html: readFileSync('./test-data/html-no-link.html', 'utf8'),
},
- {
- input: {
- desc: 'a webpage with no main article',
- html: readFileSync('./test-data/html-no-article.html', 'utf8')
- },
- expectation: null
+ expectation: null,
+ },
+ {
+ input: {
+ desc: 'a webpage with no main article',
+ html: readFileSync('./test-data/html-no-article.html', 'utf8'),
},
- {
- input: {
- desc: 'a webpage with a very short article',
- html: readFileSync('./test-data/html-too-short-article.html', 'utf8'),
- url: 'abcd'
- },
- expectation: null
+ expectation: null,
+ },
+ {
+ input: {
+ desc: 'a webpage with a very short article',
+ html: readFileSync('./test-data/html-too-short-article.html', 'utf8'),
+ url: 'abcd',
},
- {
- input: {
- desc: 'a webpage with article but no source',
- html: readFileSync('./test-data/html-article-no-source.html', 'utf8')
- },
- expectation: (result, expect) => {
- expect(result.source).toEqual('somewhere.any')
- }
+ expectation: null,
+ },
+ {
+ input: {
+ desc: 'a webpage with article but no source',
+ html: readFileSync('./test-data/html-article-no-source.html', 'utf8'),
},
- {
- input: {
- desc: 'a webpage with data-src in img tag',
- html: readFileSync('./test-data/html-article-with-data-src.html', 'utf8')
- },
- expectation: (result, expect) => {
- expect(result.content).toEqual(expect.stringContaining('
'))
- expect(result.content).toEqual(expect.stringContaining('
'))
- }
+ expectation: (result) => {
+ assert.equal(result.source, 'somewhere.any')
},
- {
- input: {
- desc: 'a webpage with regular article',
- html: readFileSync('./test-data/regular-article.html', 'utf8'),
- url: 'https://somewhere.com/path/to/article'
- },
- expectation: (result, expect) => {
- expect(result.title).toEqual('Article title here')
- expect(result.description).toEqual('Few words to summarize this article content')
- expect(result.content).toEqual(expect.stringContaining('
'))
- expect(result.content).toEqual(expect.stringContaining(' '))
- }
+ },
+ {
+ input: {
+ desc: 'a webpage with data-src in img tag',
+ html: readFileSync('./test-data/html-article-with-data-src.html', 'utf8'),
},
- {
- input: {
- desc: 'a webpage with unwanted elements',
- html: readFileSync('./test-data/vnn-article.html', 'utf8'),
- url: 'https://vnn.vn/path/to/article'
- },
- expectation: (result, expect) => {
- expect(result.title).toEqual('Article title here')
- expect(result.description).toEqual('Few words to summarize this article content')
- expect(result.content).toEqual(expect.stringContaining(' '))
- expect(result.content).toEqual(expect.stringContaining(' '))
- expect(result.content).toEqual(expect.not.stringContaining('Related articles'))
- }
- }
- ]
+ expectation: (result) => {
+ assert.equal(result.content.includes(' '), true)
+ assert.equal(result.content.includes(' '), true)
+ },
+ },
+]
+describe('test parseFromHtml', () => {
cases.forEach((acase) => {
const { input, expectation } = acase
const { desc, html, url = '' } = input
- test(`check if parseFromHtml() works with ${desc}`, async () => {
+ it(`check if parseFromHtml() works with ${desc}`, async () => {
const result = await parseFromHtml(html, url)
if (isFunction(expectation)) {
- expectation(result, expect)
+ expectation(result)
} else {
- expect(result).toEqual(expectation)
+ assert.equal(result, expectation)
}
})
})
-})
-test('check if parseFromHtml() works with transform rule', async () => {
- addQueryRules({
- patterns: [
- /http(s?):\/\/([\w]+.)?need-transform.tld\/*/
- ],
- transform: ($) => {
- $.querySelectorAll('a').forEach(node => {
- const sHtml = node.innerHTML
- const link = node.getAttribute('href')
- node.parentNode.replaceChild($.createTextNode(`[link url="${link}"]${sHtml}[/link]`), node)
- })
- return $
- }
+ it('check if parseFromHtml() works with multi transforms', async () => {
+ addTransformations([
+ {
+ patterns: [
+ /http(s?):\/\/need-transform.tld\/*/,
+ ],
+ post: (document) => {
+ document.querySelectorAll('a').forEach((node) => {
+ const sHtml = node.innerHTML
+ const link = node.getAttribute('href')
+ node.parentNode.replaceChild(document.createTextNode(`[link url="${link}"]${sHtml}[/link]`), node)
+ })
+ return document
+ },
+ },
+ {
+ patterns: [
+ /http(s?):\/\/sw.re\/*/,
+ ],
+ post: (document) => {
+ document.querySelectorAll('strong').forEach((node) => {
+ const b = document.createElement('B')
+ b.innerHTML = node.innerHTML
+ node.parentNode.replaceChild(b, node)
+ })
+ return document
+ },
+ },
+ ])
+ const html = readFileSync('./test-data/vnn-article.html', 'utf8')
+ const url = 'https://need-transform.tld/path/to/article'
+ const result = await parseFromHtml(html, url)
+ assert.equal(result.title, 'Article title here')
+ assert.equal(result.content.includes(' '), false)
+ assert.equal(result.content.includes('[link url="https://vnn.vn/dict/watermelon"]watermelon[/link]'), true)
+ assert.equal(result.content.includes('in its own way '), true)
})
- const html = readFileSync('./test-data/vnn-article.html', 'utf8')
- const url = 'https://need-transform.tld/path/to/article'
- const result = await parseFromHtml(html, url)
- expect(result.title).toEqual('Article title here')
- expect(result.content).toEqual(expect.not.stringContaining(' '))
- expect(result.content).toEqual(expect.stringContaining('[link url="https://vnn.vn/dict/watermelon"]watermelon[/link]'))
})
diff --git a/src/utils/purifyUrl.js b/src/utils/purifyUrl.js
index 3c4866c4..ac0774c5 100644
--- a/src/utils/purifyUrl.js
+++ b/src/utils/purifyUrl.js
@@ -59,7 +59,7 @@ const blacklistKeys = [
'WT.srch',
'pk_source',
'pk_medium',
- 'pk_campaign'
+ 'pk_campaign',
]
export default (url) => {
diff --git a/src/utils/purifyUrl.test.js b/src/utils/purifyUrl.test.js
deleted file mode 100644
index fc87db11..00000000
--- a/src/utils/purifyUrl.test.js
+++ /dev/null
@@ -1,51 +0,0 @@
-// purifyUrl.test
-/* eslint-env jest */
-
-import purifyUrl from './purifyUrl.js'
-
-describe('test purifyUrl()', () => {
- const entries = [
- {
- url: '',
- expected: null
- },
- {
- url: {},
- expected: null
- },
- {
- url: 'https://some.where/article/abc-xyz',
- expected: 'https://some.where/article/abc-xyz'
- },
- {
- url: 'https://some.where/article/abc-xyz#name,bob',
- expected: 'https://some.where/article/abc-xyz'
- },
- {
- url: 'https://some.where/article/abc-xyz?utm_source=news4&utm_medium=email&utm_campaign=spring-summer',
- expected: 'https://some.where/article/abc-xyz'
- },
- {
- url: 'https://some.where/article/abc-xyz?q=3&utm_source=news4&utm_medium=email&utm_campaign=spring-summer',
- expected: 'https://some.where/article/abc-xyz?q=3'
- },
- {
- url: 'https://some.where/article/abc-xyz?pk_source=news4&pk_medium=email&pk_campaign=spring-summer',
- expected: 'https://some.where/article/abc-xyz'
- },
- {
- url: 'https://some.where/article/abc-xyz?q=3&pk_source=news4&pk_medium=email&pk_campaign=spring-summer',
- expected: 'https://some.where/article/abc-xyz?q=3'
- }
- ]
- entries.forEach((entry) => {
- const {
- url,
- expected
- } = entry
- test(`purifyUrl("${url}") must become "${expected}"`, () => {
- const result = purifyUrl(url)
- expect(result).toEqual(expected)
- })
- })
-})
diff --git a/src/utils/retrieve.js b/src/utils/retrieve.js
index 555bfd21..6922a752 100644
--- a/src/utils/retrieve.js
+++ b/src/utils/retrieve.js
@@ -1,24 +1,36 @@
// utils -> retrieve
-import axios from 'axios'
+import fetch from 'cross-fetch'
-import logger from './logger.js'
-
-import { getRequestOptions } from '../config.js'
+const profetch = async (url, options = {}) => {
+ const { proxy = {}, signal = null } = options
+ const {
+ target,
+ headers = {},
+ } = proxy
+ const res = await fetch(target + encodeURIComponent(url), {
+ headers,
+ signal,
+ })
+ return res
+}
-export default async (url) => {
- try {
- const res = await axios.get(url, getRequestOptions())
+export default async (url, options = {}) => {
+ const {
+ headers = {
+ 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0',
+ },
+ proxy = null,
+ agent = null,
+ signal = null,
+ } = options
- const contentType = res.headers['content-type'] || ''
- if (!contentType || !contentType.includes('text/html')) {
- logger.error(`Content type must be "text/html", not "${contentType}"`)
- return null
- }
+ const res = proxy ? await profetch(url, { proxy, signal }) : await fetch(url, { headers, agent, signal })
- return res.data
- } catch (err) {
- logger.error(err.message || err)
- return null
+ const status = res.status
+ if (status >= 400) {
+ throw new Error(`Request failed with error code ${status}`)
}
+ const buffer = await res.arrayBuffer()
+ return buffer
}
diff --git a/src/utils/retrieve.test.js b/src/utils/retrieve.test.js
index f8efc314..a410c004 100644
--- a/src/utils/retrieve.test.js
+++ b/src/utils/retrieve.test.js
@@ -1,5 +1,6 @@
// retrieve.test
-/* eslint-env jest */
+import { describe, it } from 'node:test'
+import assert from 'node:assert'
import nock from 'nock'
@@ -9,39 +10,59 @@ const parseUrl = (url) => {
const re = new URL(url)
return {
baseUrl: `${re.protocol}//${re.host}`,
- path: re.pathname
+ path: re.pathname,
}
}
-test('test retrieve from good source', async () => {
- const url = 'https://some.where/good/page'
- const { baseUrl, path } = parseUrl(url)
- const scope = nock(baseUrl)
- scope.get(path).reply(200, 'this is content
', {
- 'Content-Type': 'text/html'
+describe('test retrieve() method', () => {
+ it('test retrieve with bad status code', async () => {
+ const url = 'https://some.where/bad/page'
+ const { baseUrl, path } = parseUrl(url)
+ nock(baseUrl).get(path).reply(500, 'Error 500')
+ assert.rejects(retrieve(url), new Error('Request failed with error code 500'))
})
- const result = await retrieve(url)
- expect(result).toBe('this is content
')
-})
-test('test retrieve with unsupported content type', async () => {
- const url = 'https://some.where/bad/page'
- const { baseUrl, path } = parseUrl(url)
- const scope = nock(baseUrl)
- scope.get(path).reply(200, '', {
- 'Content-Type': 'something/strange'
+ it('test retrieve from good source', async () => {
+ const url = 'https://some.where/good/page'
+ const { baseUrl, path } = parseUrl(url)
+ nock(baseUrl).get(path).reply(200, 'this is content
', {
+ 'Content-Type': 'text/html',
+ })
+ const buffer = await retrieve(url)
+ const html = Buffer.from(buffer).toString()
+ assert.equal(html, 'this is content
')
})
- const result = await retrieve(url)
- expect(result).toBe(null)
-})
-test('test retrieve from bad source', async () => {
- const url = 'https://some.where/bad/page'
- const { baseUrl, path } = parseUrl(url)
- const scope = nock(baseUrl)
- scope.get(path).reply(500, 'this is content
', {
- 'Content-Type': 'text/html'
+ it('test retrieve from good source with \\r\\n', async () => {
+ const url = 'https://some.where/good/page'
+ const { baseUrl, path } = parseUrl(url)
+ nock(baseUrl).get(path).reply(200, '\n\r\r\n\nthis is content
\n\r\r\n\n', {
+ 'Content-Type': 'text/html',
+ })
+ const buffer = await retrieve(url)
+ const html = Buffer.from(buffer).toString().trim()
+ assert.equal(html, 'this is content
')
+ })
+
+ it('test retrieve using proxy', async () => {
+ const url = 'https://some.where/good/source-with-proxy'
+ const { baseUrl, path } = parseUrl(url)
+ nock(baseUrl).get(path).reply(200, 'something bad', {
+ 'Content-Type': 'bad/thing',
+ })
+ nock('https://proxy-server.com')
+ .get('/api/proxy?url=https%3A%2F%2Fsome.where%2Fgood%2Fsource-with-proxy')
+ .reply(200, 'this is content
', {
+ 'Content-Type': 'text/html',
+ })
+
+ const buffer = await retrieve(url, {
+ proxy: {
+ target: 'https://proxy-server.com/api/proxy?url=',
+ },
+ })
+ const html = Buffer.from(buffer).toString()
+ assert.equal(html, 'this is content
')
+ nock.cleanAll()
})
- const result = await retrieve(url)
- expect(result).toBe(null)
})
diff --git a/src/utils/similarity.js b/src/utils/similarity.js
new file mode 100644
index 00000000..7e291cbc
--- /dev/null
+++ b/src/utils/similarity.js
@@ -0,0 +1,64 @@
+// similarity.js
+// https://github.com/aceakash/string-similarity
+
+import { isArray, isString } from '@ndaidong/bellajs'
+
+const areArgsValid = (mainString, targetStrings) => {
+ return isString(mainString) && isArray(targetStrings)
+ && targetStrings.length > 0 && targetStrings.every(s => isString(s))
+}
+
+export const compareTwoStrings = (first, second) => {
+ first = first.replace(/\s+/g, '')
+ second = second.replace(/\s+/g, '')
+
+ if (first === second) return 1 // identical or empty
+ if (first.length < 2 || second.length < 2) return 0 // if either is a 0-letter or 1-letter string
+
+ let firstBigrams = new Map()
+ for (let i = 0; i < first.length - 1; i++) {
+ const bigram = first.substring(i, i + 2)
+ const count = firstBigrams.has(bigram)
+ ? firstBigrams.get(bigram) + 1
+ : 1
+
+ firstBigrams.set(bigram, count)
+ }
+
+ let intersectionSize = 0
+ for (let i = 0; i < second.length - 1; i++) {
+ const bigram = second.substring(i, i + 2)
+ const count = firstBigrams.has(bigram)
+ ? firstBigrams.get(bigram)
+ : 0
+
+ if (count > 0) {
+ firstBigrams.set(bigram, count - 1)
+ intersectionSize++
+ }
+ }
+
+ return (2.0 * intersectionSize) / (first.length + second.length - 2)
+}
+
+export const findBestMatch = (mainString, targetStrings) => {
+ if (!areArgsValid(mainString, targetStrings)) {
+ throw new Error('Bad arguments: First argument should be a string, second should be an array of strings')
+ }
+
+ const ratings = []
+ let bestMatchIndex = 0
+
+ for (let i = 0; i < targetStrings.length; i++) {
+ const currentTargetString = targetStrings[i]
+ const currentRating = compareTwoStrings(mainString, currentTargetString)
+ ratings.push({ target: currentTargetString, rating: currentRating })
+ if (currentRating > ratings[bestMatchIndex].rating) {
+ bestMatchIndex = i
+ }
+ }
+
+ const bestMatch = ratings[bestMatchIndex]
+
+ return { ratings: ratings, bestMatch: bestMatch, bestMatchIndex: bestMatchIndex }
+}
diff --git a/src/utils/transformHtml.test.js b/src/utils/transformHtml.test.js
deleted file mode 100644
index ea9cc593..00000000
--- a/src/utils/transformHtml.test.js
+++ /dev/null
@@ -1,31 +0,0 @@
-// transformHtml.test
-/* eslint-env jest */
-
-import { readFileSync } from 'fs'
-
-import { isString } from 'bellajs'
-
-import transformHtml from './transformHtml.js'
-
-describe('test transformHtml()', () => {
- test('test transform html elements from good html content', async () => {
- const transform = (document) => {
- document.querySelectorAll('h1').forEach((node) => {
- const newNode = document.createElement('h2')
- newNode.innerHTML = node.innerHTML
- node.parentNode.replaceChild(newNode, node)
- })
- return document
- }
-
- const html = readFileSync('./test-data/regular-article.html', 'utf8')
- const result = transformHtml(html, transform)
- expect(isString(result)).toBe(true)
- expect(result).toEqual(
- expect.not.stringContaining('Article title here ')
- )
- expect(result).toEqual(
- expect.stringContaining('Article title here ')
- )
- })
-})
diff --git a/src/utils/transformation.js b/src/utils/transformation.js
new file mode 100644
index 00000000..5bb872b1
--- /dev/null
+++ b/src/utils/transformation.js
@@ -0,0 +1,69 @@
+// utils --> transformation.js
+
+import { isArray, isFunction, clone } from '@ndaidong/bellajs'
+import { DOMParser } from 'linkedom'
+
+const transformations = []
+
+const add = (tn) => {
+ const { patterns } = tn
+ if (!patterns || !isArray(patterns) || !patterns.length) {
+ return 0
+ }
+ transformations.push(tn)
+ return 1
+}
+
+export const addTransformations = (tfms) => {
+ if (isArray(tfms)) {
+ return tfms.map(tfm => add(tfm)).filter(result => result === 1).length
+ }
+ return add(tfms)
+}
+
+export const removeTransformations = (patterns) => {
+ if (!patterns) {
+ const removed = transformations.length
+ transformations.length = 0
+ return removed
+ }
+ let removing = 0
+ for (let i = transformations.length - 1; i > 0; i--) {
+ const { patterns: ipatterns } = transformations[i]
+ const matched = ipatterns.some((ptn) => patterns.some((pattern) => String(pattern) === String(ptn)))
+ if (matched) {
+ transformations.splice(i, 1)
+ removing += 1
+ }
+ }
+ return removing
+}
+
+export const getTransformations = () => {
+ return clone(transformations)
+}
+
+export const findTransformations = (links) => {
+ const urls = !isArray(links) ? [links] : links
+ const tfms = []
+ for (const transformation of transformations) {
+ const { patterns } = transformation
+ const matched = urls.some((url) => patterns.some((pattern) => pattern.test(url)))
+ if (matched) {
+ tfms.push(clone(transformation))
+ }
+ }
+ return tfms
+}
+
+export const execPreParser = (html, links) => {
+ const doc = new DOMParser().parseFromString(html, 'text/html')
+ findTransformations(links).map(tfm => tfm.pre).filter(fn => isFunction(fn)).map(fn => fn(doc))
+ return Array.from(doc.childNodes).map(it => it.outerHTML).join('')
+}
+
+export const execPostParser = (html, links) => {
+ const doc = new DOMParser().parseFromString(html, 'text/html')
+ findTransformations(links).map(tfm => tfm.post).filter(fn => isFunction(fn)).map(fn => fn(doc))
+ return Array.from(doc.childNodes).map(it => it.outerHTML).join('')
+}
diff --git a/src/utils/transformation.test.js b/src/utils/transformation.test.js
new file mode 100644
index 00000000..18a54d4a
--- /dev/null
+++ b/src/utils/transformation.test.js
@@ -0,0 +1,194 @@
+// transformation.test
+import { describe, it as test } from 'node:test'
+import assert from 'node:assert'
+
+import {
+ addTransformations,
+ removeTransformations,
+ getTransformations,
+ findTransformations,
+ execPreParser,
+ execPostParser
+} from './transformation.js'
+
+describe('test transformation', () => {
+ test(' add one transformation object', () => {
+ const result = addTransformations({
+ patterns: [
+ /http(s?):\/\/([\w]+.)?def.tld\/*/,
+ ],
+ pre: (document) => {
+ return document
+ },
+ post: (document) => {
+ return document
+ },
+ })
+ assert.equal(result, 1)
+ })
+
+ test(' add multi transformation object', () => {
+ const result = addTransformations([
+ {
+ patterns: [
+ /http(s?):\/\/google.com\/*/,
+ /http(s?):\/\/goo.gl\/*/,
+ ],
+ },
+ {
+ patterns: [
+ /http(s?):\/\/goo.gl\/*/,
+ /http(s?):\/\/google.inc\/*/,
+ ],
+ },
+ ])
+ assert.equal(result, 2)
+ })
+
+ test(' add transformation object without patterns', () => {
+ const result = addTransformations({
+ pre: (document) => {
+ return document
+ },
+ post: (document) => {
+ return document
+ },
+ })
+ assert.equal(result, 0)
+ })
+
+ test(' add transformation object without valid patterns', () => {
+ const result = addTransformations({
+ patterns: 123,
+ pre: (document) => {
+ return document
+ },
+ post: (document) => {
+ return document
+ },
+ })
+ assert.equal(result, 0)
+ })
+
+ test(' get all transformations', () => {
+ const result = getTransformations()
+ assert.equal(result.length, 3)
+ assert.deepEqual(result[0].patterns[0], /http(s?):\/\/([\w]+.)?def.tld\/*/)
+ })
+
+ test(' remove one transformation', () => {
+ addTransformations([
+ {
+ patterns: [
+ /http(s?):\/\/abc.com\/*/,
+ /http(s?):\/\/def.gl\/*/,
+ ],
+ },
+ {
+ patterns: [
+ /http(s?):\/\/hik.gl\/*/,
+ /http(s?):\/\/lmn.inc\/*/,
+ ],
+ },
+ {
+ patterns: [
+ /http(s?):\/\/opq.gl\/*/,
+ /http(s?):\/\/rst.inc\/*/,
+ ],
+ },
+ ])
+ const result = removeTransformations([
+ /http(s?):\/\/goo.gl\/*/,
+ ])
+ assert.equal(result, 2)
+ })
+
+ test(' get all transformations again', () => {
+ const result = getTransformations()
+ assert.equal(result.length, 4)
+ assert.deepEqual(result[3].patterns[1], /http(s?):\/\/rst.inc\/*/)
+ })
+
+ test(' find transformations', () => {
+ addTransformations([
+ {
+ patterns: [
+ /http(s?):\/\/def.gl\/*/,
+ /http(s?):\/\/uvw.inc\/*/,
+ ],
+ },
+ ])
+ const notFound = findTransformations([
+ 'https://goo.gl/docs/article.html',
+ ])
+ assert.deepEqual(notFound, [])
+
+ const foundOne = findTransformations([
+ 'https://lmn.inc/docs/article.html',
+ ])
+ assert.equal(foundOne.length, 1)
+
+ const foundTwo = findTransformations([
+ 'https://def.gl/docs/article.html',
+ ])
+ assert.equal(foundTwo.length, 2)
+ })
+
+ test(' run execPreParser', () => {
+ addTransformations([
+ {
+ patterns: [
+ /http(s?):\/\/xyz.com\/*/,
+ ],
+ pre: (doc) => {
+ doc.querySelectorAll('.adv').forEach((element) => {
+ element.parentNode.removeChild(element)
+ })
+ return doc
+ },
+ },
+ ])
+ const html = `
+
+ hi
user , this is an advertisement element
+
free product now!
+
user , this is an advertisement element'), true)
+ assert.equal(result.includes('free product now!
'), false)
+ })
+
+ test(' run execPostParser', () => {
+ addTransformations([
+ {
+ patterns: [
+ /http(s?):\/\/xyz.com\/*/,
+ ],
+ post: (doc) => {
+ doc.querySelectorAll('b').forEach((element) => {
+ const itag = doc.createElement('i')
+ itag.innerHTML = element.innerHTML
+ element.parentNode.replaceChild(itag, element)
+ })
+ return doc
+ },
+ },
+ ])
+ const html = `
+
+ hi
user ,
+
Thank you for your feedback!
+
user '), true)
+ assert.equal(result.includes('user '), false)
+ })
+
+ test(' remove all transformations', () => {
+ const result = removeTransformations()
+ assert.equal(result, 7)
+ assert.deepEqual(getTransformations(), [])
+ })
+})
diff --git a/test-data/article-with-classes-attributes.html b/test-data/article-with-classes-attributes.html
new file mode 100644
index 00000000..155dc66e
--- /dev/null
+++ b/test-data/article-with-classes-attributes.html
@@ -0,0 +1,64 @@
+
+
+
+ Article title here - ArticleParser
+
+
+
+ Article title here
+
+ Few can name a
rational peach that isn't a conscientious goldfish! One cannot separate snakes from plucky pomegranates? Draped neatly on a hanger, the melons could be said to resemble knowledgeable pigs. Some posit the enchanting tiger to be less than confident. The literature would have us believe that an impartial turtle is not but a hippopotamus. Unfortunately, that is wrong; on the contrary, those cows are nothing more than pandas! The chicken is a shark; A turtle can hardly be considered a kind horse without also being a pomegranate. Zebras are witty persimmons.
+ Those cheetahs are nothing more than dogs. A watermelon is an exuberant kangaroo. An octopus is the tangerine of a grapes? The cherry is a shark. Recent controversy aside, they were lost without the cheerful plum that composed their fox. As far as we can estimate, one cannot separate camels from dynamic hamsters. Those tigers are nothing more than cows! A cow is a squirrel from the right perspective. Their banana was, in this moment, a helpful bear.
+ The first fair dog is, in its own way, a lemon.
+ 4746 Kelly Drive, West Virginia
+
+
+ const add = (a, b) => {
+ return a + b
+ }
+
+
+ OK, that is good
+
+
+
+
+
+
+
diff --git a/test-data/html-article-no-source.html b/test-data/html-article-no-source.html
index c040fa2e..12237080 100644
--- a/test-data/html-article-no-source.html
+++ b/test-data/html-article-no-source.html
@@ -17,6 +17,6 @@
To be more specific, those turtles are nothing more than fishes. A grape can hardly be considered a shrewd goldfish without also being an owl. Some unbiased goats are thought of simply as tangerines.
Shouting with happiness, a courageous elephant is a duck of the mind? Some posit the upbeat hippopotamus to be less than enchanting. It's an undeniable fact, really; authors often misinterpret the grape as an endurable rabbit, when in actuality it feels more like a tough dolphin. We know that a cherry can hardly be considered a responsible apricot without also being a nectarine.
- s
+
'))
-})
+describe('test extractWithReadability()', () => {
+ it('extract from good html content', async () => {
+ const html = readFileSync('./test-data/regular-article.html', 'utf8')
+ const result = extractWithReadability(html, 'https://foo.bar')
+ assert.ok(isString(result))
+ assert.ok(result.length > 200)
+ assert.ok(result.includes('
'))
- expect(result.content).toEqual(expect.stringContaining('
'))
- }
+ expectation: (result) => {
+ assert.equal(result.source, 'somewhere.any')
},
- {
- input: {
- desc: 'a webpage with regular article',
- html: readFileSync('./test-data/regular-article.html', 'utf8'),
- url: 'https://somewhere.com/path/to/article'
- },
- expectation: (result, expect) => {
- expect(result.title).toEqual('Article title here')
- expect(result.description).toEqual('Few words to summarize this article content')
- expect(result.content).toEqual(expect.stringContaining('
+