Merge pull request #74 from fmacpro/resurrection-update

1.0.0 Resurrection update

Author: fmacpro

.eslintrc.json

@@ -263,7 +263,7 @@ Remove the style attribute on every e and under.
 
 | Param | Type |
 | --- | --- |
-| element | <code>jQuery</code> | 
+| element | <code>jQuery</code> |
 
 <a name="killBreaks"></a>
 
@@ -404,9 +404,32 @@ Cleans the article content
 Initialize a node with the readability object. Also checks the
 className/id for special names to add to its score.
 
-**Kind**: global function  
+**Kind**: global function
 
 | Param | Type |
 | --- | --- |
-| element | <code>jQuery</code> | 
+| element | <code>jQuery</code> |
+
+## Dependencies
+
+- [Puppeteer](https://github.com/GoogleChrome/puppeteer/)
+- [puppeteer-extra](https://github.com/berstend/puppeteer-extra)
+- [puppeteer-extra-plugin-stealth](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth)
+- [puppeteer-extra-plugin-user-data-dir](overrides/puppeteer-extra-plugin-user-data-dir)
+- [lighthouse](https://github.com/GoogleChrome/lighthouse)
+- [compromise](https://ghub.io/compromise)
+- [retext](https://ghub.io/retext)
+- [retext-pos](https://github.com/retextjs/retext-pos)
+- [retext-keywords](https://ghub.io/retext-keywords)
+- [retext-spell](https://ghub.io/retext-spell)
+- [sentiment](https://ghub.io/sentiment)
+- [jquery](https://ghub.io/jquery)
+- [jsdom](https://ghub.io/jsdom)
+- [lodash](https://ghub.io/lodash)
+- [absolutify](https://ghub.io/absolutify)
+- [clean-html](https://ghub.io/clean-html)
+- [dictionary-en-gb](https://ghub.io/dictionary-en-gb)
+- [html-to-text](https://ghub.io/html-to-text)
+- [nlcst-to-string](https://ghub.io/nlcst-to-string)
+- [vfile-reporter-json](https://ghub.io/vfile-reporter-json)
 

APIDOC.md

@@ -26,14 +26,14 @@ npm install horseman-article-parser --save
 ### Usage Example
 
 ```
-var parser = require('horseman-article-parser');
+import { parseArticle } from 'horseman-article-parser';
 
-var options = {
+const options = {
   url: "https://www.theguardian.com/politics/2018/sep/24/theresa-may-calls-for-immigration-based-on-skills-and-wealth",
   enabled: ['lighthouse', 'screenshot', 'links', 'sentiment', 'entities', 'spelling', 'keywords']
 }
 
-parser.parseArticle(options)
+parseArticle(options)
   .then(function (article) {
 
     var response = {
@@ -260,33 +260,34 @@ npm run docs
 ## Dependencies
 
 - [Puppeteer](https://github.com/GoogleChrome/puppeteer/): High-level API to control Chrome or Chromium over the DevTools Protocol
-- [compromise](https://ghub.io/compromise): natural language processing in the browser
+- [puppeteer-extra](https://github.com/berstend/puppeteer-extra): Framework for puppeteer plugins
+- [puppeteer-extra-plugin-stealth](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth): Plugin to evade detection
+- [puppeteer-extra-plugin-user-data-dir](overrides/puppeteer-extra-plugin-user-data-dir): Persist and reuse Chromium user data
+- [lighthouse](https://github.com/GoogleChrome/lighthouse): Automated auditing, performance metrics, and best practices
+- [compromise](https://ghub.io/compromise): Natural language processing in the browser
 - [retext](https://ghub.io/retext): Natural language processor powered by plugins
 - [retext-pos](https://github.com/retextjs/retext-pos): Plugin to add part-of-speech (POS) tags
 - [retext-keywords](https://ghub.io/retext-keywords): Keyword extraction with Retext
 - [retext-spell](https://ghub.io/retext-spell): Spelling checker for retext
 - [sentiment](https://ghub.io/sentiment): AFINN-based sentiment analysis for Node.js
 - [jquery](https://ghub.io/jquery): JavaScript library for DOM operations
 - [jsdom](https://ghub.io/jsdom): A JavaScript implementation of many web standards
-- [lodash](https://ghub.io/lodash): Lodash modular utilities.
+- [lodash](https://ghub.io/lodash): Lodash modular utilities
 - [absolutify](https://ghub.io/absolutify): Relative to Absolute URL Replacer
 - [clean-html](https://ghub.io/clean-html): HTML cleaner and beautifier
 - [dictionary-en-gb](https://ghub.io/dictionary-en-gb): English (United Kingdom) spelling dictionary in UTF-8
-- [html-to-text](https://ghub.io/html-to-text): Advanced html to plain text converter
+- [html-to-text](https://ghub.io/html-to-text): Advanced HTML to plain text converter
 - [nlcst-to-string](https://ghub.io/nlcst-to-string): Stringify NLCST
 - [vfile-reporter-json](https://ghub.io/vfile-reporter-json): JSON reporter for virtual files
 
 
 ## Dev Dependencies
 
-- [eslint](https://ghub.io/eslint): An AST-based pattern checker for JavaScript.
-- [eslint-config-standard](https://ghub.io/eslint-config-standard): JavaScript Standard Style - ESLint Shareable Config
-- [eslint-plugin-import](https://ghub.io/eslint-plugin-import): Import with sanity.
+- [eslint](https://ghub.io/eslint): An AST-based pattern checker for JavaScript
+- [eslint-plugin-import](https://ghub.io/eslint-plugin-import): Import with sanity
 - [eslint-plugin-json](https://ghub.io/eslint-plugin-json): Lint JSON files
-- [eslint-plugin-node](https://ghub.io/eslint-plugin-node): Additional ESLint's rules for Node.js
+- [eslint-plugin-n](https://ghub.io/eslint-plugin-n): Additional ESLint rules for Node.js
 - [eslint-plugin-promise](https://ghub.io/eslint-plugin-promise): Enforce best practices for JavaScript promises
-- [eslint-plugin-standard](https://ghub.io/eslint-plugin-standard): ESlint Plugin for the Standard Linter
-- [jsdoc-to-markdown](https://github.com/jsdoc2md/jsdoc-to-markdown): Generates markdown API documentation from jsdoc annotated source code.
 
 
 ## License

README.md

@@ -0,0 +1,26 @@
+import retext from 'retext'
+import nlcstToString from 'nlcst-to-string'
+import pos from 'retext-pos'
+import keywords from 'retext-keywords'
+import _ from 'lodash'
+
+export default async function keywordParser (html, options = { maximum: 10 }) {
+  const file = await retext().use(pos).use(keywords, options).process(html)
+
+  const keywordsArr = file.data.keywords.map(keyword => ({
+    keyword: nlcstToString(keyword.matches[0].node),
+    score: keyword.score
+  }))
+
+  const keyphrases = file.data.keyphrases.map(phrase => {
+    const nodes = phrase.matches[0].nodes
+    const tree = _.map(nodes)
+    return {
+      keyphrase: nlcstToString(tree, ''),
+      score: phrase.score,
+      weight: phrase.weight
+    }
+  }).sort((a, b) => (a.score > b.score) ? -1 : 1)
+
+  return { keywords: keywordsArr, keyphrases }
+}

controllers/keywordParser.js

@@ -0,0 +1,14 @@
+import lighthouseImport from 'lighthouse'
+const lighthouse = lighthouseImport.default || lighthouseImport
+
+export default async function lighthouseAnalysis (browser, options, socket) {
+  socket.emit('parse:status', 'Starting Lighthouse')
+
+  const results = await lighthouse(options.url, {
+    port: (new URL(browser.wsEndpoint())).port,
+    output: 'json'
+  })
+
+  socket.emit('parse:status', 'Lighthouse Analysis Complete')
+  return results.lhr
+}

controllers/lighthouse.js

@@ -0,0 +1,32 @@
+import retext from 'retext'
+import spell from 'retext-spell'
+import dictionary from 'dictionary-en-gb'
+import report from 'vfile-reporter-json'
+
+export default function spellCheck (text, options) {
+  text = text.replace(/[0-9]{1,}[a-zA-Z]{1,}/gi, '')
+
+  return new Promise(function (resolve, reject) {
+    if (typeof options === 'undefined') {
+      options = {
+        dictionary: dictionary
+      }
+    }
+
+    if (typeof options.dictionary === 'undefined') {
+      options.dictionary = dictionary
+    }
+
+    retext()
+      .use(spell, options)
+      .process(text, function (error, file) {
+        if (error) {
+          reject(error)
+        }
+
+        let results = JSON.parse(report(file))
+        results = results[0].messages
+        resolve(results)
+      })
+  })
+}

controllers/spellCheck.js

@@ -0,0 +1,37 @@
+import { FlatCompat } from '@eslint/eslintrc';
+import js from '@eslint/js';
+import jsonPlugin from 'eslint-plugin-json';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const compat = new FlatCompat({
+  baseDirectory: __dirname,
+  recommendedConfig: js.configs.recommended
+});
+
+export default [
+  {
+    ignores: ['eslint.config.mjs', 'overrides/**']
+  },
+  ...compat.extends(
+    'eslint:recommended',
+    'plugin:import/recommended',
+    'plugin:n/recommended',
+    'plugin:promise/recommended'
+  ),
+  jsonPlugin.configs.recommended,
+  {
+    languageOptions: {
+      globals: {
+        jQuery: 'readonly',
+        window: 'readonly'
+      }
+    },
+    rules: {
+      'no-prototype-builtins': 'off'
+    }
+  }
+];