added first metascraper iteration

Author: appinteractive

package.json

@@ -56,7 +56,7 @@
     "compression": "~1.7.1",
     "cors": "~2.8.4",
     "crypto": "~1.0.1",
-    "crypto-js": "~3.1.9-1",
+    "crypto-js": "^3.1.9-1",
     "dauria": "~2.0.0",
     "email-templates": "2.6.0",
     "feathers": "~2.2.4",
@@ -81,12 +81,23 @@
     "feathers-socketio": "~2.0.1",
     "fs-blob-store": "~5.2.1",
     "fs-extra": "~4.0.2",
-    "got": "~8.3.0",
+    "got": "^8.3.0",
     "handlebars": "~4.0.11",
     "handlebars-layouts": "~3.1.4",
     "helmet": "~3.10.0",
     "html-excerpt": "~0.1.0",
-    "metascraper": "~3.9.2",
+    "metascraper": "^3.9.2",
+    "metascraper-author": "^3.9.2",
+    "metascraper-clearbit-logo": "^3.7.0",
+    "metascraper-date": "^3.3.0",
+    "metascraper-description": "^3.9.2",
+    "metascraper-image": "^3.9.2",
+    "metascraper-logo": "^3.9.2",
+    "metascraper-logo-favicon": "^3.7.0",
+    "metascraper-publisher": "^3.4.0",
+    "metascraper-title": "^3.9.2",
+    "metascraper-url": "^3.9.2",
+    "metascraper-youtube": "^3.9.2",
     "mime": "^2.2.0",
     "mongoose": "~4.13.2",
     "multer": "~1.3.0",

server/models/contributions.model.js

@@ -14,6 +14,17 @@ module.exports = function (app) {
     reason: { type: String }
   });
 
+  const metaSchema = mongooseClient.Schema({
+    hasVideo: {
+      type: Boolean,
+      default: false
+    },
+    embedds: {
+      type: Object,
+      default: {}
+    }
+  });
+
   const contributions = new mongooseClient.Schema({
     userId: { type: String, required: true },
     categoryIds: { type: Array },
@@ -28,12 +39,7 @@ module.exports = function (app) {
     teaserImg: { type: String },
     language: { type: String, required: true },
     shoutCount: { type: Number, default: 0 },
-    meta: {
-      type: Object,
-      default: {
-        hasVideo: false
-      }
-    },
+    meta: metaSchema,
     visibility: {
       type: String,
       enum: ['public', 'friends', 'private'],

server/services/contributions/hooks/metascraper.js

@@ -1,8 +1,32 @@
 // get link metadata
 // TODO: add more services and use the metascraper to fill some metadata on the article
 
-// const metascraper = require('metascraper');
-// const got = require('got');
+const metascraper = require('metascraper').load([
+  require('metascraper-author')(),
+  require('metascraper-date')(),
+  require('metascraper-description')(),
+  require('metascraper-image')(),
+  require('metascraper-logo')(),
+  require('metascraper-clearbit-logo')(),
+  require('metascraper-logo-favicon')(),
+  require('metascraper-publisher')(),
+  require('metascraper-title')(),
+  require('metascraper-url')(),
+  require('metascraper-youtube')(),
+]);
+const got = require('got');
+const _ = require('lodash');
+
+const getMetadata = async (targetUrl, app) => {
+  const { body: html, url } = await got(targetUrl);
+  app.debug(`getMetadata - getting metadata for ${url}`);
+  // app.debug(html);
+  // app.debug(html);
+  const metadata = await metascraper({ html, url });
+  // app.debug(metadata);
+  app.debug(`getMetadata - got metadata for ${url}`);
+  return metadata;
+};
 
 module.exports = function () {
   return function (hook) {
@@ -17,28 +41,55 @@ module.exports = function () {
         return resolve(hook);
       }
 
+      let promises = [];
+      let embedds = {};
+
       try {
         // find links
         const youtubeRegex = new RegExp(/(?:(?:https?:)?\/\/)?(?:www\.)?youtu(?:be\.com\/(?:watch\?(?:.*?&(?:amp;)?)*v=|v\/|embed\/)|\.be\/)([\w‌​\-]+)(?:(?:&(?:amp;)?|\?)[\w\?=]*)*/, 'ig'); // eslint-disable-line
         const youtubeLinks = youtubeRegex.exec(hook.data.content);
 
-        // html link
-        // const htmlLink = new RegExp(/<a\s[^>]*href=\"([^\"]*)\"[^>]*>(.*)<\/a>/, 'ig'); // eslint-disable-line
+        // html links
+        const linkRegex = new RegExp(/<a\s[^>]*href=\"([^\"]*)\"[^>]*>([^<]*)<\/a>/, 'ig') // eslint-disable-line
+        let match;
+        while (match = linkRegex.exec(hook.data.content)) {
+          const url = match[1];
+          hook.app.debug(url);
 
-        // here you could scrape the url for metadata
-        // hook.app.debug('#6');
-        // const { body: html, url } = await got(youtubeLinks[0]);
-        // const metadata = await metascraper({html, url});
-        // hook.app.debug('metadata');
-        // hook.app.debug(metadata);
+          // skip if url already exists
+          if (!_.isEmpty(embedds[url])) {
+            continue;
+          }
+          // here you could scrape the url for metadata
+          // hook.app.debug(match);
+          promises.push(new Promise(async (resolve) => {
+            try {
+              const metadata = await getMetadata(url, hook.app);
+              embedds[url] = metadata;
+              return resolve(metadata);
+            } catch (err) {
+              hook.app.error('FAILED TO GRAB THE LINK');
+              return resolve();
+            }
+          }));
+        }
 
         if (youtubeLinks.length >= 2) {
           hook.data.teaserImg = `https://img.youtube.com/vi/${youtubeLinks[1]}/hqdefault.jpg`;
           hook.data.meta = Object.assign(hook.data.meta || {}, { hasVideo: true });
         }
       } catch (err) {} // eslint-disable-line
 
-      return resolve(hook);
+      return Promise.all(promises)
+        .then(() => {
+          hook.app.debug('embedds:');
+          hook.app.debug(embedds);
+
+          hook.data.meta.embedds = embedds;
+
+          hook.app.debug('FINISHED!');
+          resolve(hook);
+        });
     });
   };
 };