feat: fetch page titles for links added via telegram

Author: muety

app.js

@@ -7,6 +7,7 @@ const express = require('express')
 function connect(success, error) {
     function onConnectFailed(err) {
         setTimeout(() => { error() }, 0)
+        console.error(err)
         throw new Error(`unable to connect to database at ${config.db}`)
     }
 

app/controllers/remote.js

@@ -1,13 +1,10 @@
 const express = require('express'),
     router = express.Router(),
-    axios = require('axios'),
-    htmlparser = require('htmlparser2'),
-    cache = require('memory-cache'),
-    config = require('../../config/config'),
     morgan = require('../../config/middlewares/morgan')(),
-    logger = require('./../../config/log')()
+    logger = require('./../../config/log')(),
+    PageMetaService = require('../services/page_meta')
 
-const CACHE_TIMEOUT = 1000 * 60 * 60 * 24
+const pageMetaService = new PageMetaService()
 
 module.exports = function (app) {
     app.use('/api/remote', router)
@@ -40,53 +37,13 @@ module.exports = function (app) {
 
         if (!url || !url.length) return res.makeError(400, 'No url given')
 
-        function sendTitle(t) {
-            cache.put(url, t, CACHE_TIMEOUT)
-            return res.send({ title: t })
-        }
-
-        const cachedValue = cache.get(url)
-        if (cachedValue) return sendTitle(cachedValue)
-        else if (cache.keys().includes(url)) return sendTitle(null)
-
-        cache.put(url, null, CACHE_TIMEOUT) // prevent redirect loops
-
-        axios({
-            method: 'head',
-            url: url,
-            headers: { 'Accept': 'text/html' }
-        }).then((response) => {
-            const contentType = response.headers['content-type']
-            const contentLenth = parseInt(response['content-length']) / 1000
-            if (!contentType.includes('text/html') && (!contentLenth || contentLenth > config.maxHtmlSizeKb)) {
-                sendTitle(null)
-                return null
-            }
-            return axios({
-                method: 'get',
-                url: url,
-                headers: { 'Accept': 'text/html' }
+        pageMetaService.fetchTitle(url)
+            .then((title) => {
+                return res.send({ title })
             })
-        }).then((response) => {
-            if (!response) return
-            const contentType = response.headers['content-type']
-            if (!contentType.startsWith('text/html')) return sendTitle(null)
-            const handler = new htmlparser.DomHandler((error, dom) => {
-                if (error) return res.makeError(404, 'Not found')
-                const htmlNode = dom.filter((n) => { return n.type === 'tag' && n.name === 'html' })[0]
-                const headNode = htmlNode.children.filter((n) => { return n.type === 'tag' && n.name === 'head' })[0]
-                const titleNode = headNode.children.filter((n) => { return n.type === 'tag' && n.name === 'title' })[0]
-                let title = titleNode.children[0].data.trim()
-                title = title.replace(/&#(\d+);/g, (match, dec) => {
-                    return String.fromCharCode(dec)
-                }).replace(/&.+;/g, '')
-                return sendTitle(title)
+            .catch((err) => {
+                logger.error(`Failed to resolve title for URL ${url} - ${err}`)
+                return res.makeError(404, 'Not found')
             })
-            const parser = new htmlparser.Parser(handler)
-            parser.parseComplete(response.data)
-        }).catch((err) => {
-            logger.error(`Failed to resolve title for URL ${url} - ${err}`)
-            return res.makeError(404, 'Not found')
-        })
     })
 }

app/controllers/telegram.js

@@ -14,10 +14,12 @@ const fs = require('fs'),
     tgutils = require('./utils/telegram'),
     addLink = require('./utils/collection').addLink,
     fetchCollections = require('./utils/collection').fetchCollections,
-    addShortlink = require('./utils/shortlink').addShortlink
+    addShortlink = require('./utils/shortlink').addShortlink,
+    PageMetaService = require('./../services/page_meta')
 
 const otps = {}
 const pendingLinks = {}  // by user
+const pageMetaService = new PageMetaService()
 
 const CMD_START = 'CMD_START',
     CMD_LOGOUT = 'CMD_LOGOUT',
@@ -169,7 +171,12 @@ const commandProcessors = {
                 const link = pendingLinks[user._id]
                 const index = parseInt(args[0]) - 1
 
-                fetchCollections(user)
+                pageMetaService.fetchTitle(link.url)
+                    .then((title) => {
+                        link.description = title
+                    })
+                    .catch((err) => null)  // ignore and continue without setting title
+                    .then(() => fetchCollections(user))
                     .then(({ data }) => {
                         // this is not robust to the collections changing while the user does their choice
                         if (index < 0 || index >= data.length) return tgutils.doRequest('sendMessage', { chat_id: rawMessage.chat.id, text: '❌ Invalid collection.' })

app/services/page_meta.js

@@ -0,0 +1,72 @@
+const axios = require('axios'),
+    htmlparser = require('htmlparser2'),
+    cache = require('memory-cache'),
+    config = require('../../config/config')
+
+const CACHE_TIMEOUT = 1000 * 60 * 60 * 24
+
+class PageMetaService {
+    constructor() {
+        if (!PageMetaService._instance) {
+            PageMetaService.instance = this
+        }
+        return PageMetaService._instance
+    }
+
+    async fetchTitle(url) {
+        function done(title) {
+            cache.put(url, title, CACHE_TIMEOUT)
+            return title
+        }
+
+        const cachedValue = cache.get(url)
+        if (cachedValue) return done(cachedValue)
+        else if (cache.keys().includes(url)) return done(null)
+
+        cache.put(url, null, CACHE_TIMEOUT) // prevent redirect loops
+
+        return await axios({
+            method: 'head',
+            url: url,
+            headers: { 'Accept': 'text/html' }
+        }).then((response) => {
+            const contentType = response.headers['content-type']
+            const contentLenth = parseInt(response['content-length']) / 1000
+
+            if (!contentType.includes('text/html') && (!contentLenth || contentLenth > config.maxHtmlSizeKb)) {
+                return done(null)
+            }
+
+            return axios({
+                method: 'get',
+                url: url,
+                headers: { 'Accept': 'text/html' }
+            })
+        }).then((response) => {
+            if (!response) return
+
+            const contentType = response.headers['content-type']
+            if (!contentType.startsWith('text/html')) return done(null)
+
+            let title = null
+
+            const handler = new htmlparser.DomHandler((error, dom) => {
+                if (error) throw new Error(error)
+
+                const htmlNode = dom.filter((n) => { return n.type === 'tag' && n.name === 'html' })[0]
+                const headNode = htmlNode.children.filter((n) => { return n.type === 'tag' && n.name === 'head' })[0]
+                const titleNode = headNode.children.filter((n) => { return n.type === 'tag' && n.name === 'title' })[0]
+
+                title = titleNode.children[0].data.trim()
+                title = title.replace(/&#(\d+);/g, (match, dec) => String.fromCharCode(dec)).replace(/&.+;/g, '')
+            })
+
+            const parser = new htmlparser.Parser(handler)
+            parser.parseComplete(response.data)
+
+            return title
+        })
+    }
+}
+
+module.exports = PageMetaService