fix: missing user agent for medium (#154)

Author: bxcodec

app/util/feedtojson.ts

@@ -1,11 +1,16 @@
 import { parse } from 'rss-to-json'
+import { getRandomUserAgent } from './useragent'
 
 export const feedToJSON = async (rssUrl: string) => {
     try {
-        const feed = await parse(rssUrl)
+        const feed = await parse(rssUrl, {
+            headers: {
+                'User-Agent': getRandomUserAgent(),
+            }
+        })
         return feed
     } catch (error) {
-        console.error(error)
+        console.error('error parsing RSS', error)
         return null
     }
 }

app/util/medium.ts

@@ -2,7 +2,7 @@ import axios from 'axios';
 import moment from 'moment';
 import { feedToJSON } from './feedtojson';
 import { JSDOM } from 'jsdom';
-
+import { getRandomUserAgent } from './useragent';
 
 export const getArticle = async (index: string, username: string) => {
   const rssUrl = `https://medium.com/feed/${username}`
@@ -24,12 +24,14 @@ export const getArticle = async (index: string, username: string) => {
   ];
 
   const description = content || desc;
-
-
-  const responseThumbnail = await axios(thumbnail.src, { responseType: 'arraybuffer' });
+  const responseThumbnail = await axios(thumbnail.src, {
+    responseType: 'arraybuffer',
+    headers: {
+      'User-Agent': getRandomUserAgent(),
+    }
+  });
   const base64Img = Buffer.from(responseThumbnail.data, 'binary').toString('base64');
 
-
   const imgTypeArr = thumbnail.src.split('.');
   const imgType = imgTypeArr[imgTypeArr.length - 1];
 

app/util/useragent.ts

@@ -0,0 +1,45 @@
+const userAgents = [
+    // Chrome on Windows
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+
+    // Chrome on macOS
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+
+    // Firefox on Windows
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0",
+
+    // Firefox on macOS
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:123.0) Gecko/20100101 Firefox/123.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:122.0) Gecko/20100101 Firefox/122.0",
+
+    // Safari on macOS
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3 Safari/605.1.15",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15",
+
+    // Edge on Windows
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0",
+
+    // Mobile - Chrome on Android
+    "Mozilla/5.0 (Linux; Android 13; SM-S908B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.90 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.90 Mobile Safari/537.36",
+
+    // Mobile - Safari on iOS
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 17_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3 Mobile/15E148 Safari/604.1",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 17_2_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1",
+
+    // Common RSS readers and feed fetchers
+    "Feedly/1.0 (+http://www.feedly.com/fetcher.html)",
+    "Feedbin/latest (https://feedbin.com)",
+    "FeedParser/5.2.1 +https://github.com/kurtmckee/feedparser/",
+    "NewsBlur/4.0 (https://www.newsblur.com)",
+    "Tiny Tiny RSS/22.10 (http://tt-rss.org/)"
+];
+
+export function getRandomUserAgent() {
+    const randomIndex = Math.floor(Math.random() * userAgents.length);
+    return userAgents[randomIndex];
+}