Merge pull request #1031 from DIYgod/master

[pull] master from diygod:master

Author: pull[bot]

.github/workflows/docs-search-index.yml

@@ -6,6 +6,7 @@ on:
       - master
     paths:
       - '.github/workflows/docs-search-index.yml'
+      - 'scripts/docs-scraper/docs.rsshub.app.json'
       - 'website/**'
   workflow_dispatch: ~
   schedule:

lib/config.js

@@ -10,6 +10,7 @@ const calculateValue = () => {
     const email_config = {};
     const discuz_cookies = {};
     const medium_cookies = {};
+    const discourse_config = {};
 
     for (const name in envs) {
         if (name.startsWith('BILIBILI_COOKIE_')) {
@@ -27,6 +28,9 @@ const calculateValue = () => {
         } else if (name.startsWith('MEDIUM_COOKIE_')) {
             const username = name.slice(14).toLowerCase();
             medium_cookies[username] = envs[name];
+        } else if (name.startsWith('DISCOURSE_CONFIG_')) {
+            const id = name.slice('DISCOURSE_CONFIG_'.length);
+            discourse_config[id] = JSON.parse(envs[name]);
         }
     }
 
@@ -136,6 +140,9 @@ const calculateValue = () => {
         discord: {
             authorization: envs.DISCORD_AUTHORIZATION,
         },
+        discourse: {
+            config: discourse_config,
+        },
         discuz: {
             cookies: discuz_cookies,
         },
@@ -275,6 +282,9 @@ const calculateValue = () => {
         telegram: {
             token: envs.TELEGRAM_TOKEN,
         },
+        tophub: {
+            cookie: envs.TOPHUB_COOKIE,
+        },
         twitter: {
             consumer_key: envs.TWITTER_CONSUMER_KEY,
             consumer_secret: envs.TWITTER_CONSUMER_SECRET,

lib/radar.js

@@ -3,7 +3,8 @@ const fs = require('fs');
 const toSource = require('tosource');
 const { join } = require('path');
 
-const allowNamespace = ['ehentai', 'test'];
+// Namespaces that do not require radar.js
+const allowNamespace = ['discourse', 'ehentai', 'test'];
 // Check if a radar.js file is exist under each folder of dirname
 for (const dir of fs.readdirSync(dirname)) {
     const dirPath = join(dirname, dir);

lib/router.js

@@ -770,8 +770,8 @@ router.get('/blogs/wordpress/:domain/:https?', lazyloadRouteHandler('./routes/bl
 // 西祠胡同
 router.get('/xici/:id?', lazyloadRouteHandler('./routes/xici'));
 
-// 今日热榜
-router.get('/tophub/:id', lazyloadRouteHandler('./routes/tophub'));
+// 今日热榜 migrated to v2
+// router.get('/tophub/:id', lazyloadRouteHandler('./routes/tophub'));
 
 // 游戏时光
 router.get('/vgtime/news', lazyloadRouteHandler('./routes/vgtime/news.js'));

lib/v2/baidu/maintainer.js

@@ -1,7 +1,7 @@
 module.exports = {
     '/gushitong/index': ['CaoMeiYouRen'],
-    '/tieba/forum/:kw': ['u3u'],
-    '/tieba/forum/good/:kw/:cid?': ['u3u'],
+    '/tieba/forum/good/:kw/:cid?/:sortBy?': ['u3u'],
+    '/tieba/forum/:kw/:sortBy?': ['u3u'],
     '/tieba/post/:id': ['u3u'],
     '/tieba/post/lz/:id': ['u3u'],
     '/tieba/user/:uid': ['igxlin', 'nczitzk'],

lib/v2/baidu/router.js

@@ -1,7 +1,7 @@
 module.exports = (router) => {
     router.get('/gushitong/index', require('./gushitong'));
-    router.get('/tieba/forum/:kw', require('./tieba/forum'));
-    router.get('/tieba/forum/good/:kw/:cid?', require('./tieba/forum'));
+    router.get('/tieba/forum/good/:kw/:cid?/:sortBy?', require('./tieba/forum'));
+    router.get('/tieba/forum/:kw/:sortBy?', require('./tieba/forum'));
     router.get('/tieba/post/:id', require('./tieba/post'));
     router.get('/tieba/post/lz/:id', require('./tieba/post'));
     router.get('/tieba/user/:uid', require('./tieba/user'));

lib/v2/baidu/tieba/forum.js

@@ -6,7 +6,8 @@ const { art } = require('@/utils/render');
 const path = require('path');
 
 module.exports = async (ctx) => {
-    const { kw, cid } = ctx.params;
+    // sortBy: created, replied
+    const { kw, cid = '0', sortBy = 'created' } = ctx.params;
 
     // PC端：https://tieba.baidu.com/f?kw=${encodeURIComponent(kw)}
     // 移动端接口：https://tieba.baidu.com/mo/q/m?kw=${encodeURIComponent(kw)}&lp=5024&forum_recommend=1&lm=0&cid=0&has_url_param=1&pn=0&is_ajax=1
@@ -31,7 +32,7 @@ module.exports = async (ctx) => {
         .map((element) => {
             const item = $(element);
             const { id, author_name } = item.data('field');
-            const time = item.find('.threadlist_reply_date').text().trim();
+            const time = sortBy === 'created' ? item.find('.is_show_create_time').text().trim() : item.find('.threadlist_reply_date').text().trim();
             const title = item.find('a.j_th_tit').text().trim();
             const details = item.find('.threadlist_abs').text().trim();
             const medias = item
@@ -50,7 +51,7 @@ module.exports = async (ctx) => {
                     medias,
                     author_name,
                 }),
-                pubDate: timezone(parseDate(time, ['HH:mm', 'M-D'], true), +8),
+                pubDate: timezone(parseDate(time, ['HH:mm', 'M-D', 'YYYY-MM'], true), +8),
                 link: `https://tieba.baidu.com/p/${id}`,
             };
         });

lib/v2/discourse/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:configId/posts': ['dzx-dzx'],
+};

lib/v2/discourse/posts.js

@@ -0,0 +1,28 @@
+const config = require('@/config').value;
+const got = require('@/utils/got');
+const RSSParser = require('@/utils/rss-parser');
+
+module.exports = async (ctx) => {
+    if (!config.discourse.config[ctx.params.configId]) {
+        throw Error('Discourse RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install">relevant config</a>');
+    }
+    const { link, key } = config.discourse.config[ctx.params.configId];
+
+    const feed = await RSSParser.parseString(
+        (
+            await got(`${link}/posts.rss`, {
+                headers: {
+                    'User-Api-Key': key,
+                },
+            })
+        ).data
+    );
+
+    feed.items = feed.items.map((e) => ({
+        description: e.content,
+        author: e.creator,
+        ...e,
+    }));
+
+    ctx.state.data = { item: feed.items, ...feed };
+};

lib/v2/discourse/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:configId/posts', require('./posts'));
+};