feat(router/36kr): add WAF token handling (#21051)

Author: TonyRL

lib/routes/36kr/hot-list.ts

@@ -4,7 +4,7 @@ import cache from '@/utils/cache';
 import got from '@/utils/got';
 import { parseDate } from '@/utils/parse-date';
 
-import { ProcessItem, rootUrl } from './utils';
+import { getWafTokenId, ProcessItem, rootUrl } from './utils';
 
 const categories = {
     24: {
@@ -70,9 +70,13 @@ async function handler(ctx) {
 
     const currentUrl = category === '24' ? rootUrl : `${rootUrl}/hot-list/catalog`;
 
+    const wafTokenId = await getWafTokenId();
     const response = await got({
         method: 'get',
         url: currentUrl,
+        headers: {
+            Cookie: `_waftokenid=${wafTokenId}`,
+        },
     });
 
     const data = getProperty(JSON.parse(response.data.match(/window.initialState=({.*})/)[1]), categories[category].key);

lib/routes/36kr/utils.ts

@@ -1,21 +1,20 @@
 import { load } from 'cheerio';
 import CryptoJS from 'crypto-js';
 
-import got from '@/utils/got';
+import { solveWafChallenge } from '@/routes/juejin/utils';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
 
-const rootUrl = 'https://www.36kr.com';
+export const rootUrl = 'https://www.36kr.com';
 
-const ProcessItem = (item, tryGet) =>
+export const ProcessItem = (item, tryGet) =>
     tryGet(item.link, async () => {
-        const detailResponse = await got({
-            method: 'get',
-            url: item.link,
-        });
+        const detailResponse = await ofetch(item.link);
 
-        const cipherTextList = detailResponse.data.match(/{"state":"(.*)","isEncrypt":true}/) ?? [];
+        const cipherTextList = detailResponse.match(/{"state":"(.*)","isEncrypt":true}/) ?? [];
 
         if (cipherTextList.length === 0) {
-            const $ = load(detailResponse.body);
+            const $ = load(detailResponse);
             item.description = $('div.articleDetailContent').html();
         } else {
             const key = CryptoJS.enc.Utf8.parse('efabccee-b754-4c');
@@ -33,4 +32,33 @@ const ProcessItem = (item, tryGet) =>
         return item;
     });
 
-export { ProcessItem, rootUrl };
+export const getWafTokenId = () =>
+    cache.tryGet(
+        '36kr:_waftokenid',
+        async () => {
+            const captchaResponse = await ofetch(rootUrl);
+
+            const $ = load(captchaResponse);
+            const payload = $('script')
+                .text()
+                .match(/atob\('(.*?)'\)\),/)?.[1];
+            const response = solveWafChallenge(payload);
+
+            const tokenIdResponse = await ofetch.raw(rootUrl, {
+                headers: {
+                    Cookie: `_wafchallengeid=${response};`,
+                },
+                redirect: 'manual',
+            });
+
+            const _wafTokenId = tokenIdResponse.headers
+                .getSetCookie()
+                .find((cookie) => cookie.startsWith('_waftokenid='))
+                ?.split(';')[0]
+                .split('=')[1];
+
+            return _wafTokenId as string;
+        },
+        300, // server-provided value
+        false
+    );

lib/routes/juejin/utils.ts

@@ -17,7 +17,12 @@ const s256 = (s1: Uint8Array, s2: string) => {
     return sha.digest('hex');
 };
 
-const solveWafChallenge = (cs) => {
+/**
+ * Solve _wafchallengeid
+ * @param cs - base64 encoded challenge string {"v":{"a":"...", "b":"timestamp", "c":"..."}, "s":"..."}
+ * @returns base64 encoded solved challenge string {"v":{"a":"...", "b":"timestamp", "c":"..."}, "s":"...", "d":"solution"}
+ */
+export const solveWafChallenge = (cs: string) => {
     const c = JSON.parse(Buffer.from(cs, 'base64').toString());
     const prefix = b64tou8a(c.v.a);
     const expect = b64tohex(c.v.c);