feat(xiaohongshu): add proxy configuration support (#20952)

Allow xiaohongshu routes to bypass anti-scraping measures by routing requests through a configurable proxy. When XIAOHONGSHU_PROXY is set, all xiaohongshu HTTP requests (user, board, notes) will be routed through the proxy endpoint instead of using Puppeteer.

Co-authored-by: Claude Haiku 4.5 <[email protected]>

Author: DIYgod

lib/config.ts

@@ -219,6 +219,7 @@ type ConfigEnvKeys =
     | 'XIAOYUZHOU_ID'
     | 'XIAOYUZHOU_TOKEN'
     | 'XIAOHONGSHU_COOKIE'
+    | 'XIAOHONGSHU_PROXY'
     | 'XIMALAYA_TOKEN'
     | 'XSIJISHE_COOKIE'
     | 'XSIJISHE_USER_AGENT'
@@ -637,6 +638,7 @@ export type Config = {
     };
     xiaohongshu: {
         cookie?: string;
+        proxy?: string;
     };
     ximalaya: {
         token?: string;
@@ -1115,6 +1117,7 @@ const calculateValue = () => {
         },
         xiaohongshu: {
             cookie: envs.XIAOHONGSHU_COOKIE,
+            proxy: envs.XIAOHONGSHU_PROXY,
         },
         ximalaya: {
             token: envs.XIMALAYA_TOKEN,

lib/routes/xiaohongshu/util.ts

@@ -29,10 +29,40 @@ const getHeaders = (cookie?: string) => ({
     ...(cookie ? { Cookie: cookie } : {}),
 });
 
+// Fetch HTML through proxy when configured
+async function fetchWithProxy(url: string, cookie?: string): Promise<string> {
+    const proxy = config.xiaohongshu.proxy;
+    if (proxy) {
+        const proxyUrl = `${proxy}?url=${encodeURIComponent(url)}`;
+        logger.http(`Requesting ${url} via proxy`);
+        return await ofetch(proxyUrl, { parseResponse: (txt) => txt });
+    }
+    logger.http(`Requesting ${url}`);
+    return await ofetch(url, {
+        headers: getHeaders(cookie),
+    });
+}
+
 const getUser = (url, cache) =>
     cache.tryGet(
         url,
         async () => {
+            // Use proxy if configured
+            if (config.xiaohongshu.proxy) {
+                const res = await fetchWithProxy(url);
+                const $ = load(res);
+                const script = extractInitialState($);
+                const state = JSON.parse(script);
+
+                let { userPageData, notes } = state.user;
+                userPageData = userPageData._rawValue || userPageData;
+                notes = notes._rawValue || notes;
+
+                // Cannot get collect data without puppeteer
+                return { userPageData, notes, collect: '' };
+            }
+
+            // Use puppeteer
             const { page, destory } = await getPuppeteerPage(url, {
                 onBeforeLoad: async (page) => {
                     await page.setRequestInterception(true);
@@ -88,6 +118,16 @@ const getBoard = (url, cache) =>
     cache.tryGet(
         url,
         async () => {
+            // Use proxy if configured
+            if (config.xiaohongshu.proxy) {
+                const res = await fetchWithProxy(url);
+                const $ = load(res);
+                const script = extractInitialSsrState($);
+                const state = JSON.parse(script);
+                return state.Main;
+            }
+
+            // Use puppeteer
             const browser = await puppeteer();
             try {
                 const page = await browser.newPage();
@@ -156,9 +196,7 @@ async function renderNotesFulltext(notes, urlPrex, displayLivePhoto) {
 
 async function getFullNote(link, displayLivePhoto) {
     const data = (await cache.tryGet(link, async () => {
-        const res = await ofetch(link, {
-            headers: getHeaders(config.xiaohongshu.cookie),
-        });
+        const res = await fetchWithProxy(link, config.xiaohongshu.cookie);
         const $ = load(res);
         const script = extractInitialState($);
         const state = JSON.parse(script);
@@ -244,9 +282,7 @@ async function getFullNote(link, displayLivePhoto) {
 
 async function getUserWithCookie(url: string) {
     const cookie = config.xiaohongshu.cookie;
-    const res = await ofetch(url, {
-        headers: getHeaders(cookie),
-    });
+    const res = await fetchWithProxy(url, cookie);
     const $ = load(res);
     const paths = $('#userPostedFeeds > section > div > a.cover.ld.mask').map((i, item) => item.attributes[3].value);
     const script = extractInitialState($);
@@ -275,6 +311,29 @@ function extractInitialState($) {
     return script;
 }
 
+// Add helper function to extract initial SSR state
+function extractInitialSsrState($) {
+    let script = $('script')
+        .filter((i, script) => {
+            const text = script.children[0]?.data;
+            return text?.includes('window.__INITIAL_SSR_STATE__=');
+        })
+        .text();
+    const match = script.match(/window\.__INITIAL_SSR_STATE__\s*=\s*(\{[\s\S]*?\})\s*(?:;|$)/);
+    if (match) {
+        return match[1].replaceAll('undefined', 'null');
+    }
+    // Fallback: try simple extraction
+    const startMarker = 'window.__INITIAL_SSR_STATE__=';
+    const startIndex = script.indexOf(startMarker);
+    if (startIndex !== -1) {
+        script = script.slice(startIndex + startMarker.length);
+        script = script.replaceAll('undefined', 'null');
+        return script;
+    }
+    throw new Error('Cannot extract __INITIAL_SSR_STATE__');
+}
+
 async function checkCookie() {
     const cookie = config.xiaohongshu.cookie;
     const res = await ofetch('https://edith.xiaohongshu.com/api/sns/web/v2/user/me', {