feat: 优化百度的结果抓取

Author: muzea

.github/workflows/publish-npm.yml

@@ -20,6 +20,8 @@ jobs:
           registry-url: 'https://registry.npmjs.org'
       - run: |
           pnpm i
+          cp README.md packages/browser/README.md
+          cp README.md packages/mcp/README.md
           cd packages/browser && pnpm build && pnpm publish
           cd ../mcp && pnpm build && pnpm publish
         env:

packages/browser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiny-tool/browser",
-  "version": "1.0.0-beta.6",
+  "version": "1.0.0-beta.7",
   "repository": {
     "url": "https://github.com/tiny-tool/browser"
   },
@@ -43,7 +43,8 @@
     "remark-gfm": "^4.0.1",
     "remark-stringify": "^11.0.0",
     "unified": "^11.0.5",
-    "unist-util-visit": "^5.0.0"
+    "unist-util-visit": "^5.0.0",
+    "uuid": "^11.1.0"
   },
   "devDependencies": {
     "@types/hast": "^3.0.4",

packages/browser/src/app.ts

@@ -1,4 +1,5 @@
 import express from 'express';
+import { v4 as uuidv4 } from 'uuid';
 import { SearchAgent } from './search-agent';
 
 const app = express();
@@ -32,12 +33,15 @@ app.get('/api/v1/search', async (req, res) => {
   console.log('req.query', req.query);
   res.send(
     await agent.search(engine as string, decodeURIComponent(q as string), payload, {
-      sessionId: req.query.sessionId as string,
+      sessionId: (req.query.sessionId as string) || uuidv4(),
     }),
   );
 });
 
 app.listen(port, () => {
   console.log(`app listening on port ${port}`);
-  agent = new SearchAgent();
+  agent = new SearchAgent({
+    headless: false,
+    log: './logs',
+  });
 });

packages/browser/src/baidu/index.ts

@@ -1,9 +1,19 @@
-import { Page } from 'puppeteer-core';
+import { Page, ElementHandle } from 'puppeteer-core';
 import { OrganicResult, Result, SearchOptions } from '../define';
 import { getRedirectUrl } from '../common/get-redirect-url';
 import { EventType, LogContext } from '../logs';
 import { resolveUrlAndContent } from '../common/get-page-content';
 
+async function findElement(selectorList: string[], el: ElementHandle<Element>): Promise<ElementHandle | null> {
+  for (const selector of selectorList) {
+    const element = await el.$(selector);
+    if (element) {
+      return element;
+    }
+  }
+  return null;
+}
+
 async function extractOrganicResults(page: Page): Promise<OrganicResult[]> {
   const resultList = await page.$$('.result.c-container.new-pmd');
 
@@ -13,7 +23,12 @@ async function extractOrganicResults(page: Page): Promise<OrganicResult[]> {
   for (const el of resultList) {
     pos++;
     let currentPosition = pos;
-    const title = await el.$('.c-title');
+
+    // 提取标题
+    // 百度可能会调整结构，这里是可能的几个选择器
+    const titleSelectorList = ['.c-title', 'a[data-module="title"]', 'h3[class^="struct-title_"]', 'div[class^="title-box_"]', 'div[class^="title-wrapper_"]'];
+
+    const title = await findElement(titleSelectorList, el);
     if (!title) {
       continue;
     }
@@ -23,13 +38,19 @@ async function extractOrganicResults(page: Page): Promise<OrganicResult[]> {
     }
 
     // const site = await el.$('div[class^="source_"] a[class^="siteLink_"]');
-    const site = await el.$('a[class^="siteLink_"]');
+    // 提取站点名称
+    const siteSelectorList = ['a[class^="siteLink_"]', '.cosc-source-link', '.cosc-source-text'];
+    const site = await findElement(siteSelectorList, el);
     const siteContent = site ? await site.evaluate((el) => el.textContent) : '';
 
-    const link = (await el.$('.c-title a'))!;
+    // 结果对应的跳转链接
+    const titleIsLink = await title.evaluate((el) => el.tagName.toLowerCase() === 'a');
+    const link = titleIsLink ? (title as ElementHandle<HTMLAnchorElement>) : (await title.$('a'))!;
     const linkContent = await link.evaluate((el) => el.href);
 
-    const content = (await el.$('span[class^="content-right"]'))!;
+    // 页面展示的内容
+    const contentSelectorList = ['span[class^="content-right"]', 'div[data-module="abstract"]', 'span[class^="summary-text_"]'];
+    const content = (await findElement(contentSelectorList, el))!;
     const contentContent = content ? await content.evaluate((el) => el.textContent) : '';
 
     ret.push({

packages/browser/src/util.ts

@@ -0,0 +1,3 @@
+export async function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

packages/mcp/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiny-tool/browser-mcp",
-  "version": "1.0.0-beta.6",
+  "version": "1.0.0-beta.7",
   "bin": "dist/index.js",
   "repository": {
     "url": "https://github.com/tiny-tool/browser"