feat: crawlFile API adds new parameter types, onBeforeSaveItemFile is no longer restricted

Author: coder-hxl

packages/ai/context.ts

@@ -52,7 +52,7 @@ coder 用户: { message: string, pathMode: string }
 发来了一个 JavaScript 对象转换为 JSON 字符串的值。
 - message:
   - 类型: string,
-  - 作用: 用户的需求求。
+  - 作用: 用户的需求。
 - pathMode:
   - 类型: string, default 或者 strict
   - 作用: default 则可以不从 HTML 片段的根部开始的 selectors , 为 strict 则说明必需从 HTML 片段的根部开始的 selectors 。

packages/crawl/api.ts

@@ -18,7 +18,9 @@ import {
   logSuccess,
   logWarn,
   random,
-  whiteBold
+  whiteBold,
+  isPromise,
+  isBuffer
 } from './utils'
 
 import {
@@ -92,7 +94,7 @@ interface InfoFileConfig extends InfoCommonConfig {
         fileName: string
         filePath: string
         data: Buffer
-      }) => Promise<Buffer>)
+      }) => Promise<Buffer> | Buffer | void)
     | undefined
 }
 
@@ -197,7 +199,7 @@ interface CrawlFileConfig {
         fileName: string
         filePath: string
         data: Buffer
-      }) => Promise<Buffer>)
+      }) => Promise<Buffer> | Buffer | void)
     | undefined
   onCrawlItemComplete:
     | ((crawlDataSingleResult: CrawlDataSingleResult<any>) => void)
@@ -224,8 +226,9 @@ type UniteCrawlDataConfig<T> =
   | CrawlDataAdvancedConfig<T>
 
 type UniteCrawlFileConfig =
+  | string
   | CrawlFileDetailTargetConfig
-  | CrawlFileDetailTargetConfig[]
+  | (string | CrawlFileDetailTargetConfig)[]
   | CrawlFileAdvancedConfig
 
 /* Function */
@@ -273,7 +276,10 @@ function transformTargetToDetailTargets(
     | (string | CrawlDataDetailTargetConfig)[]
 ): CrawlDataDetailTargetConfig[]
 function transformTargetToDetailTargets(
-  config: (string | CrawlFileDetailTargetConfig)[]
+  config:
+    | string
+    | CrawlFileDetailTargetConfig
+    | (string | CrawlFileDetailTargetConfig)[]
 ): CrawlFileDetailTargetConfig[]
 function transformTargetToDetailTargets(config: any) {
   return isArray(config)
@@ -741,10 +747,13 @@ function createCrawlFileConfig(
     advancedDetailTargetsConfig.detailTargets =
       transformTargetToDetailTargets(targets)
   } else {
-    // CrawlFileDetailTargetConfig |  CrawlFileDetailTargetConfig[] 处理
-    advancedDetailTargetsConfig.detailTargets = isArray(originalConfig)
-      ? originalConfig
-      : [originalConfig as CrawlFileDetailTargetConfig]
+    // string | CrawlFileDetailTargetConfig | (string | CrawlFileDetailTargetConfig)[] 处理
+    advancedDetailTargetsConfig.detailTargets = transformTargetToDetailTargets(
+      originalConfig as
+        | string
+        | CrawlFileDetailTargetConfig
+        | (string | CrawlFileDetailTargetConfig)[]
+    )
   }
 
   loaderCommonConfigToCrawlConfig(
@@ -1022,17 +1031,17 @@ function fileSingleResultHandle(
 
     // 在保存前的回调
     const data = detailTargetResult.data
-    let dataPromise = Promise.resolve(data)
+    let onBeforeSaveItemFileResult
     if (onBeforeSaveItemFile) {
-      dataPromise = onBeforeSaveItemFile({
+      onBeforeSaveItemFileResult = onBeforeSaveItemFile({
         id,
         fileName,
         filePath,
         data
       })
     }
 
-    const saveFileItemPending = dataPromise.then(async (newData) => {
+    async function saveFile(newData: Buffer) {
       let isSuccess = true
       try {
         await writeFile(filePath, newData)
@@ -1061,7 +1070,16 @@ function fileSingleResultHandle(
       if (onCrawlItemComplete) {
         onCrawlItemComplete(device.result as CrawlFileSingleResult)
       }
-    })
+    }
+
+    let saveFileItemPending
+    if (isPromise(onBeforeSaveItemFileResult)) {
+      saveFileItemPending = onBeforeSaveItemFileResult!.then(saveFile)
+    } else if (isBuffer(onBeforeSaveItemFileResult)) {
+      saveFileItemPending = saveFile(onBeforeSaveItemFileResult)
+    } else {
+      saveFileItemPending = saveFile(data)
+    }
 
     // 存放保存文件 Promise , 后续等待即可回到 crawlFile 函数内部等待完成即可
     saveFilePendingQueue.push(saveFileItemPending)
@@ -1263,12 +1281,14 @@ export function createCrawlFile(crawlBaseConfig: CrawlBaseConfig) {
   let id = 0
   const type = 'file'
 
+  function crawlFile(config: string): Promise<CrawlFileSingleResult>
+
   function crawlFile(
     config: CrawlFileDetailTargetConfig
   ): Promise<CrawlFileSingleResult>
 
   function crawlFile(
-    config: CrawlFileDetailTargetConfig[]
+    config: (string | CrawlFileDetailTargetConfig)[]
   ): Promise<CrawlFileSingleResult[]>
 
   function crawlFile(

packages/crawl/index.ts

@@ -10,7 +10,7 @@ import { isBoolean, isObject } from './utils'
 
 let id = 0
 
-function createInstanceConfig(config: CreateCrawlConfig): CrawlBaseConfig {
+function createBaseConfig(config: CreateCrawlConfig): CrawlBaseConfig {
   const {
     mode,
     enableRandomFingerprint,
@@ -69,7 +69,7 @@ function createnApp(crawlBaseConfig: CrawlBaseConfig): CrawlApp {
 }
 
 export function createCrawl(config: CreateCrawlConfig = {}): CrawlApp {
-  const crawlBaseConfig = createInstanceConfig(config)
+  const crawlBaseConfig = createBaseConfig(config)
 
   const app = createnApp(crawlBaseConfig)
 

packages/crawl/types/api.ts

@@ -178,7 +178,7 @@ export interface CrawlFileAdvancedConfig extends CrawlCommonConfig {
     fileName: string
     filePath: string
     data: Buffer
-  }) => Promise<Buffer>
+  }) => Promise<Buffer> | Buffer | void
 }
 
 /* API Result */

packages/crawl/types/index.ts

@@ -87,9 +87,13 @@ export interface CrawlApp {
   }
 
   crawlFile: {
+    (config: string): Promise<CrawlFileSingleResult>
+
     (config: CrawlFileDetailTargetConfig): Promise<CrawlFileSingleResult>
 
-    (config: CrawlFileDetailTargetConfig[]): Promise<CrawlFileSingleResult[]>
+    (
+      config: (string | CrawlFileDetailTargetConfig)[]
+    ): Promise<CrawlFileSingleResult[]>
 
     (config: CrawlFileAdvancedConfig): Promise<CrawlFileSingleResult[]>
   }

packages/crawl/utils.ts

@@ -1,3 +1,4 @@
+import { Buffer } from 'node:buffer'
 import chalk from 'chalk'
 
 // Log
@@ -34,6 +35,14 @@ export function isArray(value: any): value is any[] {
   return Array.isArray(value)
 }
 
+export function isPromise(value: any): value is Promise<any> {
+  return typeof value === 'function' && !isUndefined(value.then)
+}
+
+export function isBuffer(value: any): value is Buffer {
+  return Buffer.isBuffer(value)
+}
+
 export function sleep(timeout: number) {
   return new Promise((resolve) => setTimeout(resolve, timeout))
 }