feat: src was renamed packages

Author: coder-hxl

jest.config.js

@@ -5,7 +5,7 @@ module.exports = {
   collectCoverage: true,
   detectOpenHandles: true,
   moduleNameMapper: {
-    '^src/(.*)$': '<rootDir>/src/$1',
+    '^packages/(.*)$': '<rootDir>/packages/$1',
     '^publish/(.*)$': '<rootDir>/publish/$1'
   }
 }

package.json

@@ -16,7 +16,7 @@
     "flexible",
     "multifunction"
   ],
-  "main": "src/index.ts",
+  "main": "packages/index.ts",
   "scripts": {
     "build": "rollup --config rollup.config.mjs",
     "build-dts": "tsc && prettier --write ./publish/src",

src/ai/context.ts renamed to packages/ai/context.ts

@@ -81,7 +81,7 @@ interface XCrawlOpenAIApp {
   custom(): OpenAI
 }
 
-function createXCrawlOpenAI(
+export function createXCrawlOpenAI(
   config: CreateXCrawlOpenAIConfig = {}
 ): XCrawlOpenAIApp {
   const { defaultModel, clientOptions } = config
@@ -200,5 +200,3 @@ function createXCrawlOpenAI(
 
   return app
 }
-
-export { createXCrawlOpenAI }

src/ai/index.ts renamed to packages/ai/index.ts

@@ -38,7 +38,7 @@ import {
   CrawlHTMLDetailTargetConfig,
   CrawlHTMLAdvancedConfig
 } from './types/api'
-import { LogConfig, XCrawlInstanceConfig } from './types'
+import { LogOptions, CrawlBaseConfig } from './types'
 import { fingerprints } from './default'
 
 /* Types */
@@ -49,7 +49,7 @@ export interface InfoCommonConfig {
   serialNumber: string
   mode: 'async' | 'sync'
   type: 'page' | 'html' | 'data' | 'file'
-  logConfig: LogConfig
+  logConfig: LogOptions
   intervalTime: IntervalTime | undefined
 }
 
@@ -403,7 +403,7 @@ function loaderPageFingerprintToDetailTarget(
 }
 
 function loaderCommonConfigToCrawlConfig(
-  xCrawlInstanceConfig: XCrawlInstanceConfig,
+  crawlBaseConfig: CrawlBaseConfig,
   advancedDetailTargetsConfig:
     | CrawlPageAdvancedDetailTargetsConfig
     | CrawlHTMLAdvancedDetailTargetsConfig
@@ -429,16 +429,16 @@ function loaderCommonConfigToCrawlConfig(
         detail
 
       // 1.1.baseUrl
-      if (xCrawlInstanceConfig.baseUrl) {
-        detail.url = xCrawlInstanceConfig.baseUrl + url
+      if (crawlBaseConfig.baseUrl) {
+        detail.url = crawlBaseConfig.baseUrl + url
       }
 
       // 1.2.timeout
       if (isUndefined(timeout)) {
         if (!isUndefined(advancedDetailTargetsConfig.timeout)) {
           detail.timeout = advancedDetailTargetsConfig.timeout ?? undefined
         } else {
-          detail.timeout = xCrawlInstanceConfig.timeout
+          detail.timeout = crawlBaseConfig.timeout
         }
       }
 
@@ -447,16 +447,16 @@ function loaderCommonConfigToCrawlConfig(
         if (!isUndefined(advancedDetailTargetsConfig.maxRetry)) {
           detail.maxRetry = advancedDetailTargetsConfig.maxRetry ?? 0
         } else {
-          detail.maxRetry = xCrawlInstanceConfig.maxRetry
+          detail.maxRetry = crawlBaseConfig.maxRetry
         }
       }
 
       // 1.4.proxy
       if (isUndefined(proxy)) {
         if (!isUndefined(advancedDetailTargetsConfig.proxy)) {
           detail.proxy = advancedDetailTargetsConfig.proxy
-        } else if (!isUndefined(xCrawlInstanceConfig.proxy)) {
-          detail.proxy = xCrawlInstanceConfig.proxy
+        } else if (!isUndefined(crawlBaseConfig.proxy)) {
+          detail.proxy = crawlBaseConfig.proxy
         }
       }
 
@@ -503,9 +503,9 @@ function loaderCommonConfigToCrawlConfig(
       } else if (
         isUndefined(fingerprint) &&
         !isArray(advancedDetailTargetsConfig.fingerprints) &&
-        xCrawlInstanceConfig.enableRandomFingerprint
+        crawlBaseConfig.enableRandomFingerprint
       ) {
-        // xCrawlInstanceConfig
+        // crawlBaseConfig
         const fingerprint = fingerprints[random(fingerprints.length)]
 
         loaderCommonFingerprintToDetailTarget(detail, fingerprint)
@@ -519,9 +519,9 @@ function loaderCommonConfigToCrawlConfig(
   crawlConfig.intervalTime = advancedDetailTargetsConfig.intervalTime
   if (
     isUndefined(advancedDetailTargetsConfig.intervalTime) &&
-    !isUndefined(xCrawlInstanceConfig.intervalTime)
+    !isUndefined(crawlBaseConfig.intervalTime)
   ) {
-    crawlConfig.intervalTime = xCrawlInstanceConfig.intervalTime
+    crawlConfig.intervalTime = crawlBaseConfig.intervalTime
   }
 
   // 3.onCrawlItemComplete
@@ -540,7 +540,7 @@ function loaderCommonConfigToCrawlConfig(
 */
 
 function createCrawlPageConfig(
-  xCrawlInstanceConfig: XCrawlInstanceConfig,
+  crawlBaseConfig: CrawlBaseConfig,
   originalConfig: UniteCrawlPageConfig
 ): CrawlPageConfig {
   const crawlPageConfig: CrawlPageConfig = {
@@ -577,7 +577,7 @@ function createCrawlPageConfig(
 
   // 装载公共配置
   loaderCommonConfigToCrawlConfig(
-    xCrawlInstanceConfig,
+    crawlBaseConfig,
     advancedDetailTargetsConfig,
     crawlPageConfig
   )
@@ -618,7 +618,7 @@ function createCrawlPageConfig(
 }
 
 function createCrawlHTMLConfig(
-  xCrawlInstanceConfig: XCrawlInstanceConfig,
+  crawlBaseConfig: CrawlBaseConfig,
   originalConfig: UniteCrawlHTMLConfig
 ): CrawlHTMLConfig {
   const crawlHTMLConfig: CrawlHTMLConfig = {
@@ -658,7 +658,7 @@ function createCrawlHTMLConfig(
   }
 
   loaderCommonConfigToCrawlConfig(
-    xCrawlInstanceConfig,
+    crawlBaseConfig,
     advancedDetailTargetsConfig,
     crawlHTMLConfig
   )
@@ -667,7 +667,7 @@ function createCrawlHTMLConfig(
 }
 
 function createCrawlDataConfig<T>(
-  xCrawlInstanceConfig: XCrawlInstanceConfig,
+  crawlBaseConfig: CrawlBaseConfig,
   originalConfig: UniteCrawlDataConfig<T>
 ): CrawlDataConfig {
   const crawlDataConfig: CrawlDataConfig = {
@@ -704,7 +704,7 @@ function createCrawlDataConfig<T>(
   }
 
   loaderCommonConfigToCrawlConfig(
-    xCrawlInstanceConfig,
+    crawlBaseConfig,
     advancedDetailTargetsConfig,
     crawlDataConfig
   )
@@ -713,7 +713,7 @@ function createCrawlDataConfig<T>(
 }
 
 function createCrawlFileConfig(
-  xCrawlInstanceConfig: XCrawlInstanceConfig,
+  crawlBaseConfig: CrawlBaseConfig,
   originalConfig: UniteCrawlFileConfig
 ): CrawlFileConfig {
   const crawlFileConfig: CrawlFileConfig = {
@@ -748,7 +748,7 @@ function createCrawlFileConfig(
   }
 
   loaderCommonConfigToCrawlConfig(
-    xCrawlInstanceConfig,
+    crawlBaseConfig,
     advancedDetailTargetsConfig,
     crawlFileConfig
   )
@@ -1074,13 +1074,13 @@ function fileSingleResultHandle(
 
 /* Create crawl API */
 
-export function createCrawlPage(xCrawlInstanceConfig: XCrawlInstanceConfig) {
+export function createCrawlPage(crawlBaseConfig: CrawlBaseConfig) {
   const {
     id: xId,
     mode,
-    logConfig,
-    crawlPage: crawlPageConfig
-  } = xCrawlInstanceConfig
+    logOptions: logConfig,
+    crawlPage: crawlPageOptions
+  } = crawlBaseConfig
 
   let id = 0
   let browser: Browser | null = null
@@ -1109,7 +1109,7 @@ export function createCrawlPage(xCrawlInstanceConfig: XCrawlInstanceConfig) {
     if (!haveCreateBrowser) {
       haveCreateBrowser = true
       createBrowserPending = puppeteer
-        .launch(crawlPageConfig?.puppeteerLaunch)
+        .launch(crawlPageOptions?.puppeteerLaunchOptions)
         .then((result) => {
           browser = result
         })
@@ -1124,7 +1124,7 @@ export function createCrawlPage(xCrawlInstanceConfig: XCrawlInstanceConfig) {
 
     // 创建新配置
     const { detailTargets, intervalTime, onCrawlItemComplete } =
-      createCrawlPageConfig(xCrawlInstanceConfig, config)
+      createCrawlPageConfig(crawlBaseConfig, config)
 
     const infoConfig: InfoPageConfig = {
       serialNumber: `${xId}-${type}-${++id}`,
@@ -1154,8 +1154,8 @@ export function createCrawlPage(xCrawlInstanceConfig: XCrawlInstanceConfig) {
   return crawlPage
 }
 
-export function createCrawlHTML(xCrawlInstanceConfig: XCrawlInstanceConfig) {
-  const { id: xId, mode, logConfig } = xCrawlInstanceConfig
+export function createCrawlHTML(crawlBaseConfig: CrawlBaseConfig) {
+  const { id: xId, mode, logOptions: logConfig } = crawlBaseConfig
   let id = 0
   const type = 'html'
 
@@ -1177,7 +1177,7 @@ export function createCrawlHTML(xCrawlInstanceConfig: XCrawlInstanceConfig) {
     config: UniteCrawlHTMLConfig
   ): Promise<CrawlHTMLSingleResult | CrawlHTMLSingleResult[]> {
     const { detailTargets, intervalTime, onCrawlItemComplete } =
-      createCrawlHTMLConfig(xCrawlInstanceConfig, config)
+      createCrawlHTMLConfig(crawlBaseConfig, config)
 
     const infoConfig: InfoHTMLConfig = {
       serialNumber: `${xId}-${type}-${++id}`,
@@ -1206,8 +1206,8 @@ export function createCrawlHTML(xCrawlInstanceConfig: XCrawlInstanceConfig) {
   return crawlHTML
 }
 
-export function createCrawlData(xCrawlInstanceConfig: XCrawlInstanceConfig) {
-  const { id: xId, mode, logConfig } = xCrawlInstanceConfig
+export function createCrawlData(crawlBaseConfig: CrawlBaseConfig) {
+  const { id: xId, mode, logOptions: logConfig } = crawlBaseConfig
   let id = 0
   const type = 'data'
 
@@ -1229,7 +1229,7 @@ export function createCrawlData(xCrawlInstanceConfig: XCrawlInstanceConfig) {
     config: UniteCrawlDataConfig<T>
   ): Promise<CrawlDataSingleResult<T> | CrawlDataSingleResult<T>[]> {
     const { detailTargets, intervalTime, onCrawlItemComplete } =
-      createCrawlDataConfig(xCrawlInstanceConfig, config)
+      createCrawlDataConfig(crawlBaseConfig, config)
 
     const infoConfig: InfoDataConfig<T> = {
       serialNumber: `${xId}-${type}-${++id}`,
@@ -1258,8 +1258,8 @@ export function createCrawlData(xCrawlInstanceConfig: XCrawlInstanceConfig) {
   return crawlData
 }
 
-export function createCrawlFile(xCrawlInstanceConfig: XCrawlInstanceConfig) {
-  const { id: xId, mode, logConfig } = xCrawlInstanceConfig
+export function createCrawlFile(crawlBaseConfig: CrawlBaseConfig) {
+  const { id: xId, mode, logOptions: logConfig } = crawlBaseConfig
   let id = 0
   const type = 'file'
 
@@ -1283,7 +1283,7 @@ export function createCrawlFile(xCrawlInstanceConfig: XCrawlInstanceConfig) {
       intervalTime,
       onBeforeSaveItemFile,
       onCrawlItemComplete
-    } = createCrawlFileConfig(xCrawlInstanceConfig, config)
+    } = createCrawlFileConfig(crawlBaseConfig, config)
 
     const infoConfig: InfoFileConfig = {
       serialNumber: `${xId}-${type}-${++id}`,

src/api.ts renamed to packages/crawl/api.ts

@@ -0,0 +1,77 @@
+import {
+  createCrawlData,
+  createCrawlFile,
+  createCrawlHTML,
+  createCrawlPage
+} from './api'
+
+import { CreateCrawlConfig, CrawlApp, CrawlBaseConfig } from './types'
+import { isBoolean, isObject } from './utils'
+
+let id = 0
+
+function createInstanceConfig(config: CreateCrawlConfig): CrawlBaseConfig {
+  const {
+    mode,
+    enableRandomFingerprint,
+    baseUrl,
+    intervalTime,
+    log,
+    crawlPage,
+    timeout,
+    proxy,
+    maxRetry
+  } = config
+
+  const crawlBaseConfig: CrawlBaseConfig = {
+    id: ++id,
+
+    mode: mode ?? 'async',
+    enableRandomFingerprint: enableRandomFingerprint ?? false,
+    timeout: timeout ?? 10000,
+    maxRetry: maxRetry ?? 0,
+    logOptions: { start: true, process: true, result: true },
+
+    baseUrl,
+    intervalTime,
+    proxy,
+    crawlPage
+  }
+
+  // logOptions
+  if (isObject(log)) {
+    crawlBaseConfig.logOptions = {
+      ...crawlBaseConfig.logOptions,
+      ...log
+    }
+  } else if (isBoolean(log) && !log) {
+    const keys = Object.keys(crawlBaseConfig.logOptions) as [
+      'start',
+      'process',
+      'result'
+    ]
+
+    keys.forEach((key) => (crawlBaseConfig.logOptions[key] = false))
+  }
+
+  return crawlBaseConfig
+}
+
+function createnApp(crawlBaseConfig: CrawlBaseConfig): CrawlApp {
+  const app: CrawlApp = {
+    crawlPage: createCrawlPage(crawlBaseConfig),
+    crawlHTML: createCrawlHTML(crawlBaseConfig),
+    crawlData: createCrawlData(crawlBaseConfig),
+    crawlFile: createCrawlFile(crawlBaseConfig)
+  }
+
+  return app
+}
+
+export function createCrawl(config: CreateCrawlConfig = {}): CrawlApp {
+  const crawlBaseConfig = createInstanceConfig(config)
+
+  const app = createnApp(crawlBaseConfig)
+
+  return app
+}