RequestConfig-like changes to DetailConfig, ConfigObject-like changes to EnhanceConfig. Other property configuration changes. Internal adjustment.

Author: coder-hxl

src/api.ts

@@ -2,9 +2,9 @@ import { isNumber, isUndefined, log, logNumber, random, sleep } from './utils'
 
 import type {
   IntervalTime,
-  LoaderDataRequestConfig,
-  LoaderFileRequestConfig,
-  LoaderPageRequestConfig
+  LoaderCrawlDataDetail,
+  LoaderCrawlFileDetail,
+  LoaderCrawlPageDetail
 } from './types/api'
 import type { ControllerConfig } from './controller'
 
@@ -33,9 +33,9 @@ async function useSleepByBatch(
 
 export async function asyncBatchCrawl<
   T extends
-    | LoaderPageRequestConfig
-    | LoaderDataRequestConfig
-    | LoaderFileRequestConfig,
+    | LoaderCrawlPageDetail
+    | LoaderCrawlDataDetail
+    | LoaderCrawlFileDetail,
   V,
   C
 >(
@@ -87,9 +87,9 @@ export async function asyncBatchCrawl<
 
 export async function syncBatchCrawl<
   T extends
-    | LoaderPageRequestConfig
-    | LoaderDataRequestConfig
-    | LoaderFileRequestConfig,
+    | LoaderCrawlPageDetail
+    | LoaderCrawlDataDetail
+    | LoaderCrawlFileDetail,
   V,
   C
 >(

src/batchCrawlHandle.ts

@@ -2,39 +2,39 @@ import { asyncBatchCrawl, syncBatchCrawl } from './batchCrawlHandle'
 import { priorityQueueMergeSort } from './sort'
 import {
   IntervalTime,
-  LoaderDataRequestConfig,
-  LoaderFileRequestConfig,
-  LoaderPageRequestConfig
+  LoaderCrawlDataDetail,
+  LoaderCrawlFileDetail,
+  LoaderCrawlPageDetail
 } from './types/api'
 import { log, logError, logNumber, logSuccess, logWarn } from './utils'
 
 export interface ControllerConfig<
   T extends
-    | LoaderPageRequestConfig
-    | LoaderDataRequestConfig
-    | LoaderFileRequestConfig,
+    | LoaderCrawlPageDetail
+    | LoaderCrawlDataDetail
+    | LoaderCrawlFileDetail,
   V
 > {
   id: number
   isSuccess: boolean
   crawlCount: number
   maxRetry: number
   errorQueue: Error[]
-  requestConfig: T
+  crawlDetailConfig: T
   crawlSingleRes: V | null
 }
 
 export async function controller<
   T extends
-    | LoaderPageRequestConfig
-    | LoaderDataRequestConfig
-    | LoaderFileRequestConfig,
+    | LoaderCrawlPageDetail
+    | LoaderCrawlDataDetail
+    | LoaderCrawlFileDetail,
   V,
   C
 >(
   name: 'page' | 'data' | 'file',
   mode: 'async' | 'sync',
-  requestConfigs: T[],
+  crawlSingleConfigs: T[],
   intervalTime: IntervalTime | undefined,
   crawlSingleFnExtraConfig: C,
   crawlSingleFn: (
@@ -43,27 +43,27 @@ export async function controller<
   ) => Promise<V>
 ): Promise<ControllerConfig<T, V>[]> {
   // 是否使用优先爬取
-  const isPriorityCrawl = !requestConfigs.every(
-    (item) => item.priority === requestConfigs[0].priority
+  const isPriorityCrawl = !crawlSingleConfigs.every(
+    (item) => item.priority === crawlSingleConfigs[0].priority
   )
   const targetRequestConfigs = isPriorityCrawl
     ? priorityQueueMergeSort(
-        requestConfigs.map((item) => ({
+        crawlSingleConfigs.map((item) => ({
           ...item,
           valueOf: () => item.priority
         }))
       )
-    : requestConfigs
+    : crawlSingleConfigs
 
   // 通过映射生成新的配置数组
   const controllerConfigs: ControllerConfig<T, V>[] = targetRequestConfigs.map(
-    (requestConfig, index) => ({
+    (crawlDetailConfig, index) => ({
       id: index + 1,
       isSuccess: false,
-      maxRetry: requestConfig.maxRetry,
+      maxRetry: crawlDetailConfig.maxRetry,
       crawlCount: 0,
       errorQueue: [],
-      requestConfig,
+      crawlDetailConfig,
       crawlSingleRes: null
     })
   )

src/controller.ts

@@ -5,16 +5,12 @@ import {
   startPolling
 } from './api'
 
-import {
-  LoaderXCrawlBaseConfig,
-  XCrawlConfig,
-  XCrawlInstance
-} from './types'
+import { LoaderXCrawlConfig, XCrawlConfig, XCrawlInstance } from './types'
 import { isUndefined } from './utils'
 
 function loaderBaseConfig(
   baseConfig: XCrawlConfig | undefined
-): LoaderXCrawlBaseConfig {
+): LoaderXCrawlConfig {
   const loaderBaseConfig = baseConfig ? baseConfig : {}
 
   if (!loaderBaseConfig.mode) {
@@ -29,10 +25,10 @@ function loaderBaseConfig(
     loaderBaseConfig.maxRetry = 0
   }
 
-  return loaderBaseConfig as LoaderXCrawlBaseConfig
+  return loaderBaseConfig as LoaderXCrawlConfig
 }
 
-function createnInstance(baseConfig: LoaderXCrawlBaseConfig): XCrawlInstance {
+function createnInstance(baseConfig: LoaderXCrawlConfig): XCrawlInstance {
   const instance: XCrawlInstance = {
     crawlPage: createCrawlPage(baseConfig),
     crawlData: createCrawlData(baseConfig),

src/index.ts

@@ -11,7 +11,7 @@ import HttpsProxyAgent from 'https-proxy-agent'
 import { isUndefined } from './utils'
 
 import { AnyObject, MapTypeEmptyObject } from './types/common'
-import { LoaderDataRequestConfig, LoaderFileRequestConfig } from './types/api'
+import { LoaderCrawlDataDetail, LoaderCrawlFileDetail } from './types/api'
 
 /* Type */
 export interface Request {
@@ -36,7 +36,7 @@ function parseParams(urlSearch: string, params?: AnyObject): string {
 }
 
 function parseHeaders(
-  rawConfig: LoaderDataRequestConfig & LoaderFileRequestConfig,
+  rawConfig: LoaderCrawlDataDetail & LoaderCrawlFileDetail,
   config: RequestOptions & MapTypeEmptyObject<URL>
 ) {
   const rawHeaders = rawConfig.headers ?? {}
@@ -55,7 +55,7 @@ function parseHeaders(
 }
 
 function handleRequestConfig(
-  rawConfig: LoaderDataRequestConfig & LoaderFileRequestConfig
+  rawConfig: LoaderCrawlDataDetail & LoaderCrawlFileDetail
 ): RequestOptions & MapTypeEmptyObject<URL> {
   const { protocol, hostname, port, pathname, search } = new Url.URL(
     rawConfig.url
@@ -85,9 +85,7 @@ function handleRequestConfig(
   return config
 }
 
-export function request(
-  config: LoaderDataRequestConfig & LoaderFileRequestConfig
-) {
+export function request(config: LoaderCrawlDataDetail & LoaderCrawlFileDetail) {
   return new Promise<Request>((resolve, reject) => {
     const isDataUndefine = isUndefined(config.data)
     config.data = !isDataUndefine ? JSON.stringify(config.data) : config.data

src/request.ts

@@ -10,44 +10,44 @@ type LoaderHasConfig = {
   priority: number
 }
 
-export type LoaderPageRequestConfig = PageRequestConfig & LoaderHasConfig
+export type LoaderCrawlPageDetail = CrawlPageDetailConfig & LoaderHasConfig
 
-export type LoaderDataRequestConfig = DataRequestConfig & LoaderHasConfig
+export type LoaderCrawlDataDetail = CrawlDataDetailConfig & LoaderHasConfig
 
-export type LoaderFileRequestConfig = FileRequestConfig & LoaderHasConfig
+export type LoaderCrawlFileDetail = CrawlFileDetailConfig & LoaderHasConfig
 
 export interface LoaderCrawlPageConfig
-  extends MapTypeObject<CrawlPageConfigObject, 'requestConfigs'> {
-  requestConfigs: LoaderPageRequestConfig[]
+  extends MapTypeObject<CrawlPageEnhanceConfig, 'crawlPages'> {
+  crawlPageDetails: LoaderCrawlPageDetail[]
 }
 
 export interface LoaderCrawlDataConfig
-  extends MapTypeObject<CrawlDataConfigObject, 'requestConfigs'> {
-  requestConfigs: LoaderDataRequestConfig[]
+  extends MapTypeObject<CrawlDataEnhanceConfig, 'crawlDatas'> {
+  crawlDataDetails: LoaderCrawlDataDetail[]
 }
 
 export interface LoaderCrawlFileConfig
-  extends MapTypeObject<CrawlFileConfigObject, 'requestConfigs'> {
-  requestConfigs: LoaderFileRequestConfig[]
+  extends MapTypeObject<CrawlFileEnhanceConfig, 'crawlFiles'> {
+  crawlFileDetails: LoaderCrawlFileDetail[]
 }
 
-// Function overloading crawl config
-export type CrawlPageConfig =
+/* Function overloading crawl config */
+export type UniteCrawlPageConfig =
   | string
-  | PageRequestConfig
-  | (string | PageRequestConfig)[]
-  | CrawlPageConfigObject
+  | CrawlPageDetailConfig
+  | (string | CrawlPageDetailConfig)[]
+  | CrawlPageEnhanceConfig
 
-export type CrawlDataConfig =
+export type UniteCrawlDataConfig =
   | string
-  | DataRequestConfig
-  | (string | DataRequestConfig)[]
-  | CrawlDataConfigObject
+  | CrawlDataDetailConfig
+  | (string | CrawlDataDetailConfig)[]
+  | CrawlDataEnhanceConfig
 
-export type CrawlFileConfig =
-  | FileRequestConfig
-  | FileRequestConfig[]
-  | CrawlFileConfigObject
+export type UniteCrawlFileConfig =
+  | CrawlFileDetailConfig
+  | CrawlFileDetailConfig[]
+  | CrawlFileEnhanceConfig
 
 /* API Config */
 // API Config Other
@@ -75,70 +75,64 @@ export type Method =
   | 'unlink'
   | 'UNLINK'
 
-export type PageRequestConfigCookies =
+export type PageCookies =
   | string
   | Protocol.Network.CookieParam
   | Protocol.Network.CookieParam[]
 
-// API Config Request
-export interface PageRequestConfig {
-  url: string
-  headers?: AnyObject
+// API crawl config
+// Common
+export interface CrawlCommonConfig {
   timeout?: number
   proxy?: string
-  cookies?: PageRequestConfigCookies
   maxRetry?: number
+}
+
+// 1.Crawl page config
+export interface CrawlPageDetailConfig extends CrawlCommonConfig {
+  url: string
+  headers?: AnyObject
+  cookies?: PageCookies
   priority?: number
 }
 
-export interface DataRequestConfig {
+export interface CrawlPageEnhanceConfig extends CrawlCommonConfig {
+  crawlPages: (string | CrawlPageDetailConfig)[]
+  intervalTime?: IntervalTime
+
+  // page common attribute
+  cookies?: PageCookies
+}
+
+// 2.Crawl data config
+export interface CrawlDataDetailConfig extends CrawlCommonConfig {
   url: string
   method?: Method
   headers?: AnyObject
   params?: AnyObject
   data?: any
-  timeout?: number
-  proxy?: string
-  maxRetry?: number
   priority?: number
 }
 
-export interface FileRequestConfig {
+export interface CrawlDataEnhanceConfig extends CrawlCommonConfig {
+  crawlDatas: (string | CrawlDataDetailConfig)[]
+  intervalTime?: IntervalTime
+}
+
+// 3.Crawl file config
+export interface CrawlFileDetailConfig extends CrawlCommonConfig {
   url: string
   headers?: AnyObject
-  timeout?: number
-  proxy?: string
-  maxRetry?: number
   priority?: number
   storeDir?: string
   fileName?: string
   extension?: string
 }
 
-// API Config Crawl
-export interface CrawlPageConfigObject {
-  requestConfigs: (string | PageRequestConfig)[]
-  proxy?: string
-  timeout?: number
-  cookies?: PageRequestConfigCookies
-  intervalTime?: IntervalTime
-  maxRetry?: number
-}
-
-export interface CrawlDataConfigObject {
-  requestConfigs: (string | DataRequestConfig)[]
-  proxy?: string
-  timeout?: number
+export interface CrawlFileEnhanceConfig extends CrawlCommonConfig {
+  crawlFiles: (string | CrawlFileDetailConfig)[]
   intervalTime?: IntervalTime
-  maxRetry?: number
-}
 
-export interface CrawlFileConfigObject {
-  requestConfigs: (string | FileRequestConfig)[]
-  proxy?: string
-  timeout?: number
-  intervalTime?: IntervalTime
-  maxRetry?: number
   fileConfig?: {
     storeDir?: string
     extension?: string
@@ -151,6 +145,7 @@ export interface CrawlFileConfigObject {
   }
 }
 
+// 4.Polling config
 export interface StartPollingConfig {
   d?: number
   h?: number