Add: Allows to set proxy uniformly in crawling API

Author: coder-hxl

src/api.ts

@@ -128,8 +128,13 @@ function loaderCommonConfig(
     }
 
     // 1.3.porxy
-    if (isUndefined(proxy) && !isUndefined(baseConfig.proxy)) {
-      proxy = baseConfig.proxy
+    // requestConfig > loaderConfig > baseConfig
+    if (isUndefined(proxy)) {
+      if (!isUndefined(loaderConfig.proxy)) {
+        proxy = loaderConfig.proxy
+      } else if (!isUndefined(baseConfig.proxy)) {
+        proxy = baseConfig.proxy
+      }
     }
 
     // 1.4.maxRetry
@@ -169,10 +174,11 @@ function loaderPageConfig(
   // requestConfig 统一转成 PageRequestConfig 类型
   if (isObject(rawConfig) && Object.hasOwn(rawConfig, 'requestConfigs')) {
     // CrawlPageConfigObject 处理
-    const { requestConfigs, timeout, cookies, intervalTime, maxRetry } =
+    const { requestConfigs, proxy, timeout, cookies, intervalTime, maxRetry } =
       rawConfig as CrawlPageConfigObject
 
     // 给 loaderConfig 装载 API Config
+    loaderConfig.proxy = proxy
     loaderConfig.cookies = cookies
     loaderConfig.intervalTime = intervalTime
     loaderConfig.maxRetry = maxRetry
@@ -215,10 +221,11 @@ function loaderDataConfig(
   const requestObjecs: DataRequestConfig[] = []
   if (isObject(rawConfig) && Object.hasOwn(rawConfig, 'requestConfigs')) {
     // CrawlDataConfigObject 处理
-    const { requestConfigs, timeout, intervalTime, maxRetry } =
+    const { requestConfigs, proxy, timeout, intervalTime, maxRetry } =
       rawConfig as CrawlDataConfigObject
 
     // 给 loaderConfig 装载 API Config
+    loaderConfig.proxy = proxy
     loaderConfig.intervalTime = intervalTime
     loaderConfig.maxRetry = maxRetry
     loaderConfig.timeout = timeout
@@ -245,6 +252,7 @@ function loaderFileConfig(
 ): LoaderCrawlFileConfig {
   const loaderConfig: LoaderCrawlFileConfig = {
     requestConfigs: [],
+    proxy: rawConfig.proxy,
     timeout: rawConfig.timeout,
     intervalTime: rawConfig.intervalTime,
     maxRetry: rawConfig.maxRetry,

src/types/api.ts

@@ -107,6 +107,7 @@ export type CrawlFileRequestConfig =
 
 export interface CrawlPageConfigObject {
   requestConfigs: (string | PageRequestConfig)[]
+  proxy?: string
   timeout?: number
   cookies?: PageRequestConfigCookies
   intervalTime?: IntervalTime
@@ -115,6 +116,7 @@ export interface CrawlPageConfigObject {
 
 export interface CrawlDataConfigObject {
   requestConfigs: (string | DataRequestConfig)[]
+  proxy?: string
   timeout?: number
   intervalTime?: IntervalTime
   maxRetry?: number
@@ -134,6 +136,7 @@ export type CrawlDataConfig =
 
 export interface CrawlFileConfig<R extends CrawlFileRequestConfig> {
   requestConfig: R
+  proxy?: string
   timeout?: number
   intervalTime?: IntervalTime
   maxRetry?: number

test/environment/crawlData.test.ts

@@ -84,12 +84,12 @@ async function loaderBaseConfig() {
 // 2.2.Loader API Config
 async function loaderAPIConfig() {
   const testXCrawl = xCrawl({
-    baseUrl: 'http://localhost:9001/api',
-    proxy: 'http://localhost:14892'
+    baseUrl: 'http://localhost:9001/api'
   })
 
   const res = await testXCrawl.crawlData({
     requestConfigs: ['/room/193581217', '/room/193581217'],
+    proxy: 'http://localhost:14892',
     timeout: 10000,
     intervalTime: { max: 1000 },
     maxRetry: 0

test/environment/crawlFile.test.ts

@@ -88,12 +88,12 @@ async function loaderBaseConfig() {
 async function loaderAPIConfig() {
   const testXCrawl = xCrawl({
     baseUrl:
-      'https://raw.githubusercontent.com/coder-hxl/airbnb-upload/master/area',
-    proxy: 'http://localhost:14892'
+      'https://raw.githubusercontent.com/coder-hxl/airbnb-upload/master/area'
   })
 
   const res = await testXCrawl.crawlFile({
     requestConfig: ['/4401.jpg', '/4403.jpg'],
+    proxy: 'http://localhost:14892',
     timeout: 10000,
     fileConfig: { storeDir },
     intervalTime: { max: 1000 },

test/environment/crawlPage.test.ts

@@ -91,13 +91,11 @@ async function loaderBaseConfig() {
 
 // 2.2.Loader API Config
 async function loaderAPIConfig() {
-  const testXCrawl = xCrawl({
-    baseUrl: 'https://github.com',
-    proxy: 'http://localhost:14892'
-  })
+  const testXCrawl = xCrawl({ baseUrl: 'https://github.com' })
 
   const res = await testXCrawl.crawlPage({
     requestConfigs: ['/coder-hxl', '/coder-hxl/x-crawl'],
+    proxy: 'http://localhost:14892',
     timeout: 10000,
     intervalTime: { max: 1000 },
     maxRetry: 0