Pending changes exported from your codespace

Author: coder-hxl

README.md

@@ -1,5 +1,5 @@
-# crawler
+# x-crawl
 
-Lightweight nodejs crawler library that can help you crawl data and files.
+XCrawl is a Nodejs crawl library, providing configurations to help you crawl data or files in batches.
 
-轻量级的 nodejs 爬虫库，可以帮助你抓取数据和文件。
+XCrawl 是 Nodejs 爬虫库，提供配置即可帮你批量抓取数据或文件。

package.json

@@ -1,9 +1,9 @@
 {
   "private": true,
-  "name": "crawler",
+  "name": "x-crawler",
   "version": "0.0.1",
   "author": "coderhxl",
-  "description": "爬虫库",
+  "description": "XCrawl is a Nodejs crawl library, providing configurations to help you crawl data or files in batches.",
   "license": "MIT",
   "main": "src/index.ts",
   "scripts": {

publish/README.md

@@ -1 +1,5 @@
-# crawler
+# x-crawl
+
+XCrawl is a Nodejs crawl library, providing configurations to help you crawl data or files in batches.
+
+XCrawl 是 Nodejs 爬虫库，提供配置即可帮你批量抓取数据或文件。

publish/package.json

@@ -1,8 +1,8 @@
 {
-  "name": "crawler",
+  "name": "x-crawler",
   "version": "0.0.1",
   "author": "coderhxl",
-  "description": "爬虫库",
+  "description": "XCrawl is a Nodejs crawl library, providing configurations to help you crawl data or files in batches.",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "",

src/index.ts

@@ -1,10 +1,18 @@
-import { fetch, fetchFile } from './service'
+import { fetch } from './service'
 
-const crawler = {
-  fetch,
-  fetchFile
-}
+import { IFetchConfig, XCrawlConifg } from './types'
+import { loaderBaseConfig } from './utils'
+
+export default class XCrawl {
+  baseConfig: XCrawlConifg
 
-export { fetch, fetchFile }
+  constructor(XCrawlConfig: XCrawlConifg) {
+    this.baseConfig = XCrawlConfig
+  }
 
-export default crawler
+  async fetch<T = any>(config: IFetchConfig): Promise<T> {
+    const loaderRes = loaderBaseConfig(this.baseConfig, config)
+
+    return fetch(loaderRes)
+  }
+}

src/service.ts

@@ -10,9 +10,11 @@ import {
   IRequestConfig
 } from './types'
 
-function request(config: IRequestConfig) {
+export function request(config: IRequestConfig) {
   return new Promise<IRequest>((resolve, reject) => {
-    const data = (config.data = JSON.stringify(config.data ?? ''))
+    const data = (config.data = config.data
+      ? JSON.stringify(config.data ?? '')
+      : config.data)
     const handleConfigRes = handleConfig(config)
 
     const req = https.request(handleConfigRes, (res) => {
@@ -54,30 +56,31 @@ function request(config: IRequestConfig) {
 
 export async function fetch<T = any>(config: IFetchConfig): Promise<T> {
   const { requestConifg, intervalTime } = config
+  const isRequestConifgArr = Array.isArray(requestConifg)
+  const requestConifgArr = isRequestConifgArr ? requestConifg : [requestConifg]
 
-  let res
-  if (Array.isArray(requestConifg)) {
-    res = []
+  const total = requestConifgArr.length
+  let currentCount = 0
 
-    for (const item of requestConifg) {
-      const requestRes = await request(item)
-      res.push(JSON.parse(requestRes.data.toString()))
+  const container = []
 
-      if (typeof intervalTime !== 'undefined') {
-        const timeout =
-          typeof intervalTime === 'number'
-            ? intervalTime
-            : random(intervalTime.max, intervalTime.min)
+  for (const item of requestConifgArr) {
+    currentCount++
 
-        await sleep(timeout)
-      }
+    const requestRes = await request(item)
+    container.push(JSON.parse(requestRes.data.toString()))
+
+    if (typeof intervalTime !== 'undefined' && currentCount !== total) {
+      const timeout =
+        typeof intervalTime === 'number'
+          ? intervalTime
+          : random(intervalTime.max, intervalTime.min)
+
+      await sleep(timeout)
     }
-  } else {
-    const requestRes = await request(requestConifg)
-    res = JSON.parse(requestRes.data.toString())
   }
 
-  return res
+  return isRequestConifgArr ? container : container[0]
 }
 
 export async function fetchFile(config: IFetchFileConfig) {

src/types.ts

@@ -1,4 +1,5 @@
 import { IncomingHttpHeaders, OutgoingHttpHeaders } from 'node:http'
+import { type } from 'node:os'
 
 export interface IAnyObject extends Object {
   [key: string | number | symbol]: any
@@ -48,14 +49,17 @@ export interface IRequestConfig {
   timeout?: number
 }
 
+export type IIntervalTime = number | { max: number; min?: number }
+
 export interface IFetchBaseConifg {
   requestConifg: IRequestConfig | IRequestConfig[]
-  intervalTime?:
-    | number
-    | {
-        max: number
-        min?: number
-      }
+  intervalTime?: IIntervalTime
+}
+
+export interface XCrawlConifg {
+  baseUrl?: string
+  timeout?: number
+  intervalTime?: IIntervalTime
 }
 
 export interface IFetchConfig extends IFetchBaseConifg {}

src/utils.ts

@@ -1,7 +1,13 @@
 import { RequestOptions } from 'http'
-
 import Url, { URL } from 'node:url'
-import { IAnyObject, IMapTypeEmptyObject, IRequestConfig } from './types'
+
+import {
+  IAnyObject,
+  IFetchConfig,
+  IMapTypeEmptyObject,
+  IRequestConfig,
+  XCrawlConifg
+} from './types'
 
 export function parseParams(urlSearch: string, params?: IAnyObject): string {
   let res = urlSearch ? `${urlSearch}` : '?'
@@ -49,14 +55,51 @@ export function handleConfig(
     search: parseParams(search, rawConfig.params),
 
     method: rawConfig.method.toLocaleUpperCase(),
-    headers: {}
+    headers: {},
+    timeout: rawConfig.timeout
   }
 
   config.headers = parseHeaders(rawConfig, config)
 
   return config
 }
 
+export function loaderBaseConfig(
+  baseConfig: XCrawlConifg,
+  config: IFetchConfig
+) {
+  const {
+    baseUrl,
+    timeout: baseTimeout,
+    intervalTime: baseIntervalTime
+  } = baseConfig
+  const { requestConifg, intervalTime } = config
+
+  const requestConifgArr = Array.isArray(requestConifg)
+    ? [...requestConifg]
+    : [requestConifg]
+
+  for (const requestItem of requestConifgArr) {
+    const { url, timeout } = requestItem
+
+    requestItem.url = baseUrl + url
+
+    if (isUndefined(timeout) && !isUndefined(baseTimeout)) {
+      requestItem.timeout = baseTimeout
+    }
+  }
+
+  if (isUndefined(intervalTime) && !isUndefined(baseIntervalTime)) {
+    config.intervalTime = baseIntervalTime
+  }
+
+  return config
+}
+
+export function isUndefined(value: any) {
+  return typeof value === 'undefined'
+}
+
 export function sleep(timeout: number) {
   return new Promise((resolve) => setTimeout(resolve, timeout))
 }