重构: 将 fetch 和 fetchFile 方法集中在默认导出的 XCrawl 类, 提供 baseConfig

Author: coder-hxl

src/index.ts

@@ -1,18 +1,73 @@
-import { fetch } from './service'
+import fs from 'node:fs'
 
-import { IFetchConfig, XCrawlConifg } from './types'
-import { loaderBaseConfig } from './utils'
+import { batchRequest } from './request'
+import { isArray, mergeConfig } from './utils'
+
+import {
+  IFetchConfig,
+  IFetchFileConfig,
+  IRequest,
+  IXCrawlBaseConifg
+} from './types'
 
 export default class XCrawl {
-  baseConfig: XCrawlConifg
+  baseConfig: IXCrawlBaseConifg
 
-  constructor(XCrawlConfig: XCrawlConifg) {
-    this.baseConfig = XCrawlConfig
+  constructor(baseConfig: IXCrawlBaseConifg = {}) {
+    this.baseConfig = baseConfig
   }
 
   async fetch<T = any>(config: IFetchConfig): Promise<T> {
-    const loaderRes = loaderBaseConfig(this.baseConfig, config)
+    const { requestConifg, intervalTime } = mergeConfig(this.baseConfig, config)
+
+    const isRequestConifgArr = isArray(requestConifg)
+    const requestConifgArr = isRequestConifgArr
+      ? requestConifg
+      : [requestConifg]
+
+    const container = [] as T[]
+
+    await batchRequest(requestConifgArr, intervalTime, (requestRes) => {
+      container.push(JSON.parse(requestRes.data.toString()))
+    })
+
+    const res = isRequestConifgArr ? container : container[0]
+    return res as T
+  }
+
+  async fetchFile(config: IFetchFileConfig): Promise<void> {
+    const { requestConifg, intervalTime, fileConfig } = mergeConfig(
+      this.baseConfig,
+      config
+    )
+
+    let successCount = 0
+
+    function eachRequestResHandle(requestRes: IRequest, currentCount: number) {
+      const { headers, data } = requestRes
+
+      const filename = `${new Date().getTime()}.${headers['content-type']
+        ?.split('/')
+        .pop()}`
+      const path = `${fileConfig.storeDir}/${filename}`
+
+      fs.createWriteStream(path, 'binary').write(data, (err) => {
+        if (err) {
+          return console.log(
+            `File save error requested for the ${currentCount}: ${err.message}`
+          )
+        }
+
+        if (++successCount === requestConifgArr.length) {
+          console.log('All files downloaded successfully!')
+        }
+      })
+    }
+
+    const requestConifgArr = isArray(requestConifg)
+      ? requestConifg
+      : [requestConifg]
 
-    return fetch(loaderRes)
+    await batchRequest(requestConifgArr, intervalTime, eachRequestResHandle)
   }
 }

src/request.ts

@@ -0,0 +1,84 @@
+import https from 'node:http'
+
+import { handleConfig, isNumber, isUndefined, random, sleep } from './utils'
+
+import { IIntervalTime, IRequest, IRequestConfig } from './types'
+
+export function request(config: IRequestConfig) {
+  return new Promise<IRequest>((resolve, reject) => {
+    const data = (config.data = config.data
+      ? JSON.stringify(config.data ?? '')
+      : config.data)
+    const requestConfig = handleConfig(config)
+
+    const req = https.request(requestConfig, (res) => {
+      const { headers } = res
+
+      const container: Buffer[] = []
+
+      res.on('data', (chunk) => container.push(chunk))
+
+      res.on('end', () => {
+        const data = Buffer.concat(container)
+        const resolveRes: IRequest = {
+          headers,
+          data
+        }
+
+        resolve(resolveRes)
+      })
+    })
+
+    req.on('timeout', () => {
+      console.log(`Timeout Error`)
+      reject(new Error('Timeout'))
+    })
+
+    req.on('error', (err) => {
+      console.log('Error: ', err.message)
+      reject(err)
+    })
+
+    // 其他处理
+    if (requestConfig.method === 'POST') {
+      req.write(data)
+    }
+
+    req.end()
+  })
+}
+
+export async function batchRequest(
+  requestConifgs: IRequestConfig[],
+  intervalTime: IIntervalTime | undefined,
+  eachRequestResHandle: (requestRes: IRequest, currentCount: number) => any
+) {
+  const total = requestConifgs.length
+  let currentCount = 0
+
+  console.log(`Begin execution, total: ${total} `)
+
+  for (const requestConifg of requestConifgs) {
+    currentCount++
+
+    const requestRes = await request(requestConifg)
+
+    eachRequestResHandle(requestRes, currentCount)
+
+    if (!isUndefined(intervalTime) && currentCount !== total) {
+      const timeout = isNumber(intervalTime)
+        ? intervalTime
+        : random(intervalTime.max, intervalTime.min)
+
+      console.log(
+        `The ${currentCount} request is success, sleep for ${timeout}ms`
+      )
+
+      await sleep(timeout)
+    } else {
+      console.log(
+        `The ${currentCount} request is success, all requests completed!`
+      )
+    }
+  }
+}

src/service.ts

@@ -1,5 +1,4 @@
 import { IncomingHttpHeaders, OutgoingHttpHeaders } from 'node:http'
-import { type } from 'node:os'
 
 export interface IAnyObject extends Object {
   [key: string | number | symbol]: any
@@ -56,7 +55,7 @@ export interface IFetchBaseConifg {
   intervalTime?: IIntervalTime
 }
 
-export interface XCrawlConifg {
+export interface IXCrawlBaseConifg {
   baseUrl?: string
   timeout?: number
   intervalTime?: IIntervalTime

src/types.ts

@@ -6,7 +6,7 @@ import {
   IFetchConfig,
   IMapTypeEmptyObject,
   IRequestConfig,
-  XCrawlConifg
+  IXCrawlBaseConifg
 } from './types'
 
 export function parseParams(urlSearch: string, params?: IAnyObject): string {
@@ -64,19 +64,19 @@ export function handleConfig(
   return config
 }
 
-export function loaderBaseConfig(
-  baseConfig: XCrawlConifg,
-  config: IFetchConfig
-) {
+export function mergeConfig<T extends IFetchConfig>(
+  baseConfig: IXCrawlBaseConifg,
+  config: T
+): T {
   const {
     baseUrl,
     timeout: baseTimeout,
     intervalTime: baseIntervalTime
   } = baseConfig
   const { requestConifg, intervalTime } = config
 
-  const requestConifgArr = Array.isArray(requestConifg)
-    ? [...requestConifg]
+  const requestConifgArr = isArray(requestConifg)
+    ? requestConifg
     : [requestConifg]
 
   for (const requestItem of requestConifgArr) {
@@ -96,10 +96,6 @@ export function loaderBaseConfig(
   return config
 }
 
-export function isUndefined(value: any) {
-  return typeof value === 'undefined'
-}
-
 export function sleep(timeout: number) {
   return new Promise((resolve) => setTimeout(resolve, timeout))
 }
@@ -113,3 +109,15 @@ export function random(max: number, min = 0) {
 
   return res
 }
+
+export function isUndefined(value: any): value is undefined {
+  return typeof value === 'undefined'
+}
+
+export function isNumber(value: any): value is number {
+  return typeof value === 'number'
+}
+
+export function isArray(value: any): value is any[] {
+  return Array.isArray(value)
+}