取消fileConfig的suffix选项, intervalTime可以为number, log打印提示, 细节修改

Author: coder-hxl

src/index.ts

@@ -1 +1,10 @@
-export { fetch, fetchFile } from './service'
+import { fetch, fetchFile } from './service'
+
+const crawler = {
+  fetch,
+  fetchFile
+}
+
+export { fetch, fetchFile }
+
+export default crawler

src/service.ts

@@ -3,31 +3,46 @@ import fs from 'node:fs'
 
 import { handleConfig, random, sleep } from './utils'
 
-import { IFetchConfig, IFetchFileConfig, IRequestConfig } from './types'
+import {
+  IFetchConfig,
+  IFetchFileConfig,
+  IRequest,
+  IRequestConfig
+} from './types'
 
 function request(config: IRequestConfig) {
-  return new Promise<Buffer>((resolve, reject) => {
+  return new Promise<IRequest>((resolve, reject) => {
     const data = (config.data = JSON.stringify(config.data ?? ''))
-    const requestConfig = handleConfig(config)
-    // console.log('requestConfig: ', requestConfig)
+    const handleConfigRes = handleConfig(config)
 
-    const req = https.request(requestConfig, (res) => {
-      const content: Buffer[] = []
+    const req = https.request(handleConfigRes, (res) => {
+      const container: Buffer[] = []
 
-      res.on('data', (chunk) => content.push(chunk))
+      res.on('data', (chunk) => container.push(chunk))
 
       res.on('end', () => {
-        resolve(Buffer.concat(content))
+        const data = Buffer.concat(container)
+        const resolveRes: IRequest = {
+          contentType: res.headers['content-type'],
+          contentLength: res.headers['content-length'],
+          data
+        }
+        resolve(resolveRes)
       })
     })
 
+    req.on('timeout', () => {
+      console.log(`Timeout Error`)
+      reject(new Error('Timeout'))
+    })
+
     req.on('error', (err) => {
-      console.log('err: ', err.message)
+      console.log('Error: ', err.message)
       reject(err)
     })
 
-    if (requestConfig.method.toLowerCase() === 'post') {
-      // console.log('requestConfig data: ', data)
+    // 其他处理
+    if (handleConfigRes.method === 'POST') {
       req.write(data)
     }
 
@@ -37,27 +52,27 @@ function request(config: IRequestConfig) {
 
 export async function fetch<T = any>(config: IFetchConfig): Promise<T> {
   const { requestConifg, intervalTime } = config
-  let res
-
-  async function getValue(req: IRequestConfig) {
-    const bufferRes = await request(req)
-    return JSON.parse(bufferRes.toString())
-  }
 
+  let res
   if (Array.isArray(requestConifg)) {
     res = []
 
     for (const item of requestConifg) {
-      const value = await getValue(item)
-      res.push(value)
+      const requestRes = await request(item)
+      res.push(JSON.parse(requestRes.data.toString()))
+
+      if (typeof intervalTime !== 'undefined') {
+        const timeout =
+          typeof intervalTime === 'number'
+            ? intervalTime
+            : random(intervalTime.max, intervalTime.min)
 
-      if (intervalTime) {
-        const timeout = random(intervalTime.max, intervalTime.min)
         await sleep(timeout)
       }
     }
   } else {
-    res = getValue(requestConifg)
+    const requestRes = await request(requestConifg)
+    res = JSON.parse(requestRes.data.toString())
   }
 
   return res
@@ -66,29 +81,52 @@ export async function fetch<T = any>(config: IFetchConfig): Promise<T> {
 export async function fetchFile(config: IFetchFileConfig) {
   const { requestConifg, intervalTime, fileConfig } = config
 
-  const isRqeArr = Array.isArray(requestConifg)
-  const requestConifgArr = isRqeArr ? requestConifg : [requestConifg]
+  const requestConifgArr = Array.isArray(requestConifg)
+    ? requestConifg
+    : [requestConifg]
 
-  const sum = requestConifgArr.length
+  const total = requestConifgArr.length
   let currentCount = 0
+  let successCount = 0
+
+  await Promise.resolve()
 
-  console.log(`开始下载, 总数: ${sum} `)
+  console.log(`Start downloading, total: ${total} `)
 
   for (const item of requestConifgArr) {
     currentCount++
 
-    const res = await request(item)
-    const filename = new Date().getTime() + fileConfig.suffix
-    const path = fileConfig.storeDir + '/' + filename
+    const requestRes = await request(item)
 
-    fs.createWriteStream(path, 'binary').write(res)
-    console.log(`当前: ${currentCount}`)
+    const { contentType, data } = requestRes
+    const filename = `${new Date().getTime()}.${contentType?.split('/').pop()}`
+    const path = `${fileConfig.storeDir}/${filename}`
+
+    fs.createWriteStream(path, 'binary').write(data, (err) => {
+      if (err) {
+        return console.log(
+          `File save error requested for the ${currentCount}: ${err.message}`
+        )
+      }
+
+      if (++successCount === total) {
+        console.log('All files downloaded successfully!')
+      }
+    })
+
+    if (typeof intervalTime !== 'undefined' && currentCount !== total) {
+      const timeout =
+        typeof intervalTime === 'number'
+          ? intervalTime
+          : random(intervalTime.max, intervalTime.min)
+
+      console.log(
+        `The ${currentCount} request is success, sleep for ${timeout}ms`
+      )
 
-    if (intervalTime && isRqeArr && currentCount !== sum) {
-      const timeout = random(intervalTime.max, intervalTime.min)
       await sleep(timeout)
+    } else {
+      console.log(`The ${currentCount} request is success`)
     }
   }
-
-  console.log('下载完成')
 }

src/types.ts

@@ -1,3 +1,15 @@
+export interface IAnyObject extends Object {
+  [key: string | number | symbol]: any
+}
+
+export type IMapTypeObject<T extends object, E extends string = ''> = {
+  [P in keyof T as Exclude<P, E>]: T[P]
+}
+
+export type IMapTypeEmptyObject<T extends object, E extends string = ''> = {
+  [P in keyof T as Exclude<P, E>]?: T[P]
+}
+
 export type IMethod =
   | 'get'
   | 'GET'
@@ -20,30 +32,37 @@ export type IMethod =
   | 'unlink'
   | 'UNLINK'
 
-export interface IRequestConfig<T = any> {
+export interface IRequestConfig {
   url: string
   method: IMethod
   headers?: {
     [key: string]: number | string | string[]
   }
-  params?: any
-  data?: T
+  params?: IAnyObject
+  data?: any
   timeout?: number
 }
 
-export interface IBaseConifg {
+export interface IFetchBaseConifg {
   requestConifg: IRequestConfig | IRequestConfig[]
-  intervalTime?: {
-    max: number
-    min?: number
-  }
+  intervalTime?:
+    | number
+    | {
+        max: number
+        min?: number
+      }
 }
 
-export type IFetchConfig = IBaseConifg
+export type IFetchConfig = IFetchBaseConifg
 
-export interface IFetchFileConfig extends IBaseConifg {
+export interface IFetchFileConfig extends IFetchBaseConifg {
   fileConfig: {
     storeDir: string
-    suffix: string
   }
 }
+
+export interface IRequest {
+  contentType: string | undefined
+  contentLength: string | undefined
+  data: Buffer
+}

src/utils.ts

@@ -1,44 +1,60 @@
-import Url from 'node:url'
-import { IRequestConfig } from './types'
+import { RequestOptions } from 'https'
 
-export function parseParams(params: any): string {
-  let res = ''
+import Url, { URL } from 'node:url'
+import { IAnyObject, IMapTypeEmptyObject, IRequestConfig } from './types'
 
-  if (typeof params === 'object') {
+export function parseParams(urlSearch: string, params?: IAnyObject): string {
+  let res = urlSearch ? `${urlSearch}` : '?'
+
+  if (params) {
     for (const key in params) {
       const value = params[key]
-      res += `${key}=${value}`
+      res += `&${key}=${value}`
     }
   } else {
-    res = params
+    res = urlSearch
   }
 
   return res
 }
 
-export function handleConfig(rawConfig: IRequestConfig) {
+export function parseHeaders(
+  rawConfig: IRequestConfig,
+  config: RequestOptions & IMapTypeEmptyObject<URL>
+) {
+  let headers
+
+  if (config.method === 'POST' && rawConfig.data) {
+    headers = {
+      ...rawConfig.headers,
+      'Content-Type': 'application/json',
+      'Content-Length': Buffer.byteLength(rawConfig.data)
+    }
+  } else {
+    headers = rawConfig.headers
+  }
+
+  return headers
+}
+
+export function handleConfig(
+  rawConfig: IRequestConfig
+): RequestOptions & IMapTypeEmptyObject<URL> {
   const { hostname, port, pathname, search } = new Url.URL(rawConfig.url)
-  const params = search ? search : parseParams(rawConfig.params)
-  const headers =
-    rawConfig.method.toLowerCase() === 'post' && rawConfig.data
-      ? {
-          ...rawConfig.headers,
-          'Content-Type': 'application/json',
-          'Content-Length': Buffer.byteLength(rawConfig.data)
-        }
-      : rawConfig.headers
-
-  const requestConfig = {
+
+  const config: RequestOptions & IMapTypeEmptyObject<URL> = {
     hostname,
     port,
     path: pathname,
-    search: params,
-    method: rawConfig.method,
-    timeout: rawConfig.timeout,
-    headers
+    search: parseParams(search, rawConfig.params),
+
+    method: rawConfig.method.toLocaleUpperCase(),
+    headers: {}
   }
 
-  return requestConfig
+  config.headers = parseHeaders(rawConfig, config)
+
+  return config
 }
 
 export function sleep(timeout: number) {