fetchFile 返回 promise resolve 是文件信息数组

Author: coder-hxl

src/index.ts

@@ -6,10 +6,12 @@ import { isArray, mergeConfig } from './utils'
 
 import {
   IFetchConfig,
+  IFetchFile,
   IFetchFileConfig,
   IRequest,
   IXCrawlBaseConifg
 } from './types'
+import path from 'node:path'
 
 export default class XCrawl {
   private readonly baseConfig: IXCrawlBaseConifg
@@ -36,39 +38,57 @@ export default class XCrawl {
     return res as T
   }
 
-  async fetchFile(config: IFetchFileConfig): Promise<void> {
-    const { requestConifg, intervalTime, fileConfig } = mergeConfig(
-      this.baseConfig,
-      config
-    )
-
-    let successCount = 0
-
-    function eachRequestResHandle(requestRes: IRequest, currentCount: number) {
-      const { headers, data } = requestRes
-
-      const fileType = headers['content-type']?.split('/').pop()
-      const filename = `${new Date().getTime()}.${fileType}`
-      const path = `${fileConfig.storeDir}/${filename}`
-
-      fs.createWriteStream(path, 'binary').write(data, (err) => {
-        if (err) {
-          return console.log(
-            `File save error requested for the ${currentCount}: ${err.message}`
-          )
-        }
-
-        if (++successCount === requestConifgArr.length) {
-          console.log('All files downloaded successfully!')
-        }
-      })
-    }
-
-    const requestConifgArr = isArray(requestConifg)
-      ? requestConifg
-      : [requestConifg]
-
-    await batchRequest(requestConifgArr, intervalTime, eachRequestResHandle)
+  fetchFile(config: IFetchFileConfig): Promise<IFetchFile> {
+    return new Promise((resolve) => {
+      const { requestConifg, intervalTime, fileConfig } = mergeConfig(
+        this.baseConfig,
+        config
+      )
+
+      let successCount = 0
+      const res: IFetchFile = []
+
+      function eachRequestResHandle(
+        requestRes: IRequest,
+        currentCount: number
+      ) {
+        const { headers, data } = requestRes
+
+        const mimeType = headers['content-type'] ?? ''
+        const suffix = mimeType.split('/').pop()
+        const fileName = new Date().getTime().toString()
+        const filePath = path.resolve(
+          fileConfig.storeDir,
+          `${fileName}.${suffix}`
+        )
+
+        fs.createWriteStream(filePath, 'binary').write(data, (err) => {
+          if (err) {
+            return console.log(
+              `File save error requested for the ${currentCount}: ${err.message}`
+            )
+          }
+
+          res.push({
+            fileName,
+            mimeType,
+            size: data.length,
+            filePath
+          })
+
+          if (++successCount === requestConifgArr.length) {
+            console.log('All files downloaded successfully!')
+            resolve(res)
+          }
+        })
+      }
+
+      const requestConifgArr = isArray(requestConifg)
+        ? requestConifg
+        : [requestConifg]
+
+      batchRequest(requestConifgArr, intervalTime, eachRequestResHandle)
+    })
   }
 
   async fetchHTML(url: string): Promise<JSDOM> {

src/types.ts

@@ -68,3 +68,10 @@ export interface IFetchFileConfig extends IFetchBaseConifg {
     storeDir: string
   }
 }
+
+export type IFetchFile = {
+  fileName: string
+  mimeType: string
+  size: number
+  filePath: string
+}[]

test/start/index.js

@@ -63,43 +63,42 @@ const bilibiliXCrawl = new XCrawl({
   }
 })
 
-function bilibiliRecommendData() {
-  bilibiliXCrawl
-    .fetch<IBRecommend>({
-      requestConifg: {
-        url: 'https://api.bilibili.com/x/web-interface/wbi/index/top/feed/rcmd',
-        method: 'GET',
-        params: {
-          y_num: 5,
-          fresh_type: 3,
-          feed_version: 'V8',
-          fresh_idx_1h: 1,
-          fetch_row: 1,
-          fresh_idx: 1,
-          brush: 0,
-          homepage_ver: 1,
-          ps: 10,
-          outside_trigger: '',
-          w_rid: '2e4be8e9830ecd780c5b0ff2bef805c9',
-          wts: 1674556002
-        },
-        headers: {}
+async function bilibiliRecommendData() {
+  const recommend = await bilibiliXCrawl.fetch<IBRecommend>({
+    requestConifg: {
+      url: 'https://api.bilibili.com/x/web-interface/wbi/index/top/feed/rcmd',
+      method: 'GET',
+      params: {
+        y_num: 5,
+        fresh_type: 3,
+        feed_version: 'V8',
+        fresh_idx_1h: 1,
+        fetch_row: 1,
+        fresh_idx: 1,
+        brush: 0,
+        homepage_ver: 1,
+        ps: 10,
+        outside_trigger: '',
+        w_rid: '2e4be8e9830ecd780c5b0ff2bef805c9',
+        wts: 1674556002
       }
-    })
-    .then((res) => {
-      const pictureUrls: IRequestConfig[] = res.data.item.map((item) => ({
-        url: item.pic,
-        method: 'GET'
-      }))
+    }
+  })
 
-      bilibiliXCrawl.fetchFile({
-        requestConifg: pictureUrls,
-        intervalTime: { max: 3000, min: 2000 },
-        fileConfig: { storeDir: path.resolve(__dirname, './upload') }
-      })
-    })
+  const pictureUrls: IRequestConfig[] = recommend.data.item.map((item) => ({
+    url: item.pic,
+    method: 'GET'
+  }))
+
+  const storeFile = await bilibiliXCrawl.fetchFile({
+    requestConifg: pictureUrls,
+    intervalTime: { max: 3000, min: 2000 },
+    fileConfig: { storeDir: path.resolve(__dirname, './upload') }
+  })
+
+  console.log(storeFile)
 }
-// bilibiliRecommendData()
+bilibiliRecommendData()
 
 // 2.2 HTML: b站首页标题
 // 采用 jsdom 对 HTML String 解析
@@ -109,4 +108,4 @@ function bilibiliHTMLData() {
     console.log(dom.window.document.querySelector('title')?.textContent)
   })
 }
-bilibiliHTMLData()
+// bilibiliHTMLData()