fetch API renamed to fetchData API

Author: coder-hxl

README.md

@@ -45,7 +45,7 @@ Create a crawler instance via new XCrawl.
 class XCrawl {
   private readonly baseConfig
   constructor(baseConfig?: IXCrawlBaseConifg)
-  fetch<T = any>(config: IFetchConfig): Promise<IFetch<T>>
+  fetchData<T = any>(config: IFetchDataConfig): Promise<IFetchData<T>>
   fetchFile(config: IFetchFileConfig): Promise<IFetchFile>
   fetchHTML(url: string): Promise<JSDOM>
 }
@@ -67,14 +67,14 @@ const myXCrawl = new XCrawl({
 })
 ```
 
-### fetch
+### fetchData
 
-fetch is the method of the above <a href="#myXCrawl" style="text-decoration: none">myXCrawl</a> instance, which is usually used to crawl APIs to obtain JSON data and so on.
+fetchData is the method of the above <a href="#myXCrawl" style="text-decoration: none">myXCrawl</a> instance, which is usually used to crawl APIs to obtain JSON data and so on.
 
 - Type
 
 ```ts
-function fetch<T = any>(config: IFetchConfig): Promise<T>
+function fetchData <T = any>(config: IFetchDataConfig): Promise<T>
 ```
 
 - Example
@@ -86,7 +86,7 @@ const requestConifg = [
   { url: '/xxxx', method: 'GET' }
 ]
 
-myXCrawl.fetch({ 
+myXCrawl.fetchData({ 
   requestConifg, // Request configuration, can be IRequestConfig | IRequestConfig[]
   intervalTime: 800 // Interval between next requests, multiple requests are valid
 }).then(res => {
@@ -188,10 +188,10 @@ interface IFetchBaseConifg {
 }
 ```
 
-- IFech
+- IFechData
 
 ```ts
-type IFetch<T> = {
+type IFetchData<T> = {
   statusCode: number | undefined
   headers: IncomingHttpHeaders // node:http
   data: T
@@ -219,10 +219,10 @@ interface IXCrawlBaseConifg {
 }
 ```
 
-- IFetchConfig
+- IFetchDataConfig
 
 ```ts
-interface IFetchConfig extends IFetchBaseConifg {
+interface IFetchDataConfig extends IFetchBaseConifg {
 }
 ```
 
@@ -291,7 +291,7 @@ docsXCrawl.fetchHTML('/zh/get-started').then((jsdom) => {
 class XCrawl {
   private readonly baseConfig
   constructor(baseConfig?: IXCrawlBaseConifg)
-  fetch<T = any>(config: IFetchConfig): Promise<IFetch<T>>
+  fetchData<T = any>(config: IFetchDataConfig): Promise<IFetchData<T>>
   fetchFile(config: IFetchFileConfig): Promise<IFetchFile>
   fetchHTML(url: string): Promise<JSDOM>
 }
@@ -313,14 +313,14 @@ const myXCrawl = new XCrawl({
 })
 ```
 
-### fetch
+### fetchData
 
 fetch 是上面 <a href="#cn-myXCrawl"  style="text-decoration: none">myXCrawl</a> 实例的方法，通常用于爬取 API ，可获取 JSON 数据等等。
 
 - 类型
 
 ```ts
-function fetch<T = any>(config: IFetchConfig): Promise<T>
+function fetchData<T = any>(config: IFetchDataConfig): Promise<T>
 ```
 
 - 示例
@@ -332,7 +332,7 @@ const requestConifg = [
   { url: '/xxxx', method: 'GET' }
 ]
 
-myXCrawl.fetch({ 
+myXCrawl.fetchData({ 
   requestConifg, // 请求配置, 可以是 IRequestConfig | IRequestConfig[]
   intervalTime: 800 // 下次请求的间隔时间, 多个请求才有效
 }).then(res => {
@@ -434,7 +434,7 @@ interface IFetchBaseConifg {
 }
 ```
 
-- IFetch
+- IFetchData
 
 ```ts
 type IFetch<T> = {
@@ -465,10 +465,10 @@ interface IXCrawlBaseConifg {
 }
 ```
 
-- IFetchConfig
+- IFetchDataConfig
 
 ```ts
-interface IFetchConfig extends IFetchBaseConifg {
+interface IFetchDataConfig extends IFetchBaseConifg {
 }
 ```
 

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "x-crawl",
-  "version": "0.0.3",
+  "version": "0.1.0",
   "author": "CoderHxl",
   "description": "XCrawl is a Nodejs multifunctional crawler library.",
   "license": "MIT",

publish/package.json

@@ -1,6 +1,6 @@
 {
   "name": "x-crawl",
-  "version": "0.0.3",
+  "version": "0.1.0",
   "author": "CoderHxl",
   "description": "XCrawl is a Nodejs multifunctional crawler library.",
   "license": "MIT",

src/index.ts

@@ -6,8 +6,8 @@ import { batchRequest, request } from './request'
 import { isArray, mergeConfig } from './utils'
 
 import {
-  IFetch,
-  IFetchConfig,
+  IFetchData,
+  IFetchDataConfig,
   IFetchFile,
   IFetchFileConfig,
   IRequest,
@@ -21,14 +21,14 @@ export default class XCrawl {
     this.baseConfig = baseConfig
   }
 
-  async fetch<T = any>(config: IFetchConfig): Promise<IFetch<T>> {
+  async fetchData<T = any>(config: IFetchDataConfig): Promise<IFetchData<T>> {
     const { requestConifg, intervalTime } = mergeConfig(this.baseConfig, config)
 
     const requestConfigQueue = isArray(requestConifg)
       ? requestConifg
       : [requestConifg]
 
-    const container: IFetch<T> = []
+    const container: IFetchData<T> = []
 
     await batchRequest(requestConfigQueue, intervalTime, (requestRes) => {
       const data: T = JSON.parse(requestRes.data.toString())
@@ -45,6 +45,10 @@ export default class XCrawl {
         config
       )
 
+      const requestConfigQueue = isArray(requestConifg)
+        ? requestConifg
+        : [requestConifg]
+
       let successCount = 0
       const container: IFetchFile = []
 
@@ -83,10 +87,6 @@ export default class XCrawl {
         })
       }
 
-      const requestConfigQueue = isArray(requestConifg)
-        ? requestConifg
-        : [requestConifg]
-
       batchRequest(requestConfigQueue, intervalTime, eachRequestResHandle)
     })
   }

src/types.ts

@@ -62,15 +62,15 @@ export interface IFetchBaseConifg {
   intervalTime?: IIntervalTime
 }
 
-export interface IFetchConfig extends IFetchBaseConifg {}
+export interface IFetchDataConfig extends IFetchBaseConifg {}
 
 export interface IFetchFileConfig extends IFetchBaseConifg {
   fileConfig: {
     storeDir: string
   }
 }
 
-export type IFetch<T> = {
+export type IFetchData<T> = {
   statusCode: number | undefined
   headers: IncomingHttpHeaders
   data: T

test/modal/test.ts

@@ -21,7 +21,7 @@ function httpProtocol() {
     const httpXCrawl = new XCrawl()
 
     httpXCrawl
-      .fetch({
+      .fetchData({
         requestConifg: {
           url: 'http://localhost:9001/api/home/goodprice'
         }
@@ -32,7 +32,7 @@ function httpProtocol() {
 
 function httpsProtocol() {
   return new Promise((resolve) => {
-    const httpsXCrawl = new XCrawl()
+    const httpsXCrawl = new XCrawl({ timeout: 10000 })
 
     httpsXCrawl
       .fetchHTML('https://docs.github.com/zh')
@@ -49,12 +49,12 @@ test('https protocol', async () => {
 })
 
 // API
-function fetchAPI() {
+function fetchDataAPI() {
   return new Promise((resolve) => {
     const myXCrawl = new XCrawl()
 
     myXCrawl
-      .fetch({
+      .fetchData({
         requestConifg: {
           url: 'http://localhost:9001/api/area/阳江市',
           method: 'POST',
@@ -80,7 +80,7 @@ function fetchFileAPI() {
     })
 
     myXCrawl
-      .fetch({
+      .fetchData({
         requestConifg: {
           url: 'http://localhost:9001/api/area/阳江市',
           method: 'POST',
@@ -117,8 +117,8 @@ function fetchHTMLAPI() {
   })
 }
 
-test('fetch API', async () => {
-  await expect(fetchAPI()).resolves.toBe(true)
+test('fetchData API', async () => {
+  await expect(fetchDataAPI()).resolves.toBe(true)
 })
 
 test('fetchFile API', async () => {