Merge pull request #44 from coder-hxl/fix/43

fix(crawlData): correct handle the header of the post method

Author: coder-hxl

src/request.ts

@@ -28,14 +28,24 @@ function parseHeaders(
 ) {
   const rawHeaders = rawConfig.headers ?? {}
   const headers: AnyObject = {
-    'user-agent':
-      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36',
+    'User-Agent':
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
     ...rawHeaders
   }
 
   if (config.method === 'POST' && rawConfig.data) {
-    headers['Content-Type'] = 'application/json'
-    headers['Content-Length'] = Buffer.byteLength(rawConfig.data)
+    const defaultHeaderConfig = [
+      { key: 'Content-Type', value: 'application/json' },
+      { key: 'Content-Length', value: Buffer.byteLength(rawConfig.data) }
+    ]
+
+    defaultHeaderConfig.forEach((item) => {
+      const { key, value } = item
+
+      if (isUndefined(rawHeaders[key])) {
+        headers[key] = value
+      }
+    })
   }
 
   return headers

test/server/index.js

@@ -1 +1 @@
-"use strict";require("node:http").createServer(((e,t)=>{t.setHeader("Content-Type","text/plain"),t.end("success")})).listen(8888,(()=>{console.log("服务器在 8888 端口启动成功~")}));
+"use strict";require("node:http").createServer(((e,t)=>{console.log(e.headers),t.setHeader("Content-Type","text/plain"),t.end("success")})).listen(8888,(()=>{console.log("服务器在 8888 端口启动成功~")}));

test/server/index.ts

@@ -2,6 +2,8 @@ import http from 'node:http'
 
 http
   .createServer((req, res) => {
+    console.log(req.headers)
+
     res.setHeader('Content-Type', 'text/plain')
     res.end('success')
   })

test/start/index.js

@@ -1,40 +1,16 @@
 import xCrawl from 'x-crawl'
 
-const testXCrawl = xCrawl({
-  baseUrl: 'http://8.210.98.225:9001/api/room/193581217/room_picture'
-})
+const testXCrawl = xCrawl()
 
-const names = [
-  '1672468265289r193581217',
-  '1672468265300r193581217',
-  '1672468265284r193581217',
-  '1672468265279r193581217',
-  '1672468265298r193581217',
-  '1672468265293r193581217'
-]
-
-const targets: string[] = []
-const fileNames: (string | null)[] = []
-const storeDirs: string[] = []
-const extensions: (string | null)[] = []
-
-names.forEach((name, i) => {
-  targets.push(`/${name}.jpg`)
-
-  if (i % 2) {
-    fileNames.push(name)
-    storeDirs.push(`./upload/${name}`)
-    extensions.push('.jpg')
-  } else {
-    fileNames.push(null)
-    storeDirs.push('./upload')
-    extensions.push(null)
-  }
-})
-
-testXCrawl.crawlFile({
-  targets,
-  storeDirs,
-  fileNames,
-  extensions
-})
+testXCrawl
+  .crawlData({
+    url: 'http://localhost:8888',
+    method: 'post',
+    headers: {
+      'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'
+    },
+    data: { name: 'hxl', age: 19 }
+  })
+  .then((res) => {
+    console.log(res)
+  })