fix: Properly parse charset for specific parts

Signed-off-by: Richie Bendall <[email protected]>

Author: Richienb

package.json

@@ -20,7 +20,7 @@
     "scripts": {
         "dev": "yarn js --watch",
         "build": "yarn js && yarn docs",
-        "js": "tsc --outDir dist/",
+        "js": "tsc",
         "docs": "typedoc --out ./docs --mode file --target ES6 --ignoreCompilerErrors ./src",
         "lint": "eslint src/**/*",
         "test": "ava"
@@ -29,7 +29,8 @@
         "cheerio": "^1.0.0-rc.3",
         "content-type": "^1.0.4",
         "iconv-lite": "^0.5.0",
-        "lodash": "^4.17.15"
+        "lodash": "^4.17.15",
+        "nice-try": "^2.0.0"
     },
     "devDependencies": {
         "@types/cheerio": "^0.22.13",
@@ -59,5 +60,8 @@
         "require": [
             "ts-node/register"
         ]
+    },
+    "engines": {
+        "node": ">=0.12"
     }
 }

src/lib/convert-body.ts

@@ -1,5 +1,5 @@
 import getCharset from "../utils/get-charset"
-import { decode as convert } from "iconv-lite"
+import { decode } from "iconv-lite"
 import { load as $ } from "cheerio"
 import _ from "lodash"
 
@@ -31,10 +31,10 @@ export function convertBody(buffer: Buffer, headers?: Headers): string {
 
     // Prevent decode issues when sites use incorrect encoding
     // ref: https://hsivonen.fi/encoding-menu/
-    if (charset && _.lowerCase(charset) in ["gb2312", "gbk"]) charset = "gb18030"
+    if (charset && _.includes(["gb2312", "gbk"], _.lowerCase(charset))) charset = "gb18030"
 
     // Turn raw buffers into a single utf-8 buffer
-    return convert(
+    return decode(
         buffer,
         charset || "utf-8",
     )

src/utils/get-charset.ts

@@ -1,10 +1,15 @@
 import { parse as parseContentType } from "content-type"
 import _ from "lodash"
+import niceTry from "nice-try"
 
 /**
  * Get the character set from a Content-Type header.
  * @param contentType The Content-Type HTTP header.
  */
-export default function getCharSet(contentType: string): string | null {
-    return !_.isNil(contentType) ? parseContentType(contentType).parameters.charset : null
+export default function getCharset(contentType: string): string | null {
+    if (_.isNil(contentType)) return null
+
+    const parsed = niceTry(() => parseContentType(contentType))
+    if (!_.isNil(parsed)) return parsed.parameters.charset
+    else return contentType
 }

tsconfig.json

@@ -1,5 +1,12 @@
 {
     "compilerOptions": {
-        "esModuleInterop": true
-    }
+        "esModuleInterop": true,
+        "sourceMap": true,
+        "declaration": true,
+        "outDir": "dist/",
+        "target": "es5"
+    },
+    "include": [
+        "src/**/*"
+    ]
 }

yarn.lock

@@ -2887,6 +2887,11 @@ nice-try@^1.0.4:
   resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366"
   integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==
 
+nice-try@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-2.0.0.tgz#0ccb36ce82be13528f0217334adcf870c8eb3356"
+  integrity sha512-/o4KoXbUkEjD2NDwvBCkSzEDOfMuHPmbzSD5Pp2Bb5U/69MnEvpc4YfW2Y1BJqJpcaam04+fR0kF8n5xm8b6MQ==
+
 node-fetch@^2.6.0:
   version "2.6.0"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"