feat: Add convertBody, extractContentType and getTotalBytes

Signed-off-by: Richie Bendall <[email protected]>

Author: Richienb

.gitignore

@@ -103,6 +103,12 @@ typings/
 # Optional eslint cache
 .eslintcache
 
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
 # Optional REPL history
 .node_repl_history
 
@@ -125,8 +131,9 @@ typings/
 # nuxt.js build output
 .nuxt
 
-# react / gatsby
-public/
+# gatsby files
+.cache/
+public
 
 # vuepress build output
 .vuepress/dist

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 Richie Bendall
+Copyright (c) 2016 - 1019 The Node Fetch Team
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,34 +1,17 @@
-# Typescript QuickStart
+# Fetch Charset Detection [![Travis CI Build Status](https://img.shields.io/travis/com/Richienb/fetch-charset-detection/master.svg?style=for-the-badge)](https://travis-ci.com/Richienb/fetch-charset-detection)
 
-A quick start template for Typescript.
+Convert a buffer and headers to UTF-8 text, originally from `node-fetch`.
 
-[![Travis CI Build Status](https://img.shields.io/travis/com/Richienb/typescript-quickstart/master.svg?style=for-the-badge)](https://travis-ci.com/Richienb/typescript-quickstart)
-[![CodeFactor Score](https://www.codefactor.io/repository/github/Richienb/typescript-quickstart/badge?style=for-the-badge)](https://www.codefactor.io/repository/github/Richienb/typescript-quickstart)
-
-[![NPM](https://nodei.co/npm/typescript-quickstart.png?downloads=true&downloadRank=true&stars=true)](https://nodei.co/npm/typescript-quickstart)
+[![NPM](https://nodei.co/npm/fetch-charset-detection.png?downloads=true&downloadRank=true&stars=true)](https://nodei.co/npm/fetch-charset-detection)
 
 ## Importing
 
 From your NodeJS application:
 
 ```js
-const TypeScriptQuickStart = require("typescript-quickstart")
-```
-
-From your web application:
-
-```html
-<script src="https://unpkg.com/typescript-quickstart/dist/index.min.js"></script>
-```
-
-## Initialisation
-
-```js
-const TypeScriptQuickStart = new TypeScriptQuickStart()
+const charsetDetection = require("fetch-charset-detection");
 ```
 
 ## Usage
 
-Read the [documentation](https://richienb.github.io/typescript-quickstart).
-
-> The typescript-quickstart npm module is not maintained by this project.
+Read the [documentation](https://richienb.github.io/fetch-charset-detection).

package.json

@@ -1,32 +1,39 @@
 {
-    "name": "typescript-quickstart",
-    "description": "A quick start template for Typescript.",
+    "name": "fetch-charset-detection",
+    "description": "Convert a buffer and headers to UTF-8 text, originally from node-fetch.",
     "keywords": [
-        "quickstart",
-        "typescript",
-        "template",
-        "compiler",
+        "content-type",
+        "headers",
+        "http",
         "meta",
-        "node"
+        "node-fetch"
     ],
     "version": "0.0.0",
     "main": "dist/index.js",
     "files": [
         "src/**/*",
         "dist/**/*"
     ],
-    "repository": "https://github.com/Richienb/typescript-quickstart.git",
+    "repository": "https://github.com/Richienb/fetch-charset-detection.git",
     "author": "Richie Bendall <[email protected]>",
     "license": "MIT",
     "scripts": {
         "dev": "microbundle --target node --watch",
         "build": "yarn js && yarn docs",
-        "js": "microbundle --target node",
-        "docs": "typedoc --out ./docs --mode file --target ES6 ./src",
+        "js": "microbundle --target node --format es,cjs --external http,https,stream,zlib,cheerio,content-type,iconv-lite",
+        "docs": "typedoc --out ./docs --mode file --target ES6 --ignoreCompilerErrors ./src",
         "lint": "eslint src/*"
     },
+    "dependencies": {
+        "cheerio": "^1.0.0-rc.3",
+        "content-type": "^1.0.4",
+        "iconv-lite": "^0.5.0"
+    },
     "devDependencies": {
+        "@types/cheerio": "^0.22.13",
+        "@types/content-type": "^1.1.3",
         "@typescript-eslint/eslint-plugin": "^2.3.1",
+        "@typescript-eslint/parser": "^2.3.1",
         "eslint": "^6.4.0",
         "eslint-config-google": "^0.14.0",
         "microbundle": "^0.11.0",

src/index.ts

@@ -3,7 +3,7 @@
  *
  * MIT License
  *
- * Copyright (c) 2019 Richie Bendall
+ * Copyright (c) 2016 - 1019 The Node Fetch Team
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the 'Software'), to deal
@@ -24,15 +24,129 @@
  * SOFTWARE.
  */
 
+import { decode as convert } from "iconv-lite"
+import getCharSet from "./utils/getCharSet"
+import { parse as $ } from "cheerio"
+import { isURLSearchParams, isBlob, isArrayBuffer } from "./utils/is"
+import { Stream, Writable } from "stream"
+
 /**
- * A quick start template for Typescript.
+* Detect buffer encoding and convert to target encoding
+* ref: http://www.w3.org/TR/2011/WD-html5-20110113/parsing.html#determining-the-character-encoding
+*
+* @param buffer Incoming buffer.
+* @param headers Headers provided with the request.
+*/
+export function convertBody(buffer: Buffer, headers?: Headers): string {
+    const contentType = headers instanceof Headers ? headers.get("content-type") : null
+    let charset: string
+
+    // Header
+    if (contentType) {
+        charset = getCharSet(contentType)
+    }
+
+    // No charset in content type, peek at response body for at most 1024 bytes
+    const res = buffer.slice(0, 1024).toString()
+
+    // HTML5, HTML4 and XML
+    if (!charset && res) {
+        charset = getCharSet(
+            $(res)("meta[charset]").attr("charset") || // HTML5
+            $(res)("meta[http-equiv][content]").attr("content") || // HTML4
+            $(res.replace(/<\?(.*)\?>/im, "<$1>"), { xmlMode: true }).root().find("xml").attr("encoding") // XML
+        )
+    }
+
+    // Prevent decode issues when sites use incorrect encoding
+    // ref: https://hsivonen.fi/encoding-menu/
+    if (charset && charset.toLowerCase() in ["gb2312", "gbk"]) {
+        charset = "gb18030"
+    }
+
+    // Turn raw buffers into a single utf-8 buffer
+    return convert(
+        buffer,
+        charset || "utf-8"
+    )
+}
+
+/**
+ * Performs the operation "extract a `Content-Type` value from |object|" as
+ * specified in the specification:
+ * https://fetch.spec.whatwg.org/#concept-bodyinit-extract
+ *
+ * This function assumes that instance.body is present.
+ *
+ * @param body Any options.body input
  */
-export class TypeScriptQuickStart {
-    /**
-     * Your first method.
-     * @param text - The text to return
-    */
-    public helloWorld(text: string = ""): string {
-        return text
+export function extractContentType(body: any): string | null {
+    // Body is string
+    if (typeof body === "string") return "text/plain;charset=UTF-8"
+
+    // Body is a URLSearchParams
+    if (isURLSearchParams(body)) return "application/x-www-form-urlencoded;charset=UTF-8"
+
+    // Body is blob
+    if (isBlob(body)) return body.type || null
+
+    // Body is a Buffer (Buffer, ArrayBuffer or ArrayBufferView)
+    if (Buffer.isBuffer(body) || isArrayBuffer(body) || ArrayBuffer.isView(body)) return null
+
+    // Detect form data input from form-data module
+    if (typeof body.getBoundary === "function") return `multipart/form-data;boundary=${body.getBoundary()}`
+
+    // Body is stream - can't really do much about this
+    if (body instanceof Stream) return null
+
+    // Body constructor defaults other things to string
+    return "text/plain;charset=UTF-8"
+}
+
+/**
+ * The Fetch Standard treats this as if "total bytes" is a property on the body.
+ * For us, we have to explicitly get it with a function.
+ *
+ * ref: https://fetch.spec.whatwg.org/#concept-body-total-bytes
+ *
+ * @param body Body object from the Body instance.
+ */
+export function getTotalBytes(body: any): number | null {
+    // Body is null or undefined
+    if (body == null) return 0
+
+    // Body is Blob
+    if (isBlob(body)) return body.size
+
+    // Body is Buffer
+    if (Buffer.isBuffer(body)) return body.length
+
+    // Detect form data input from form-data module
+    if (body && typeof body.getLengthSync === "function") return body.hasKnownLength && body.hasKnownLength() ? body.getLengthSync() : null
+
+    // Body is stream
+    return null
+}
+
+/**
+ * Write a Body to a Node.js WritableStream (e.g. http.Request) object.
+ *
+ * @param body Body object from the Body instance.
+ * @param dest The stream to write to.
+ */
+export function writeToStream(body: any, dest: Writable): void {
+    // Body is null
+    if (body == null) dest.end()
+
+    // Body is Blob
+    else if (isBlob(body)) body.stream().pipe(dest)
+
+    // Body is buffer
+    else if (Buffer.isBuffer(body)) {
+        dest.write(body)
+        dest.end()
+    } else {
+        // Body is stream
+        body.pipe(dest)
     }
 }

src/utils/getCharSet.ts

@@ -0,0 +1,9 @@
+import { parse as parseContentType } from "content-type"
+
+/**
+ * Get the character set from a Content-Type header.
+ * @param contentType The Content-Type HTTP header.
+ */
+export default function getCharSet(contentType: string): string | null {
+    return contentType != null ? parseContentType(contentType).parameters.charset : null
+}

src/utils/is.ts

@@ -0,0 +1,67 @@
+const NAME = Symbol.toStringTag;
+
+/**
+ * Check if `obj` is a URLSearchParams object
+ * ref: https://github.com/bitinn/node-fetch/issues/296#issuecomment-307598143
+ *
+ * @param obj The object to check.
+ */
+export function isURLSearchParams(obj): boolean {
+    return (
+        typeof obj === 'object' &&
+        typeof obj.append === 'function' &&
+        typeof obj.delete === 'function' &&
+        typeof obj.get === 'function' &&
+        typeof obj.getAll === 'function' &&
+        typeof obj.has === 'function' &&
+        typeof obj.set === 'function' &&
+        typeof obj.sort === 'function' &&
+        obj[NAME] === 'URLSearchParams'
+    );
+}
+
+/**
+ * Check if `obj` is a W3C `Blob` object (which `File` inherits from)
+ *
+ * @param obj The object to check.
+ */
+export function isBlob(obj): boolean {
+    return (
+        typeof obj === 'object' &&
+        typeof obj.arrayBuffer === 'function' &&
+        typeof obj.type === 'string' &&
+        typeof obj.stream === 'function' &&
+        typeof obj.constructor === 'function' &&
+        /^(Blob|File)$/.test(obj[NAME])
+    );
+}
+
+/**
+ * Check if `obj` is an instance of AbortSignal.
+ *
+ * @param obj The object to check.
+ */
+export function isAbortSignal(obj): boolean {
+    return (
+        typeof obj === 'object' &&
+        obj[NAME] === 'AbortSignal'
+    );
+}
+
+/**
+ * Check if `obj` is an instance of ArrayBuffer.
+ *
+ * @param obj The object to check.
+ */
+export function isArrayBuffer(obj): boolean {
+    return obj[NAME] === 'ArrayBuffer';
+}
+
+/**
+ * Check if `obj` is an instance of AbortError.
+ *
+ * @param obj The object to check.
+ */
+export function isAbortError(obj): boolean {
+    return obj.name === 'AbortError';
+}