Node.js document loader updates.

- Add 'httpAgent' and 'httpsAgent' options.
- Default 'user-agent' header to 'jsonld.js'.
- Fix document loader tests and enable by default.
  - Add local test server.
  - Fix old tests.
  - Add more coverage.

Author: davidlehn

CHANGELOG.md

@@ -19,10 +19,12 @@
   versions.
 - Use Babel `usage` mode rather than listing individual polyfills. May increase
   bundle size due to more accurate support of intended output targets.
+- Default Node.js document loader `user-agent` header set to "jsonld.js".
 
 ### Added
 - Distribute a `jsonld.esm.min.js` bundle optimized for the features available
   in browsers that support ES Modules.
+- Default Node.js document loader `httpAgent` and `httpsAgent` options.
 
 ### Removed
 - Use of deprecated library `request` in Node.js documentLoader.

README.md

@@ -340,6 +340,12 @@ const compacted = await jsonld.compact(
   doc, context, {documentLoader: customLoader});
 ```
 
+### Node.js Document Loader User-Agent
+
+It is recommended to set a default `user-agent` header for Node.js
+applications. The default for the default Node.js document loader is
+`jsonld.js`.
+
 Related Modules
 ---------------
 

lib/documentLoaders/node.js

@@ -15,23 +15,35 @@ const {httpClient} = require('@digitalbazaar/http-client');
  * Creates a built-in node document loader.
  *
  * @param options the options to use:
- *          secure: require all URLs to use HTTPS.
- *          strictSSL: true to require SSL certificates to be valid,
- *            false not to (default: true).
- *          maxRedirects: the maximum number of redirects to permit, none by
- *            default.
- *          headers: an object (map) of headers which will be passed as request
- *            headers for the requested document. Accept is not allowed.
+ *          [secure]: require all URLs to use HTTPS. (default: false)
+ *          [strictSSL]: true to require SSL certificates to be valid,
+ *            false not to. (default: true)
+ *          [maxRedirects]: the maximum number of redirects to permit.
+ *            (default: none)
+ *          [headers]: an object (map) of headers which will be passed as
+ *            request headers for the requested document. Accept is not
+ *            allowed. (default: none).
+ *          [httpAgent]: a Node.js `http.Agent` to use with 'http' requests.
+ *            (default: none)
+ *          [httpsAgent]: a Node.js `https.Agent` to use with 'https' requests.
+ *            (default: An agent with rejectUnauthorized to the strictSSL
+ *            value)
  *
  * @return the node document loader.
  */
 module.exports = ({
   secure,
   strictSSL = true,
   maxRedirects = -1,
-  headers = {}
+  headers = {},
+  httpAgent,
+  httpsAgent
 } = {strictSSL: true, maxRedirects: -1, headers: {}}) => {
   headers = buildHeaders(headers);
+  // set a default header if none was set
+  if(!('user-agent' in headers)) {
+    headers['user-agent'] = 'jsonld.js';
+  }
   const http = require('http');
 
   const queue = new RequestQueue();
@@ -40,13 +52,15 @@ module.exports = ({
   });
 
   async function loadDocument(url, redirects) {
-    if(url.indexOf('http:') !== 0 && url.indexOf('https:') !== 0) {
+    const isHttp = url.startsWith('http:');
+    const isHttps = url.startsWith('https:');
+    if(!isHttp && !isHttps) {
       throw new JsonLdError(
         'URL could not be dereferenced; only "http" and "https" URLs are ' +
         'supported.',
         'jsonld.InvalidUrl', {code: 'loading document failed', url});
     }
-    if(secure && url.indexOf('https') !== 0) {
+    if(secure && !isHttps) {
       throw new JsonLdError(
         'URL could not be dereferenced; secure mode is enabled and ' +
         'the URL\'s scheme is not "https".',
@@ -60,7 +74,9 @@ module.exports = ({
 
     let alternate = null;
 
-    const {res, body} = await _fetch({url, headers, strictSSL});
+    const {res, body} = await _fetch({
+      url, headers, strictSSL, httpAgent, httpsAgent
+    });
     doc = {contextUrl: null, documentUrl: url, document: body || null};
     // handle error
     const statusText = http.STATUS_CODES[res.status];
@@ -143,11 +159,19 @@ module.exports = ({
   }
 };
 
-async function _fetch({url, headers, strictSSL}) {
+async function _fetch({url, headers, strictSSL, httpAgent, httpsAgent}) {
   try {
-    const agent = new https.Agent({rejectUnauthorized: strictSSL});
-    const res = await httpClient.get(
-      url, {headers, agent, redirect: 'manual'});
+    const options = {headers, redirect: 'manual'};
+    const isHttps = url.startsWith('https:');
+    if(isHttps) {
+      options.agent =
+        httpsAgent || new https.Agent({rejectUnauthorized: strictSSL});
+    } else {
+      if(httpAgent) {
+        options.agent = httpAgent;
+      }
+    }
+    const res = await httpClient.get(url, options);
     const body = JSON.stringify(res.data, null, 2);
     return {res, body};
   } catch(e) {