try to decompress binary when header content-encoding missing (#109)

* try to decompress if needed.

* added unit test.

* add unit tests.

* bump version.

* opimtization only decompress if content type is of text.

* update test cases.

Author: xinghengwang

lib/dataUtils.js

@@ -4,6 +4,88 @@ var url = require('url');
 var hash = require('crypto-js/md5');
 var isCreditCard = require('card-validator');
 var assign = require('lodash/assign');
+var zlib = require('zlib');
+// Helper to check if buffer is binary (not utf8 string)
+function isBinaryBuffer(buf) {
+  if (!Buffer.isBuffer(buf)) return false;
+  // Heuristic: if buffer contains non-printable characters, treat as binary
+  var text = buf.toString('utf8');
+  // If conversion to string produces replacement chars, it's likely binary
+  return text.includes('\uFFFD') || /[\x00-\x08\x0E-\x1F]/.test(text);
+}
+
+// Decompress if needed based on headers and heuristics
+function decompressIfNeeded(body, headers) {
+  if (!body) return body;
+  var buf = Buffer.isBuffer(body) ? body : Buffer.from(body);
+  var encoding = (headers && (headers['content-encoding'] || headers['Content-Encoding'])) || '';
+  var server = (headers && (headers['server'] || headers['Server'])) || '';
+  var contentType = (headers && (headers['content-type'] || headers['Content-Type'])) || '';
+  var decompressed = null;
+  var isBinary = isBinaryBuffer(buf);
+
+  // Only decompress if content-type is json, xml, or text
+  var isTextType = false;
+  if (contentType) {
+    var ct = contentType.toLowerCase();
+    if (ct.indexOf('json') !== -1 || ct.indexOf('xml') !== -1 || ct.indexOf('text') !== -1) {
+      isTextType = true;
+    }
+  }
+  if (!isTextType) {
+    // Not a text type, skip decompression
+    return body;
+  }
+
+  try {
+    if (encoding && isBinary) {
+      if (encoding.indexOf('gzip') !== -1) {
+        decompressed = zlib.gunzipSync(buf);
+      } else if (encoding.indexOf('deflate') !== -1) {
+        decompressed = zlib.inflateSync(buf);
+      } else if (encoding.indexOf('br') !== -1) {
+        decompressed = zlib.brotliDecompressSync(buf);
+      }
+    } else if (isBinary) {
+      // Heuristic: try brotli if server is cloudflare
+      if (server.toLowerCase().indexOf('cloudflare') !== -1) {
+        try {
+          decompressed = zlib.brotliDecompressSync(buf);
+        } catch (e) {
+          // fallback below
+        }
+      }
+      // Try gzip as fallback
+      if (!decompressed) {
+        try {
+          decompressed = zlib.gunzipSync(buf);
+        } catch (e) {}
+      }
+      // Try deflate as fallback
+      if (!decompressed) {
+        try {
+          decompressed = zlib.inflateSync(buf);
+        } catch (e) {}
+      }
+      // As last resort, try brotli if not already tried
+      if (!decompressed && server.toLowerCase().indexOf('cloudflare') === -1) {
+        try {
+          decompressed = zlib.brotliDecompressSync(buf);
+        } catch (e) {
+          // fallback below
+        }
+      }
+    }
+    if (decompressed) {
+      // Try to return as string if possible
+      return decompressed.toString('utf8');
+    }
+  } catch (err) {
+    // fallback below
+  }
+  // If not decompressed, return original
+  return body;
+}
 
 var logMessage = function (debug, functionName, message, details) {
   if (debug) {
@@ -18,8 +100,7 @@ var logMessage = function (debug, functionName, message, details) {
           finalMessage = message + '\n' + JSON.stringify(details);
         }
       }
-    } catch (err) {
-    }
+    } catch (err) {}
     console.log('MOESIF: [' + functionName + '] ' + finalMessage);
   }
 };
@@ -38,11 +119,7 @@ function _hashSensitive(jsonBody, debug) {
       if (itemType === 'number' || itemType === 'string') {
         var creditCardCheck = isCreditCard.number('' + item);
         if (creditCardCheck.isValid) {
-          logMessage(
-            debug,
-            'hashSensitive',
-            'looks like a credit card, performing hash.'
-          );
+          logMessage(debug, 'hashSensitive', 'looks like a credit card, performing hash.');
           return hash(item).toString();
         }
       }
@@ -58,24 +135,13 @@ function _hashSensitive(jsonBody, debug) {
       var innerVal = jsonBody[key];
       var innerValType = typeof innerVal;
 
-      if (
-        key.toLowerCase().indexOf('password') !== -1 &&
-        typeof innerVal === 'string'
-      ) {
-        logMessage(
-          debug,
-          'hashSensitive',
-          'key is password, so hashing the value.'
-        );
+      if (key.toLowerCase().indexOf('password') !== -1 && typeof innerVal === 'string') {
+        logMessage(debug, 'hashSensitive', 'key is password, so hashing the value.');
         returnObject[key] = hash(jsonBody[key]).toString();
       } else if (innerValType === 'number' || innerValType === 'string') {
         var creditCardCheck = isCreditCard.number('' + innerVal);
         if (creditCardCheck.isValid) {
-          logMessage(
-            debug,
-            'hashSensitive',
-            'a field looks like credit card, performing hash.'
-          );
+          logMessage(debug, 'hashSensitive', 'a field looks like credit card, performing hash.');
           returnObject[key] = hash(jsonBody[key]).toString();
         } else {
           returnObject[key] = _hashSensitive(innerVal, debug);
@@ -97,10 +163,10 @@ function _getUrlFromRequestOptions(options, request) {
     options = url.parse(options);
   } else {
     // Avoid modifying the original options object.
-    let originalOptions = options;
+    var originalOptions = options;
     options = {};
     if (originalOptions) {
-      Object.keys(originalOptions).forEach((key) => {
+      Object.keys(originalOptions).forEach(function (key) {
         options[key] = originalOptions[key];
       });
     }
@@ -130,8 +196,7 @@ function _getUrlFromRequestOptions(options, request) {
   }
 
   // Mix in default values used by http.request and others
-  options.protocol =
-    options.protocol || (request.agent && request.agent.protocol) || undefined;
+  options.protocol = options.protocol || (request.agent && request.agent.protocol) || undefined;
   options.hostname = options.hostname || 'localhost';
 
   return url.format(options);
@@ -156,15 +221,15 @@ function isPlainObject(value) {
   if (Object.prototype.toString.call(value) !== '[object Object]') {
     return false;
   }
-  const prototype = Object.getPrototypeOf(value);
+  var prototype = Object.getPrototypeOf(value);
   return prototype === null || prototype === Object.prototype;
 }
 
 function isPlainObjectOrPrimitive(value) {
   if (isPlainObject(value)) {
     return true;
   }
-  const type = typeof value;
+  var type = typeof value;
   return (
     type === 'number' ||
     type === 'boolean' ||
@@ -184,11 +249,7 @@ function _safeJsonParse(body) {
           transferEncoding: undefined,
         };
       }
-      if (
-        Array.isArray(body) &&
-        body.every &&
-        body.every(isPlainObjectOrPrimitive)
-      ) {
+      if (Array.isArray(body) && body.every && body.every(isPlainObjectOrPrimitive)) {
         return {
           body: body,
           transferEncoding: undefined,
@@ -241,6 +302,7 @@ function getRequestHeaders(requestOptions, request) {
   return {};
 }
 
+// this is mostly for outgoing data capture.
 function _getEventModelFromRequestAndResponse(
   requestOptions,
   request,
@@ -260,17 +322,23 @@ function _getEventModelFromRequestAndResponse(
   logData.request.headers = getRequestHeaders(requestOptions, request);
   logData.request.time = requestTime;
 
-  if (requestBody) {
-    var isReqBodyMaybeJson = _startWithJson(requestBody);
+  // Decompress requestBody if needed
+  var reqHeaders = logData.request.headers || {};
+  var decompressedRequestBody = requestBody
+    ? decompressIfNeeded(requestBody, reqHeaders)
+    : requestBody;
+
+  if (decompressedRequestBody) {
+    var isReqBodyMaybeJson = _startWithJson(decompressedRequestBody);
 
     if (isReqBodyMaybeJson) {
-      var parsedReqBody = _safeJsonParse(requestBody);
+      var parsedReqBody = _safeJsonParse(decompressedRequestBody);
 
       logData.request.transferEncoding = parsedReqBody.transferEncoding;
       logData.request.body = parsedReqBody.body;
     } else {
       logData.request.transferEncoding = 'base64';
-      logData.request.body = _bodyToBase64(requestBody);
+      logData.request.body = _bodyToBase64(decompressedRequestBody);
     }
   }
 
@@ -279,17 +347,23 @@ function _getEventModelFromRequestAndResponse(
   logData.response.status = (response && (response.statusCode || response.status)) || 599;
   logData.response.headers = assign({}, (response && response.headers) || {});
 
-  if (responseBody) {
-    var isResBodyMaybeJson = _startWithJson(responseBody);
+  // Decompress responseBody if needed
+  var resHeaders = logData.response.headers || {};
+  var decompressedResponseBody = responseBody
+    ? decompressIfNeeded(responseBody, resHeaders)
+    : responseBody;
+
+  if (decompressedResponseBody) {
+    var isResBodyMaybeJson = _startWithJson(decompressedResponseBody);
 
     if (isResBodyMaybeJson) {
-      var parsedResBody = _safeJsonParse(responseBody);
+      var parsedResBody = _safeJsonParse(decompressedResponseBody);
 
       logData.response.transferEncoding = parsedResBody.transferEncoding;
       logData.response.body = parsedResBody.body;
     } else {
       logData.response.transferEncoding = 'base64';
-      logData.response.body = _bodyToBase64(responseBody);
+      logData.response.body = _bodyToBase64(decompressedResponseBody);
     }
   }
 
@@ -302,10 +376,7 @@ function isJsonHeader(msg) {
     if (headers['content-encoding']) {
       return false;
     }
-    if (
-      headers['content-type'] &&
-      headers['content-type'].indexOf('json') >= 0
-    ) {
+    if (headers['content-type'] && headers['content-type'].indexOf('json') >= 0) {
       return true;
     }
   }
@@ -314,7 +385,7 @@ function isJsonHeader(msg) {
 
 function approximateObjectSize(obj) {
   try {
-    const str = JSON.stringify(obj);
+    var str = JSON.stringify(obj);
     return str.length;
   } catch (err) {
     return 0;
@@ -347,9 +418,7 @@ function appendChunk(buf, chunk) {
       }
     } else if (typeof chunk === 'string') {
       try {
-        return buf
-          ? Buffer.concat([buf, Buffer.from(chunk)])
-          : Buffer.from(chunk);
+        return buf ? Buffer.concat([buf, Buffer.from(chunk)]) : Buffer.from(chunk);
       } catch (err) {
         return buf;
       }
@@ -401,14 +470,14 @@ function getReqHeaders(req) {
 }
 
 function generateUUIDv4() {
-  let timeNow = new Date().getTime();  // Current time in milliseconds
-  let timeRandom = timeNow + Math.random();  // Combine time and random number
-
-  return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
-      let r = (timeRandom + Math.random() * 16) % 16 | 0;  // Mix in timeRandom for extra entropy
-      timeRandom = Math.floor(timeRandom / 16);
-      let v = c === 'x' ? r : (r & 0x3 | 0x8);  // Handle the fixed bits for UUIDv4
-      return v.toString(16);
+  let timeNow = new Date().getTime(); // Current time in milliseconds
+  let timeRandom = timeNow + Math.random(); // Combine time and random number
+
+  return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
+    let r = (timeRandom + Math.random() * 16) % 16 | 0; // Mix in timeRandom for extra entropy
+    timeRandom = Math.floor(timeRandom / 16);
+    let v = c === 'x' ? r : (r & 0x3) | 0x8; // Handle the fixed bits for UUIDv4
+    return v.toString(16);
   });
 }
 
@@ -428,4 +497,5 @@ module.exports = {
   ensureToString: ensureToString,
   getReqHeaders: getReqHeaders,
   generateUUIDv4: generateUUIDv4,
+  decompressIfNeeded: decompressIfNeeded,
 };

lib/index.js

@@ -111,7 +111,7 @@ function makeMoesifMiddleware(options) {
    * @type {string}
    */
   config.ApplicationId = options.applicationId || options.ApplicationId;
-  config.UserAgent = 'moesif-nodejs/' + '3.9.1';
+  config.UserAgent = 'moesif-nodejs/' + '3.10.0';
   config.BaseUri = options.baseUri || options.BaseUri || config.BaseUri;
   // default retry to 1.
   config.retry = isNil(options.retry) ? 1 : options.retry;

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "moesif-nodejs",
-  "version": "3.9.1",
+  "version": "3.10.0",
   "description": "Monitoring agent to log API calls to Moesif for deep API analytics",
   "main": "lib/index.js",
   "typings": "dist/index.d.ts",