Gzip compression (#167)

Author: rrooij

docs/api/woqlClient.js.md

@@ -644,6 +644,7 @@ Common request dispatch function
 | apiUrl | <code>string</code> | the server call endpoint |
 | [payload] | <code>object</code> | the post body |
 | [getDataVersion] | <code>boolean</code> | If true return response with data version |
+| [compress] | <code>boolean</code> | If true, compress the data if it is bigger than 1024 bytes |
 
 
 ## generateCommitInfo

lib/dispatchRequest.js

@@ -1,4 +1,5 @@
 /* eslint-disable camelcase */
+const pako = require('pako');
 const axiosInstance = require('./axiosInstance');
 const UTILS = require('./utils');
 const CONST = require('./const');
@@ -38,9 +39,10 @@ function getResultWithDataVersion(response) {
  * @param {object} payload data to be transmitted to endpoint
  * @param {string} key optional basic auth string to be passed
  * @param {object} customHeaders the unique reqID
+ * @param {boolean} compress If true, compress the data with gzip if its size is bigger than 1024
  */
 // eslint-disable-next-line max-len
-function DispatchRequest(url, action, payload, local_auth, remote_auth = null, customHeaders = null, getDataVersion = false) {
+function DispatchRequest(url, action, payload, local_auth, remote_auth = null, customHeaders = null, getDataVersion = false, compress = false) {
   /*
      *CORS is only required when trying to fetch data from a browser,
      *as browsers by default will block requests to different origins
@@ -102,7 +104,7 @@ function DispatchRequest(url, action, payload, local_auth, remote_auth = null, c
   }
 
   switch (action) {
-    case CONST.DELETE:
+    case CONST.DELETE: {
       if (payload) {
         options.headers = options.headers ? options.headers : {};
         options.headers['Content-Type'] = 'application/json; charset=utf-8';
@@ -116,7 +118,8 @@ function DispatchRequest(url, action, payload, local_auth, remote_auth = null, c
           if (err.response && err.response.data) e.data = err.response.data;
           throw e;
         });
-    case CONST.HEAD:
+    }
+    case CONST.HEAD: {
       return axiosInstance
         .head(url, options)
         .then((response) => (getDataVersion ? getResultWithDataVersion(response) : response.data))
@@ -127,7 +130,8 @@ function DispatchRequest(url, action, payload, local_auth, remote_auth = null, c
           }
           throw e;
         });
-    case CONST.GET:
+    }
+    case CONST.GET: {
       if (payload) {
         const ext = UTILS.URIEncodePayload(payload);
         // eslint-disable-next-line no-param-reassign
@@ -143,8 +147,9 @@ function DispatchRequest(url, action, payload, local_auth, remote_auth = null, c
           }
           throw e;
         });
+    }
     case CONST.ADD_CSV:
-    case CONST.INSERT_TRIPLES:
+    case CONST.INSERT_TRIPLES: {
       options.headers = options.headers ? options.headers : {};
       options.headers['Content-Type'] = 'application/form-data; charset=utf-8';
       return axiosInstance
@@ -155,32 +160,49 @@ function DispatchRequest(url, action, payload, local_auth, remote_auth = null, c
           if (err.response && err.response.data) e.data = err.response.data;
           throw e;
         });
-    case CONST.PUT:
+    }
+    case CONST.PUT: {
       options.headers = options.headers ? options.headers : {};
       options.headers['Content-Type'] = 'application/json; charset=utf-8';
+      let compressedContent = null;
+      const jsonString = JSON.stringify(payload);
+
+      if (jsonString.length > 1024 && compress) {
+        options.headers['Content-Encoding'] = 'gzip';
+        compressedContent = pako.gzip(jsonString);
+      }
       return axiosInstance
-        .put(url, payload, options)
+        .put(url, compressedContent || payload, options)
         .then((response) => (getDataVersion ? getResultWithDataVersion(response) : response.data))
         .catch((err) => {
           const e = new Error(ErrorMessage.getAPIErrorMessage(url, options, err));
           if (err.response && err.response.data) e.data = err.response.data;
           throw e;
         });
-    case CONST.QUERY_DOCUMENT:
+    }
+    case CONST.QUERY_DOCUMENT: {
       options.headers = options.headers ? options.headers : {};
       options.headers['X-HTTP-Method-Override'] = 'GET';
-    // eslint-disable-next-line no-fallthrough
-    default:
+      // eslint-disable-next-line no-fallthrough
+    }
+    default: {
+      let compressedContentPost = null;
       options.headers = options.headers ? options.headers : {};
       options.headers['Content-Type'] = 'application/json; charset=utf-8';
+      const jsonStringPost = JSON.stringify(payload);
+      if (jsonStringPost.length > 1024 && compress) {
+        options.headers['Content-Encoding'] = 'gzip';
+        compressedContentPost = pako.gzip(jsonStringPost);
+      }
       return axiosInstance
-        .post(url, payload, options)
+        .post(url, compressedContentPost || payload, options)
         .then((response) => (getDataVersion ? getResultWithDataVersion(response) : response.data))
         .catch((err) => {
           const e = new Error(ErrorMessage.getAPIErrorMessage(url, options, err));
           if (err.response && err.response.data) e.data = err.response.data;
           throw e;
         });
+    }
   }
 }
 

lib/woqlClient.js

@@ -842,9 +842,16 @@ WOQLClient.prototype.clonedb = function (cloneSource, newDbId, orgId) {
  * @property {string} apiUrl - the server call endpoint
  * @property {object} [payload] - the post body
  * @property {boolean} [getDataVersion] - If true return response with data version
+ * @property {boolean} [compress] - If true, compress the data if it is bigger than 1024 bytes
  * @returns {Promise}  A promise that returns the call response object, or an Error if rejected.
  */
-WOQLClient.prototype.dispatch = function (action, apiUrl, payload, getDataVersion) {
+WOQLClient.prototype.dispatch = function (
+  action,
+  apiUrl,
+  payload,
+  getDataVersion,
+  compress = false,
+) {
   if (!apiUrl) {
     return Promise.reject(
       new Error(
@@ -881,6 +888,7 @@ WOQLClient.prototype.dispatch = function (action, apiUrl, payload, getDataVersio
     this.remoteAuth(),
     this.customHeaders(),
     getDataVersion,
+    compress,
   );
   // }
 };
@@ -983,15 +991,14 @@ WOQLClient.prototype.updateDatabase = function (dbDoc) {
  * )
  */
 
-WOQLClient.prototype.addDocument = function (json, params, dbId, message = 'add a new document', lastDataVersion = '', getDataVersion = false) {
+WOQLClient.prototype.addDocument = function (json, params, dbId, message = 'add a new document', lastDataVersion = '', getDataVersion = false, compress = false) {
   if (dbId) {
     this.db(dbId);
   }
 
   if (typeof lastDataVersion === 'string' && lastDataVersion !== '') {
     this.customHeaders({ 'TerminusDB-Data-Version': lastDataVersion });
   }
-
   const docParams = params || {};
   docParams.author = this.author();
   docParams.message = message;
@@ -1000,6 +1007,7 @@ WOQLClient.prototype.addDocument = function (json, params, dbId, message = 'add
     this.connectionConfig.documentURL(docParams),
     json,
     getDataVersion,
+    compress,
   );
 };
 
@@ -1202,7 +1210,7 @@ WOQLClient.prototype.getDocument = function (params, dbId, branch, lastDataVersi
     );
  */
 
-WOQLClient.prototype.updateDocument = function (json, params, dbId, message = 'update document', lastDataVersion = '', getDataVersion = false) {
+WOQLClient.prototype.updateDocument = function (json, params, dbId, message = 'update document', lastDataVersion = '', getDataVersion = false, compress = false) {
   const docParams = params || {};
   docParams.author = this.author();
   docParams.message = message;
@@ -1217,6 +1225,7 @@ WOQLClient.prototype.updateDocument = function (json, params, dbId, message = 'u
     this.connectionConfig.documentURL(docParams),
     json,
     getDataVersion,
+    compress,
   );
 };
 

package-lock.json

@@ -26,6 +26,7 @@
     "follow-redirects": ">=1.14.8",
     "form-data": "^4.0.0",
     "node-forge": ">=1.0.0",
+    "pako": "^2.0.4",
     "pathval": ">=1.1.1",
     "underscore": ">=1.13.2"
   },