Skip to content

Commit 0cb391c

Browse files
anu3990anu3990
committed
DEVEXP-596 : Configure gZIP option
1 parent 1ca1b35 commit 0cb391c

File tree

12 files changed

+536
-838
lines changed

12 files changed

+536
-838
lines changed

‎README.md‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,10 @@ const db = marklogic.createDatabaseClient({
4545
database: 'Documents',
4646
user: 'admin',
4747
password: 'admin',
48-
authType: 'DIGEST'
48+
authType: 'DIGEST',
49+
// enableGzippedResponses is optional and can be set to true in order to request MarkLogic to compress the response for better performance,
50+
// the client will automatically decompress the response before it returns a value.
51+
enableGzippedResponses: true
4952
});
5053

5154
// For MarkLogic Cloud
@@ -56,7 +59,10 @@ const db = marklogic.createDatabaseClient({
5659
// basePath is optional.
5760
basePath: '/marklogic/test',
5861
// accessTokenDuration (in seconds) is optional and can be used to customize the expiration of the access token.
59-
accessTokenDuration: 10
62+
accessTokenDuration: 10,
63+
// enableGzippedResponses is optional and can be set to true in order to request MarkLogic to compress the response for better performance,
64+
// the client will automatically decompress the response before it returns a value.
65+
enableGzippedResponses: true
6066
});
6167

6268
db.createCollection(

‎etc/test-config.js‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -55,42 +55,48 @@ module.exports = {
5555
port: restPort,
5656
user: restAdminUser,
5757
password: restAdminPassword,
58-
authType: restAuthType
58+
authType: restAuthType,
59+
enableGzippedResponses: true
5960
},
6061
restReaderConnection: {
6162
host: testHost,
6263
port: restPort,
6364
user: restReaderUser,
6465
password: restReaderPassword,
65-
authType: restAuthType
66+
authType: restAuthType,
67+
enableGzippedResponses: true
6668
},
6769
restWriterConnection: {
6870
host: testHost,
6971
port: restPort,
7072
user: restWriterUser,
7173
password: restWriterPassword,
72-
authType: restAuthType
74+
authType: restAuthType,
75+
enableGzippedResponses: true
7376
},
7477
restEvaluatorConnection: {
7578
host: testHost,
7679
port: restPort,
7780
user: restEvaluatorUser,
7881
password: restEvaluatorPassword,
79-
authType: restAuthType
82+
authType: restAuthType,
83+
enableGzippedResponses: true
8084
},
8185
restTemporalConnection: {
8286
host: testHost,
8387
port: restPort,
8488
user: restTemporalUser,
8589
password: restTemporalPassword,
86-
authType: restAuthType
90+
authType: restAuthType,
91+
enableGzippedResponses: true
8792
},
8893
manageAdminConnection: {
8994
host: testHost,
9095
port: managePort,
9196
user: restAdminUser,
9297
password: restAdminPassword,
93-
authType: manageAuthType
98+
authType: manageAuthType,
99+
enableGzippedResponses: true
94100
},
95101
restSslConnection: {
96102
host: testHost,
@@ -99,30 +105,34 @@ module.exports = {
99105
password: restAdminPassword,
100106
authType: 'BASIC',
101107
rejectUnauthorized: false,
102-
ssl: true
108+
ssl: true,
109+
enableGzippedResponses: true
103110
},
104111
testConnection: {
105112
host: testHost,
106113
port: restPort,
107114
user: testUser,
108115
password: testPassword,
109116
authType: restAuthType,
110-
rejectUnauthorized: false
117+
rejectUnauthorized: false,
118+
enableGzippedResponses: true
111119
},
112120
tdeConnection: {
113121
host: testHost,
114122
port: restPort,
115123
user: tdeUser,
116124
password: tdePassword,
117125
authType: restAuthType,
118-
rejectUnauthorized: false
126+
rejectUnauthorized: false,
127+
enableGzippedResponses: true
119128
},
120129
restWriterConnectionWithBasePath: {
121130
host: testHost,
122131
port: restPort,
123132
user: restWriterUser,
124133
password: restWriterPassword,
125134
authType: restAuthType,
126-
basePath: ''
135+
basePath: '',
136+
enableGzippedResponses: true
127137
}
128138
};

‎lib/marklogic.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -728,7 +728,8 @@ function initClient(client, inputParams) {
728728
if(inputParams.authType && inputParams.authType.toString().toLowerCase() === 'cloud' && !isSSL){
729729
isSSL = true;
730730
}
731-
var keys = ['host', 'port', 'database', 'user', 'password', 'authType', 'token', 'basePath','apiKey','accessTokenDuration'];
731+
var keys = ['host', 'port', 'database', 'user', 'password', 'authType', 'token', 'basePath','apiKey','accessTokenDuration',
732+
'enableGzippedResponses'];
732733
if(isSSL) {
733734
keys.push('ca', 'cert', 'ciphers', 'clientCertEngine', 'crl', 'dhparam', 'ecdhCurve', 'honorCipherOrder', 'key', 'passphrase',
734735
'pfx', 'rejectUnauthorized', 'secureOptions', 'secureProtocol', 'servername', 'sessionIdContext', 'highWaterMark');

‎lib/requester.js‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,9 @@ function startRequest(operation) {
105105
headers['X-Error-Accept'] = 'application/json';
106106
headers['ML-Agent-ID'] = 'nodejs'; // Telemetry header
107107

108+
if(options.enableGzippedResponses) {
109+
headers['Accept-Encoding'] = 'gzip';
110+
}
108111
var started = null;
109112
var operationResultPromise = null;
110113

‎lib/responder.js‎

Lines changed: 85 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ var Dicer = require('@fastify/busboy/deps/dicer/lib/Dicer');
2020
var through2 = require('through2');
2121
var mlutil = require('./mlutil.js');
2222
const requester = require("./requester");
23+
const {createGunzip} = require('zlib');
2324

2425
/**
2526
* Handle response from the REST API based on response and operation
@@ -169,11 +170,17 @@ CSVDispatcher.prototype.promise = function dispatchCSVPromise(
169170
};
170171

171172
var isString = operation.copyResponseHeaders(response);
172-
173-
response.pipe(concatStream(
174-
{encoding: (isString ? 'string' : 'buffer')},
175-
collectObject
176-
));
173+
if(isResponseGzipped(response.headers)) {
174+
response.pipe(createGunzip()).pipe(concatStream(
175+
{encoding: (isString ? 'string' : 'buffer')},
176+
collectObject
177+
));
178+
} else {
179+
response.pipe(concatStream(
180+
{encoding: (isString ? 'string' : 'buffer')},
181+
collectObject
182+
));
183+
}
177184
};
178185
CSVDispatcher.prototype.chunkedStream = function dispatchCSVChunkedStream(
179186
contentType, response
@@ -185,7 +192,11 @@ CSVDispatcher.prototype.chunkedStream = function dispatchCSVChunkedStream(
185192
// HTTP response gives a chunked stream to begin with
186193
// .stream('chunked') creates through2 stream (writable and readable)
187194
// Simply pipe HTTP response to the through2 stream
188-
response.pipe(operation.outputStream);
195+
if(isResponseGzipped(response.headers)) {
196+
response.pipe(createGunzip()).pipe(operation.outputStream);
197+
} else {
198+
response.pipe(operation.outputStream);
199+
}
189200
};
190201
function JSONSeqDispatcher(operation) {
191202
if (!(this instanceof JSONSeqDispatcher)) {
@@ -229,8 +240,11 @@ JSONSeqDispatcher.prototype.promise = function dispatchJSONSeqPromise(
229240
})
230241
.on('invalid', errorListener)
231242
.on('finish', finishListener);
232-
233-
response.pipe(parser);
243+
if(isResponseGzipped(response.headers)) {
244+
response.pipe(createGunzip()).pipe(parser);
245+
} else {
246+
response.pipe(parser);
247+
}
234248

235249
};
236250
JSONSeqDispatcher.prototype.sequenceStream = function dispatchJSONSeqSequenceStream(
@@ -326,9 +340,11 @@ JSONSeqDispatcher.prototype.stream = function dispatchJSONSeqStream(
326340
response.on('end', responseEndListener);
327341

328342
outputStream.on('drain', drainListener);
329-
330-
response.pipe(parser);
331-
343+
if(isResponseGzipped(response.headers)) {
344+
response.pipe(createGunzip()).pipe(parser);
345+
} else {
346+
response.pipe(parser);
347+
}
332348
};
333349

334350
function BodyDispatcher(operation) {
@@ -378,11 +394,20 @@ BodyDispatcher.prototype.promise = function dispatchBodyPromise(
378394

379395
var isString = operation.copyResponseHeaders(response);
380396

381-
// concatStream accumulates response with callback
382-
response.pipe(concatStream(
383-
{encoding: (isString ? 'string' : 'buffer')},
384-
collectObject
385-
));
397+
if(isResponseGzipped(response.headers)) {
398+
const gunzip = createGunzip();
399+
response.pipe(gunzip).pipe(concatStream(
400+
{encoding: (isString ? 'string' : 'buffer')},
401+
collectObject
402+
));
403+
404+
} else {
405+
// concatStream accumulates response with callback
406+
response.pipe(concatStream(
407+
{encoding: (isString ? 'string' : 'buffer')},
408+
collectObject
409+
));
410+
}
386411
};
387412
BodyDispatcher.prototype.chunkedStream = function dispatchBodyChunkedStream(
388413
contentType, response
@@ -394,7 +419,11 @@ BodyDispatcher.prototype.chunkedStream = function dispatchBodyChunkedStream(
394419
// HTTP response gives a chunked stream to begin with
395420
// .stream('chunked') creates through2 stream (writable and readable)
396421
// Simply pipe HTTP response to the through2 stream
397-
response.pipe(operation.outputStream);
422+
if(isResponseGzipped(response.headers)) {
423+
response.pipe(createGunzip()).pipe(operation.outputStream);
424+
} else {
425+
response.pipe(operation.outputStream);
426+
}
398427
};
399428
BodyDispatcher.prototype.objectStream = function dispatchBodyObjectStream(
400429
contentType, response
@@ -416,11 +445,17 @@ BodyDispatcher.prototype.objectStream = function dispatchBodyObjectStream(
416445
};
417446

418447
var isString = operation.copyResponseHeaders(response);
419-
420-
response.pipe(concatStream(
421-
{encoding: (isString ? 'string' : 'buffer')},
422-
collectObject
423-
));
448+
if(isResponseGzipped(response.headers)) {
449+
response.pipe(createGunzip()).pipe(concatStream(
450+
{encoding: (isString ? 'string' : 'buffer')},
451+
collectObject
452+
));
453+
} else {
454+
response.pipe(concatStream(
455+
{encoding: (isString ? 'string' : 'buffer')},
456+
collectObject
457+
));
458+
}
424459
};
425460

426461
// Multipart cases similar to the above, but with multiple objects
@@ -475,14 +510,17 @@ MultipartDispatcher.prototype.promise = function dispatchMultipartPromise(
475510
const errorListenerCheck = (operation.options.headers['Accept'] === 'application/json' && (operation.name.includes('rows') || operation.name.includes('query') || operation.name.includes('/v1/rows')));
476511

477512
if(errorListenerCheck) {
478-
response.setEncoding("utf8");
513+
if(response.headers['content-encoding']!=='gzip'){
514+
response.setEncoding("utf8");
515+
}
516+
517+
const multipartResponse = (isResponseGzipped(response.headers))?response.pipe(createGunzip()):response;
479518
let chunks = '';
480519

481-
response.on('data', function(data) {
520+
multipartResponse.on('data', function(data) {
482521
chunks += data;
483522
});
484-
485-
response.on('end', function() {
523+
multipartResponse.on('end', function() {
486524
response.pipe(concatStream(
487525
{encoding: 'json'},
488526
() => {
@@ -491,7 +529,6 @@ MultipartDispatcher.prototype.promise = function dispatchMultipartPromise(
491529
resolvedPromise(operation, operation.resolve);
492530
}
493531
));
494-
495532
});
496533
return;
497534
}
@@ -595,8 +632,11 @@ MultipartDispatcher.prototype.promise = function dispatchMultipartPromise(
595632
parser.on('finish', parseFinishListener);
596633

597634
response.on('end', responseEndListener);
598-
599-
response.pipe(parser);
635+
if(isResponseGzipped(response.headers)) {
636+
response.pipe(createGunzip()).pipe(parser);
637+
} else {
638+
response.pipe(parser);
639+
}
600640
};
601641
MultipartDispatcher.prototype.chunkedStream = function dispatchMultipartChunkedStream(
602642
boundary, response
@@ -661,10 +701,12 @@ MultipartDispatcher.prototype.chunkedStream = function dispatchMultipartChunkedS
661701
parser.on('part', partListener);
662702
parser.on('error', errorListener);
663703
parser.on('finish', parseFinishListener);
664-
665704
response.on('end', responseEndListener);
666-
667-
response.pipe(parser);
705+
if(isResponseGzipped(response.headers)) {
706+
response.pipe(createGunzip()).pipe(parser);
707+
} else {
708+
response.pipe(parser);
709+
}
668710
};
669711
MultipartDispatcher.prototype.objectStream = function dispatchMultipartObjectStream(
670712
boundary, response
@@ -853,9 +895,11 @@ MultipartDispatcher.prototype.objectStream = function dispatchMultipartObjectStr
853895
response.on('end', responseEndListener);
854896

855897
operation.outputStream.on('drain', drainListener);
856-
857-
response.pipe(parser);
858-
898+
if(isResponseGzipped(response.headers)) {
899+
response.pipe(createGunzip()).pipe(parser);
900+
} else {
901+
response.pipe(parser);
902+
}
859903
};
860904

861905
/* Note: Dicer appears to read ahead.
@@ -996,7 +1040,8 @@ function isResponseStatusOkay(response) {
9961040
var clientError = operation.makeError(errMsg);
9971041
clientError.statusCode = statusCode;
9981042
if (statusCode >= 400) {
999-
response.pipe(concatStream(
1043+
let errorResponse = (isResponseGzipped(response.headers))?response.pipe(createGunzip()):response;
1044+
errorResponse.pipe(concatStream(
10001045
{encoding: 'string'},
10011046
function errorBodyDispatcher(body) {
10021047
if (body.length > 0) {
@@ -1212,6 +1257,10 @@ function operationErrorListener(error) {
12121257
resolvedPromise(operation, operation.resolve);
12131258
}
12141259

1260+
function isResponseGzipped(headers){
1261+
return (headers['content-encoding'] === 'gzip' || headers['Content-Encoding'] === 'gzip');
1262+
}
1263+
12151264
module.exports = {
12161265
operationErrorListener: operationErrorListener,
12171266
operationResultPromise: operationResultPromise,

0 commit comments

Comments
 (0)