Skip to content

Commit 16a1d69

Browse files
author
Ruben Bridgewater
committed
Move parsers into seperate module and improve js parser performance
1 parent 0207163 commit 16a1d69

File tree

10 files changed

+91
-650
lines changed

10 files changed

+91
-650
lines changed

README.md

Lines changed: 35 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -653,43 +653,43 @@ hiredis parser (Lenovo T450s i7-5600U):
653653

654654
```
655655
Client count: 1, node version: 4.2.2, server version: 3.0.3, parser: hiredis
656-
PING, 1/1 min/max/avg/p95: 0/ 2/ 0.02/ 0.00 2501ms total, 47503.80 ops/sec
657-
PING, batch 50/1 min/max/avg/p95: 0/ 2/ 0.09/ 1.00 2501ms total, 529668.13 ops/sec
658-
SET 4B str, 1/1 min/max/avg/p95: 0/ 2/ 0.02/ 0.00 2501ms total, 41900.04 ops/sec
659-
SET 4B str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.14/ 1.00 2501ms total, 354658.14 ops/sec
660-
SET 4B buf, 1/1 min/max/avg/p95: 0/ 4/ 0.04/ 0.00 2501ms total, 23499.00 ops/sec
661-
SET 4B buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.31/ 1.00 2501ms total, 159836.07 ops/sec
662-
GET 4B str, 1/1 min/max/avg/p95: 0/ 4/ 0.02/ 0.00 2501ms total, 43489.80 ops/sec
663-
GET 4B str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.11/ 1.00 2501ms total, 444202.32 ops/sec
664-
GET 4B buf, 1/1 min/max/avg/p95: 0/ 3/ 0.02/ 0.00 2501ms total, 38561.38 ops/sec
665-
GET 4B buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.11/ 1.00 2501ms total, 452139.14 ops/sec
666-
SET 4KiB str, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 32990.80 ops/sec
667-
SET 4KiB str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.34/ 1.00 2501ms total, 146161.54 ops/sec
668-
SET 4KiB buf, 1/1 min/max/avg/p95: 0/ 1/ 0.04/ 0.00 2501ms total, 23294.28 ops/sec
669-
SET 4KiB buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.36/ 1.00 2501ms total, 137584.97 ops/sec
670-
GET 4KiB str, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 36350.66 ops/sec
671-
GET 4KiB str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 155157.94 ops/sec
672-
GET 4KiB buf, 1/1 min/max/avg/p95: 0/ 4/ 0.02/ 0.00 2501ms total, 39776.49 ops/sec
673-
GET 4KiB buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 155457.82 ops/sec
674-
INCR, 1/1 min/max/avg/p95: 0/ 3/ 0.02/ 0.00 2501ms total, 43972.41 ops/sec
675-
INCR, batch 50/1 min/max/avg/p95: 0/ 1/ 0.12/ 1.00 2501ms total, 425809.68 ops/sec
676-
LPUSH, 1/1 min/max/avg/p95: 0/ 2/ 0.02/ 0.00 2501ms total, 38998.40 ops/sec
677-
LPUSH, batch 50/1 min/max/avg/p95: 0/ 4/ 0.14/ 1.00 2501ms total, 365013.99 ops/sec
678-
LRANGE 10, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 31879.25 ops/sec
679-
LRANGE 10, batch 50/1 min/max/avg/p95: 0/ 1/ 0.32/ 1.00 2501ms total, 153698.52 ops/sec
680-
LRANGE 100, 1/1 min/max/avg/p95: 0/ 4/ 0.06/ 0.00 2501ms total, 16676.13 ops/sec
681-
LRANGE 100, batch 50/1 min/max/avg/p95: 1/ 6/ 2.03/ 2.00 2502ms total, 24520.38 ops/sec
682-
SET 4MiB str, 1/1 min/max/avg/p95: 1/ 6/ 2.11/ 3.00 2502ms total, 472.82 ops/sec
683-
SET 4MiB str, batch 20/1 min/max/avg/p95: 85/ 112/ 94.93/ 109.60 2563ms total, 210.69 ops/sec
684-
SET 4MiB buf, 1/1 min/max/avg/p95: 1/ 8/ 2.02/ 3.00 2502ms total, 490.01 ops/sec
685-
SET 4MiB buf, batch 20/1 min/max/avg/p95: 37/ 52/ 39.48/ 46.75 2528ms total, 506.33 ops/sec
686-
GET 4MiB str, 1/1 min/max/avg/p95: 3/ 13/ 5.26/ 9.00 2504ms total, 190.10 ops/sec
687-
GET 4MiB str, batch 20/1 min/max/avg/p95: 70/ 106/ 89.36/ 103.75 2503ms total, 223.73 ops/sec
688-
GET 4MiB buf, 1/1 min/max/avg/p95: 3/ 11/ 5.04/ 8.15 2502ms total, 198.24 ops/sec
689-
GET 4MiB buf, batch 20/1 min/max/avg/p95: 70/ 105/ 88.07/ 103.00 2554ms total, 227.09 ops/sec
656+
PING, 1/1 min/max/avg: 0/ 2/ 0.02/ 2501ms total, 47503.80 ops/sec
657+
PING, batch 50/1 min/max/avg: 0/ 2/ 0.09/ 2501ms total, 529668.13 ops/sec
658+
SET 4B str, 1/1 min/max/avg: 0/ 2/ 0.02/ 2501ms total, 41900.04 ops/sec
659+
SET 4B str, batch 50/1 min/max/avg: 0/ 2/ 0.14/ 2501ms total, 354658.14 ops/sec
660+
SET 4B buf, 1/1 min/max/avg: 0/ 4/ 0.04/ 2501ms total, 23499.00 ops/sec
661+
SET 4B buf, batch 50/1 min/max/avg: 0/ 2/ 0.31/ 2501ms total, 159836.07 ops/sec
662+
GET 4B str, 1/1 min/max/avg: 0/ 4/ 0.02/ 2501ms total, 43489.80 ops/sec
663+
GET 4B str, batch 50/1 min/max/avg: 0/ 2/ 0.11/ 2501ms total, 444202.32 ops/sec
664+
GET 4B buf, 1/1 min/max/avg: 0/ 3/ 0.02/ 2501ms total, 38561.38 ops/sec
665+
GET 4B buf, batch 50/1 min/max/avg: 0/ 2/ 0.11/ 2501ms total, 452139.14 ops/sec
666+
SET 4KiB str, 1/1 min/max/avg: 0/ 2/ 0.03/ 2501ms total, 32990.80 ops/sec
667+
SET 4KiB str, batch 50/1 min/max/avg: 0/ 2/ 0.34/ 2501ms total, 146161.54 ops/sec
668+
SET 4KiB buf, 1/1 min/max/avg: 0/ 1/ 0.04/ 2501ms total, 23294.28 ops/sec
669+
SET 4KiB buf, batch 50/1 min/max/avg: 0/ 2/ 0.36/ 2501ms total, 137584.97 ops/sec
670+
GET 4KiB str, 1/1 min/max/avg: 0/ 2/ 0.03/ 2501ms total, 36350.66 ops/sec
671+
GET 4KiB str, batch 50/1 min/max/avg: 0/ 2/ 0.32/ 2501ms total, 155157.94 ops/sec
672+
GET 4KiB buf, 1/1 min/max/avg: 0/ 4/ 0.02/ 2501ms total, 39776.49 ops/sec
673+
GET 4KiB buf, batch 50/1 min/max/avg: 0/ 2/ 0.32/ 2501ms total, 155457.82 ops/sec
674+
INCR, 1/1 min/max/avg: 0/ 3/ 0.02/ 2501ms total, 43972.41 ops/sec
675+
INCR, batch 50/1 min/max/avg: 0/ 1/ 0.12/ 2501ms total, 425809.68 ops/sec
676+
LPUSH, 1/1 min/max/avg: 0/ 2/ 0.02/ 2501ms total, 38998.40 ops/sec
677+
LPUSH, batch 50/1 min/max/avg: 0/ 4/ 0.14/ 2501ms total, 365013.99 ops/sec
678+
LRANGE 10, 1/1 min/max/avg: 0/ 2/ 0.03/ 2501ms total, 31879.25 ops/sec
679+
LRANGE 10, batch 50/1 min/max/avg: 0/ 1/ 0.32/ 2501ms total, 153698.52 ops/sec
680+
LRANGE 100, 1/1 min/max/avg: 0/ 4/ 0.06/ 2501ms total, 16676.13 ops/sec
681+
LRANGE 100, batch 50/1 min/max/avg: 1/ 6/ 2.03/ 2502ms total, 24520.38 ops/sec
682+
SET 4MiB str, 1/1 min/max/avg: 1/ 6/ 2.11/ 2502ms total, 472.82 ops/sec
683+
SET 4MiB str, batch 20/1 min/max/avg: 85/ 112/ 94.93/ 2563ms total, 210.69 ops/sec
684+
SET 4MiB buf, 1/1 min/max/avg: 1/ 8/ 2.02/ 2502ms total, 490.01 ops/sec
685+
SET 4MiB buf, batch 20/1 min/max/avg: 37/ 52/ 39.48/ 2528ms total, 506.33 ops/sec
686+
GET 4MiB str, 1/1 min/max/avg: 3/ 13/ 5.26/ 2504ms total, 190.10 ops/sec
687+
GET 4MiB str, batch 20/1 min/max/avg: 70/ 106/ 89.36/ 2503ms total, 223.73 ops/sec
688+
GET 4MiB buf, 1/1 min/max/avg: 3/ 11/ 5.04/ 2502ms total, 198.24 ops/sec
689+
GET 4MiB buf, batch 20/1 min/max/avg: 70/ 105/ 88.07/ 2554ms total, 227.09 ops/sec
690690
```
691691

692-
The hiredis and js parser should most of the time be on the same level. But if you use Redis for big SUNION/SINTER/LRANGE/ZRANGE hiredis is significantly faster.
692+
The hiredis and js parser should most of the time be on the same level. But if you use Redis for big SUNION/SINTER/LRANGE/ZRANGE hiredis is faster.
693693
Therefor the hiredis parser is the default used in node_redis. To use `hiredis`, do:
694694

695695
npm install hiredis redis

benchmarks/multi_bench.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ function lpad(input, len, chr) {
4242

4343
metrics.Histogram.prototype.print_line = function () {
4444
var obj = this.printObj();
45-
return lpad(obj.min, 4) + '/' + lpad(obj.max, 4) + '/' + lpad(obj.mean.toFixed(2), 7) + '/' + lpad(obj.p95.toFixed(2), 7);
45+
return lpad(obj.min, 4) + '/' + lpad(obj.max, 4) + '/' + lpad(obj.mean.toFixed(2), 7);
4646
};
4747

4848
function Test(args) {
@@ -205,7 +205,7 @@ Test.prototype.print_stats = function () {
205205
var duration = Date.now() - this.test_start;
206206
totalTime += duration;
207207

208-
console.log('min/max/avg/p95: ' + this.command_latency.print_line() + ' ' + lpad(duration, 6) + 'ms total, ' +
208+
console.log('min/max/avg: ' + this.command_latency.print_line() + ' ' + lpad(duration, 6) + 'ms total, ' +
209209
lpad((this.commands_completed / (duration / 1000)).toFixed(2), 9) + ' ops/sec');
210210
};
211211

changelog.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,13 @@
11
Changelog
22
=========
33

4+
## v.2.5.0 - xx Dez, 2015
5+
6+
Features
7+
8+
- The parsers moved into the [redis-parser](https://github.com/NodeRedis/node-redis-parser) module and will be maintained in there from now on ([@BridgeAR](https://github.com/BridgeAR))
9+
- Improve js parser speed significantly for big SUNION/SINTER/LRANGE/ZRANGE ([@BridgeAR](https://github.com/BridgeAR))
10+
411
## v.2.4.2 - 27 Nov, 2015
512

613
Bugfixes

index.js

Lines changed: 42 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ var utils = require('./lib/utils');
88
var Queue = require('double-ended-queue');
99
var Command = require('./lib/command');
1010
var events = require('events');
11-
var parsers = [];
11+
var Parser = require('redis-parser');
1212
var commands = require('redis-commands');
1313
var connection_id = 0;
1414
var default_port = 6379;
@@ -18,17 +18,20 @@ function noop () {}
1818
function clone (obj) { return JSON.parse(JSON.stringify(obj || {})); }
1919
function debug (msg) { if (exports.debug_mode) { console.error(msg); } }
2020

21-
exports.debug_mode = /\bredis\b/i.test(process.env.NODE_DEBUG);
21+
function handle_detect_buffers_reply (reply, command, buffer_args) {
22+
if (buffer_args === false) {
23+
// If detect_buffers option was specified, then the reply from the parser will be a buffer.
24+
// If this command did not use Buffer arguments, then convert the reply to Strings here.
25+
reply = utils.reply_to_strings(reply);
26+
}
2227

23-
// Hiredis might not be installed
24-
try {
25-
parsers.push(require('./lib/parsers/hiredis'));
26-
} catch (err) {
27-
/* istanbul ignore next: won't be reached with tests */
28-
debug('Hiredis parser not installed.');
28+
if (command === 'hgetall') {
29+
reply = utils.reply_to_object(reply);
30+
}
31+
return reply;
2932
}
3033

31-
parsers.push(require('./lib/parsers/javascript'));
34+
exports.debug_mode = /\bredis\b/i.test(process.env.NODE_DEBUG);
3235

3336
function RedisClient (options) {
3437
// Copy the options so they are not mutated
@@ -69,6 +72,10 @@ function RedisClient (options) {
6972
console.warn('>> WARNING: You activated return_buffers and detect_buffers at the same time. The return value is always going to be a buffer.');
7073
options.detect_buffers = false;
7174
}
75+
if (options.detect_buffers) {
76+
// We only need to look at the arguments if we do not know what we have to return
77+
this.handle_reply = handle_detect_buffers_reply;
78+
}
7279
this.should_buffer = false;
7380
this.max_attempts = options.max_attempts | 0;
7481
this.command_queue = new Queue(); // Holds sent commands to de-pipeline them
@@ -83,13 +90,22 @@ function RedisClient (options) {
8390
this.closing = false;
8491
this.server_info = {};
8592
this.auth_pass = options.auth_pass;
86-
this.parser_module = null;
8793
this.selected_db = null; // Save the selected db here, used when reconnecting
8894
this.old_state = null;
8995
this.pipeline = 0;
9096
this.options = options;
91-
// Init parser once per instance
92-
this.init_parser();
97+
// Init parser
98+
var self = this;
99+
this.reply_parser = new Parser({
100+
returnReply: function (data) {
101+
self.return_reply(data);
102+
},
103+
returnError: function (data) {
104+
self.return_error(data);
105+
},
106+
returnBuffers: options.return_buffers || options.detect_buffers,
107+
name: options.parser
108+
});
93109
this.create_stream();
94110
}
95111
util.inherits(RedisClient, events.EventEmitter);
@@ -153,6 +169,13 @@ RedisClient.prototype.create_stream = function () {
153169
});
154170
};
155171

172+
RedisClient.prototype.handle_reply = function (reply, command) {
173+
if (command === 'hgetall') {
174+
reply = utils.reply_to_object(reply);
175+
}
176+
return reply;
177+
};
178+
156179
RedisClient.prototype.cork = noop;
157180
RedisClient.prototype.uncork = noop;
158181

@@ -300,39 +323,6 @@ RedisClient.prototype.on_connect = function () {
300323
}
301324
};
302325

303-
RedisClient.prototype.init_parser = function () {
304-
var self = this;
305-
306-
if (this.options.parser) {
307-
if (!parsers.some(function (parser) {
308-
if (parser.name === self.options.parser) {
309-
self.parser_module = parser;
310-
debug('Using parser module: ' + self.parser_module.name);
311-
return true;
312-
}
313-
})) {
314-
// Do not emit this error
315-
// This should take down the app if anyone made such a huge mistake or should somehow be handled in user code
316-
throw new Error("Couldn't find named parser " + self.options.parser + " on this system");
317-
}
318-
} else {
319-
debug('Using default parser module: ' + parsers[0].name);
320-
this.parser_module = parsers[0];
321-
}
322-
323-
// return_buffers sends back Buffers from parser to callback. detect_buffers sends back Buffers from parser, but
324-
// converts to Strings if the input arguments are not Buffers.
325-
this.reply_parser = new this.parser_module.Parser(self.options.return_buffers || self.options.detect_buffers);
326-
// Important: Only send results / errors async.
327-
// That way the result / error won't stay in a try catch block and catch user things
328-
this.reply_parser.send_error = function (data) {
329-
self.return_error(data);
330-
};
331-
this.reply_parser.send_reply = function (data) {
332-
self.return_reply(data);
333-
};
334-
};
335-
336326
RedisClient.prototype.on_ready = function () {
337327
var self = this;
338328

@@ -599,7 +589,7 @@ RedisClient.prototype.return_error = function (err) {
599589
err.command = command_obj.command;
600590
}
601591

602-
var match = err.message.match(utils.errCode);
592+
var match = err.message.match(utils.err_code);
603593
// LUA script could return user errors that don't behave like all other errors!
604594
if (match) {
605595
err.code = match[1];
@@ -650,16 +640,7 @@ RedisClient.prototype.return_reply = function (reply) {
650640
if (command_obj && !command_obj.sub_command) {
651641
if (typeof command_obj.callback === 'function') {
652642
if ('exec' !== command_obj.command) {
653-
if (command_obj.buffer_args === false) {
654-
// If detect_buffers option was specified, then the reply from the parser will be Buffers.
655-
// If this command did not use Buffer arguments, then convert the reply to Strings here.
656-
reply = utils.reply_to_strings(reply);
657-
}
658-
659-
// TODO - confusing and error-prone that hgetall is special cased in two places
660-
if ('hgetall' === command_obj.command) {
661-
reply = utils.reply_to_object(reply);
662-
}
643+
reply = this.handle_reply(reply, command_obj.command, command_obj.buffer_args);
663644
}
664645
command_obj.callback(null, reply);
665646
} else {
@@ -722,8 +703,7 @@ RedisClient.prototype.send_command = function (command, args, callback) {
722703
command_str = '',
723704
buffer_args = false,
724705
big_data = false,
725-
prefix_keys,
726-
buffer = this.options.return_buffers;
706+
prefix_keys;
727707

728708
if (args === undefined) {
729709
args = [];
@@ -770,11 +750,8 @@ RedisClient.prototype.send_command = function (command, args, callback) {
770750
}
771751
}
772752
}
773-
if (this.options.detect_buffers) {
774-
buffer = buffer_args;
775-
}
776753

777-
command_obj = new Command(command, args, false, buffer, callback);
754+
command_obj = new Command(command, args, false, buffer_args, callback);
778755

779756
if (!this.ready && !this.send_anyway || !stream.writable) {
780757
if (this.closing || !this.enable_offline_queue) {
@@ -1149,11 +1126,7 @@ Multi.prototype.exec_transaction = function (callback) {
11491126
cb = undefined;
11501127
}
11511128
// Keep track of who wants buffer responses:
1152-
if (this._client.options.return_buffers) {
1153-
this.wants_buffers[index] = true;
1154-
} else if (!this._client.options.detect_buffers) {
1155-
this.wants_buffers[index] = false;
1156-
} else {
1129+
if (this._client.options.detect_buffers) {
11571130
this.wants_buffers[index] = false;
11581131
for (var i = 0; i < args.length; i += 1) {
11591132
if (Buffer.isBuffer(args[i])) {
@@ -1193,20 +1166,14 @@ Multi.prototype.execute_callback = function (err, replies) {
11931166
while (args = this.queue.shift()) {
11941167
// If we asked for strings, even in detect_buffers mode, then return strings:
11951168
if (replies[i] instanceof Error) {
1196-
var match = replies[i].message.match(utils.errCode);
1169+
var match = replies[i].message.match(utils.err_code);
11971170
// LUA script could return user errors that don't behave like all other errors!
11981171
if (match) {
11991172
replies[i].code = match[1];
12001173
}
12011174
replies[i].command = args[0].toUpperCase();
12021175
} else if (replies[i]) {
1203-
if (this.wants_buffers[i] === false) {
1204-
replies[i] = utils.reply_to_strings(replies[i]);
1205-
}
1206-
if (args[0] === 'hgetall') {
1207-
// TODO - confusing and error-prone that hgetall is special cased in two places
1208-
replies[i] = utils.reply_to_object(replies[i]);
1209-
}
1176+
replies[i] = this._client.handle_reply(replies[i], args[0], this.wants_buffers[i]);
12101177
}
12111178

12121179
if (typeof args[args.length - 1] === 'function') {

lib/parsers/hiredis.js

Lines changed: 0 additions & 37 deletions
This file was deleted.

0 commit comments

Comments
 (0)