Skip to content

Commit ac9ff9a

Browse files
author
Ruben Bridgewater
committed
Refactor js parser
Fix tests to work with Node.js 0.10 Improve average use case speed by up to 20% Fix some small js parser issues
1 parent b6a81a4 commit ac9ff9a

File tree

4 files changed

+172
-130
lines changed

4 files changed

+172
-130
lines changed

‎README.md

Lines changed: 35 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ redis - a node.js redis client
66
[![Windows Tests](https://img.shields.io/appveyor/ci/BridgeAR/node-redis/master.svg?label=Windows%20Tests)](https://ci.appveyor.com/project/BridgeAR/node-redis/branch/master)
77
[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/NodeRedis/node_redis?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
88

9-
This is a complete and feature rich Redis client for node.js. It supports all Redis commands and focuses on performance.
9+
This is a complete and feature rich Redis client for node.js. It supports all Redis commands and focuses on high performance.
1010

1111
Install with:
1212

@@ -646,40 +646,40 @@ hiredis parser (Lenovo T450s i7-5600U):
646646

647647
```
648648
Client count: 1, node version: 4.2.2, server version: 3.0.3, parser: hiredis
649-
PING, 1/1 min/max/avg/p95: 0/ 3/ 0.02/ 0.00 2501ms total, 39862.85 ops/sec
650-
PING, batch 50/1 min/max/avg/p95: 0/ 2/ 0.10/ 1.00 2501ms total, 491223.51 ops/sec
651-
SET 4B str, 1/1 min/max/avg/p95: 0/ 3/ 0.03/ 0.00 2501ms total, 36387.45 ops/sec
652-
SET 4B str, batch 50/1 min/max/avg/p95: 0/ 3/ 0.14/ 1.00 2501ms total, 346381.45 ops/sec
653-
SET 4B buf, 1/1 min/max/avg/p95: 0/ 2/ 0.04/ 0.00 2501ms total, 24395.84 ops/sec
654-
SET 4B buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 156457.42 ops/sec
655-
GET 4B str, 1/1 min/max/avg/p95: 0/ 3/ 0.03/ 0.00 2501ms total, 36906.44 ops/sec
656-
GET 4B str, batch 50/1 min/max/avg/p95: 0/ 3/ 0.12/ 1.00 2501ms total, 425729.71 ops/sec
657-
GET 4B buf, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 36221.91 ops/sec
658-
GET 4B buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.11/ 1.00 2501ms total, 430407.84 ops/sec
659-
SET 4KiB str, 1/1 min/max/avg/p95: 0/ 3/ 0.03/ 0.00 2501ms total, 30951.22 ops/sec
660-
SET 4KiB str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.33/ 1.00 2501ms total, 150299.88 ops/sec
661-
SET 4KiB buf, 1/1 min/max/avg/p95: 0/ 2/ 0.04/ 1.00 2501ms total, 23919.63 ops/sec
662-
SET 4KiB buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.36/ 1.00 2501ms total, 139204.32 ops/sec
663-
GET 4KiB str, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 32739.30 ops/sec
664-
GET 4KiB str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 154158.34 ops/sec
665-
GET 4KiB buf, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 34654.94 ops/sec
666-
GET 4KiB buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 153758.50 ops/sec
667-
INCR, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 37530.19 ops/sec
668-
INCR, batch 50/1 min/max/avg/p95: 0/ 3/ 0.12/ 1.00 2501ms total, 415993.60 ops/sec
669-
LPUSH, 1/1 min/max/avg/p95: 0/ 1/ 0.03/ 0.00 2501ms total, 37409.04 ops/sec
670-
LPUSH, batch 50/1 min/max/avg/p95: 0/ 2/ 0.14/ 1.00 2501ms total, 354778.09 ops/sec
671-
LRANGE 10, 1/1 min/max/avg/p95: 0/ 3/ 0.03/ 0.00 2501ms total, 31768.49 ops/sec
672-
LRANGE 10, batch 50/1 min/max/avg/p95: 0/ 3/ 0.33/ 1.00 2501ms total, 151379.45 ops/sec
673-
LRANGE 100, 1/1 min/max/avg/p95: 0/ 2/ 0.06/ 1.00 2501ms total, 16801.68 ops/sec
674-
LRANGE 100, batch 50/1 min/max/avg/p95: 2/ 4/ 2.07/ 3.00 2501ms total, 24150.34 ops/sec
675-
SET 4MiB str, 1/1 min/max/avg/p95: 1/ 5/ 1.96/ 2.00 2501ms total, 510.20 ops/sec
676-
SET 4MiB str, batch 20/1 min/max/avg/p95: 83/ 108/ 94.44/ 106.40 2550ms total, 211.76 ops/sec
677-
SET 4MiB buf, 1/1 min/max/avg/p95: 1/ 7/ 2.06/ 3.00 2501ms total, 484.21 ops/sec
678-
SET 4MiB buf, batch 20/1 min/max/avg/p95: 38/ 48/ 40.90/ 46.00 2536ms total, 488.96 ops/sec
679-
GET 4MiB str, 1/1 min/max/avg/p95: 3/ 13/ 5.20/ 9.00 2503ms total, 192.17 ops/sec
680-
GET 4MiB str, batch 20/1 min/max/avg/p95: 74/ 105/ 87.24/ 104.00 2530ms total, 229.25 ops/sec
681-
GET 4MiB buf, 1/1 min/max/avg/p95: 3/ 11/ 5.01/ 9.00 2501ms total, 199.12 ops/sec
682-
GET 4MiB buf, batch 20/1 min/max/avg/p95: 78/ 93/ 84.23/ 91.90 2528ms total, 237.34 ops/sec
649+
PING, 1/1 min/max/avg/p95: 0/ 2/ 0.02/ 0.00 2501ms total, 47503.80 ops/sec
650+
PING, batch 50/1 min/max/avg/p95: 0/ 2/ 0.09/ 1.00 2501ms total, 529668.13 ops/sec
651+
SET 4B str, 1/1 min/max/avg/p95: 0/ 2/ 0.02/ 0.00 2501ms total, 41900.04 ops/sec
652+
SET 4B str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.14/ 1.00 2501ms total, 354658.14 ops/sec
653+
SET 4B buf, 1/1 min/max/avg/p95: 0/ 4/ 0.04/ 0.00 2501ms total, 23499.00 ops/sec
654+
SET 4B buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.31/ 1.00 2501ms total, 159836.07 ops/sec
655+
GET 4B str, 1/1 min/max/avg/p95: 0/ 4/ 0.02/ 0.00 2501ms total, 43489.80 ops/sec
656+
GET 4B str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.11/ 1.00 2501ms total, 444202.32 ops/sec
657+
GET 4B buf, 1/1 min/max/avg/p95: 0/ 3/ 0.02/ 0.00 2501ms total, 38561.38 ops/sec
658+
GET 4B buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.11/ 1.00 2501ms total, 452139.14 ops/sec
659+
SET 4KiB str, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 32990.80 ops/sec
660+
SET 4KiB str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.34/ 1.00 2501ms total, 146161.54 ops/sec
661+
SET 4KiB buf, 1/1 min/max/avg/p95: 0/ 1/ 0.04/ 0.00 2501ms total, 23294.28 ops/sec
662+
SET 4KiB buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.36/ 1.00 2501ms total, 137584.97 ops/sec
663+
GET 4KiB str, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 36350.66 ops/sec
664+
GET 4KiB str, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 155157.94 ops/sec
665+
GET 4KiB buf, 1/1 min/max/avg/p95: 0/ 4/ 0.02/ 0.00 2501ms total, 39776.49 ops/sec
666+
GET 4KiB buf, batch 50/1 min/max/avg/p95: 0/ 2/ 0.32/ 1.00 2501ms total, 155457.82 ops/sec
667+
INCR, 1/1 min/max/avg/p95: 0/ 3/ 0.02/ 0.00 2501ms total, 43972.41 ops/sec
668+
INCR, batch 50/1 min/max/avg/p95: 0/ 1/ 0.12/ 1.00 2501ms total, 425809.68 ops/sec
669+
LPUSH, 1/1 min/max/avg/p95: 0/ 2/ 0.02/ 0.00 2501ms total, 38998.40 ops/sec
670+
LPUSH, batch 50/1 min/max/avg/p95: 0/ 4/ 0.14/ 1.00 2501ms total, 365013.99 ops/sec
671+
LRANGE 10, 1/1 min/max/avg/p95: 0/ 2/ 0.03/ 0.00 2501ms total, 31879.25 ops/sec
672+
LRANGE 10, batch 50/1 min/max/avg/p95: 0/ 1/ 0.32/ 1.00 2501ms total, 153698.52 ops/sec
673+
LRANGE 100, 1/1 min/max/avg/p95: 0/ 4/ 0.06/ 0.00 2501ms total, 16676.13 ops/sec
674+
LRANGE 100, batch 50/1 min/max/avg/p95: 1/ 6/ 2.03/ 2.00 2502ms total, 24520.38 ops/sec
675+
SET 4MiB str, 1/1 min/max/avg/p95: 1/ 6/ 2.11/ 3.00 2502ms total, 472.82 ops/sec
676+
SET 4MiB str, batch 20/1 min/max/avg/p95: 85/ 112/ 94.93/ 109.60 2563ms total, 210.69 ops/sec
677+
SET 4MiB buf, 1/1 min/max/avg/p95: 1/ 8/ 2.02/ 3.00 2502ms total, 490.01 ops/sec
678+
SET 4MiB buf, batch 20/1 min/max/avg/p95: 37/ 52/ 39.48/ 46.75 2528ms total, 506.33 ops/sec
679+
GET 4MiB str, 1/1 min/max/avg/p95: 3/ 13/ 5.26/ 9.00 2504ms total, 190.10 ops/sec
680+
GET 4MiB str, batch 20/1 min/max/avg/p95: 70/ 106/ 89.36/ 103.75 2503ms total, 223.73 ops/sec
681+
GET 4MiB buf, 1/1 min/max/avg/p95: 3/ 11/ 5.04/ 8.15 2502ms total, 198.24 ops/sec
682+
GET 4MiB buf, batch 20/1 min/max/avg/p95: 70/ 105/ 88.07/ 103.00 2554ms total, 227.09 ops/sec
683683
```
684684

685685
The hiredis and js parser should most of the time be on the same level. But if you use Redis for big SUNION/SINTER/LRANGE/ZRANGE hiredis is significantly faster.

‎index.js

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -294,14 +294,10 @@ RedisClient.prototype.init_parser = function () {
294294
// Important: Only send results / errors async.
295295
// That way the result / error won't stay in a try catch block and catch user things
296296
this.reply_parser.send_error = function (data) {
297-
process.nextTick(function() {
298-
self.return_error(data);
299-
});
297+
self.return_error(data);
300298
};
301299
this.reply_parser.send_reply = function (data) {
302-
process.nextTick(function() {
303-
self.return_reply(data);
304-
});
300+
self.return_reply(data);
305301
};
306302
};
307303

‎lib/parsers/javascript.js

Lines changed: 50 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ function JavascriptReplyParser() {
99
this._big_offset = 0;
1010
this._chunks_size = 0;
1111
this._buffers = [];
12+
this._type = 0;
13+
this._protocol_error = false;
1214
}
1315

1416
function IncompleteReadBuffer(message) {
@@ -21,87 +23,73 @@ JavascriptReplyParser.prototype._parseResult = function (type) {
2123
var start = 0,
2224
end = 0,
2325
offset = 0,
24-
packetHeader = 0;
26+
packetHeader = 0,
27+
res,
28+
reply;
2529

2630
if (type === 43 || type === 58 || type === 45) { // + or : or -
27-
// up to the delimiter
31+
// Up to the delimiter
2832
end = this._packetEndOffset();
2933
start = this._offset;
30-
31-
// include the delimiter
34+
// Include the delimiter
3235
this._offset = end + 2;
3336

3437
if (type === 43) {
3538
return this._buffer.slice(start, end);
3639
} else if (type === 58) {
37-
// return the coerced numeric value
40+
// Return the coerced numeric value
3841
return +this._buffer.toString('ascii', start, end);
3942
}
4043
return new Error(this._buffer.toString('utf-8', start, end));
4144
} else if (type === 36) { // $
4245
packetHeader = this.parseHeader();
4346

44-
// packets with a size of -1 are considered null
47+
// Packets with a size of -1 are considered null
4548
if (packetHeader === -1) {
4649
return null;
4750
}
48-
4951
end = this._offset + packetHeader;
5052
start = this._offset;
5153

52-
if (end > this._buffer.length) {
54+
if (end + 2 > this._buffer.length) {
5355
this._chunks_size = this._buffer.length - this._offset - 2;
5456
this._big_offset = packetHeader;
5557
throw new IncompleteReadBuffer('Wait for more data.');
5658
}
57-
58-
// set the offset to after the delimiter
59+
// Set the offset to after the delimiter
5960
this._offset = end + 2;
6061

6162
return this._buffer.slice(start, end);
6263
} else if (type === 42) { // *
63-
// set a rewind point, as the packet is larger than the buffer in memory
64+
// Set a rewind point, as the packet is larger than the buffer in memory
6465
offset = this._offset;
6566
packetHeader = this.parseHeader();
6667

6768
if (packetHeader === -1) {
6869
return null;
6970
}
70-
71-
if (packetHeader > this._buffer.length - this._offset) {
72-
this._offset = offset - 1;
73-
throw new IncompleteReadBuffer('Wait for more data.');
74-
}
75-
76-
var reply = [];
77-
var ntype, i, res;
78-
71+
reply = [];
7972
offset = this._offset - 1;
8073

81-
for (i = 0; i < packetHeader; i++) {
82-
ntype = this._buffer[this._offset++];
83-
84-
if (this._offset > this._buffer.length) {
74+
for (var i = 0; i < packetHeader; i++) {
75+
if (this._offset >= this._buffer.length) {
8576
throw new IncompleteReadBuffer('Wait for more data.');
8677
}
87-
res = this._parseResult(ntype);
78+
res = this._parseResult(this._buffer[this._offset++]);
8879
reply.push(res);
8980
}
90-
9181
return reply;
9282
} else {
93-
return null;
83+
return void 0;
9484
}
9585
};
9686

9787
JavascriptReplyParser.prototype.execute = function (buffer) {
98-
9988
if (this._chunks_size !== 0 && this._big_offset > this._chunks_size + buffer.length) {
10089
this._buffers.push(buffer);
10190
this._chunks_size += buffer.length;
10291
return;
10392
}
104-
10593
if (this._buffers.length !== 0) {
10694
this._buffers.unshift(this._offset === 0 ? this._buffer : this._buffer.slice(this._offset));
10795
this._buffers.push(buffer);
@@ -115,44 +103,41 @@ JavascriptReplyParser.prototype.execute = function (buffer) {
115103
this._buffer = Buffer.concat([this._buffer.slice(this._offset), buffer]);
116104
}
117105
this._offset = 0;
106+
this._protocol_error = true;
118107
this.run();
119108
};
120109

121-
JavascriptReplyParser.prototype.run = function (buffer) {
122-
var type, offset = this._offset;
123-
124-
while (true) {
125-
offset = this._offset;
126-
// at least 4 bytes: :1\r\n
127-
if (this._buffer.length - this._offset < 4) {
128-
break;
129-
}
110+
JavascriptReplyParser.prototype.try_parsing = function () {
111+
// Set a rewind point. If a failure occurs, wait for the next execute()/append() and try again
112+
var offset = this._offset - 1;
113+
try {
114+
return this._parseResult(this._type);
115+
} catch (err) {
116+
// Catch the error (not enough data), rewind if it's an array,
117+
// and wait for the next packet to appear
118+
this._offset = offset;
119+
this._protocol_error = false;
120+
return void 0;
121+
}
122+
};
130123

131-
try {
132-
type = this._buffer[this._offset++];
133-
134-
if (type === 43 || type === 58 || type === 36) { // Strings + // Integers : // Bulk strings $
135-
this.send_reply(this._parseResult(type));
136-
} else if (type === 45) { // Errors -
137-
this.send_error(this._parseResult(type));
138-
} else if (type === 42) { // Arrays *
139-
// set a rewind point. if a failure occurs,
140-
// wait for the next execute()/append() and try again
141-
offset = this._offset - 1;
142-
143-
this.send_reply(this._parseResult(type));
144-
} else if (type !== 10 && type !== 13) {
145-
// Reset the buffer so the parser can handle following commands properly
146-
this._buffer = new Buffer(0);
147-
var err = new Error('Protocol error, got "' + String.fromCharCode(type) + '" as reply type byte');
148-
this.send_error(err);
149-
}
150-
} catch (err) {
151-
// catch the error (not enough data), rewind, and wait
152-
// for the next packet to appear
153-
this._offset = offset;
154-
break;
124+
JavascriptReplyParser.prototype.run = function (buffer) {
125+
this._type = this._buffer[this._offset++];
126+
var reply = this.try_parsing();
127+
128+
while (reply !== undefined) {
129+
if (this._type === 45) { // Errors -
130+
this.send_error(reply);
131+
} else {
132+
this.send_reply(reply); // Strings + // Integers : // Bulk strings $ // Arrays *
155133
}
134+
this._type = this._buffer[this._offset++];
135+
reply = this.try_parsing();
136+
}
137+
if (this._type !== undefined && this._protocol_error === true) {
138+
// Reset the buffer so the parser can handle following commands properly
139+
this._buffer = new Buffer(0);
140+
this.send_error(new Error('Protocol error, got "' + String.fromCharCode(this._type) + '" as reply type byte'));
156141
}
157142
};
158143

@@ -161,21 +146,20 @@ JavascriptReplyParser.prototype.parseHeader = function () {
161146
value = this._buffer.toString('ascii', this._offset, end) | 0;
162147

163148
this._offset = end + 2;
164-
165149
return value;
166150
};
167151

168152
JavascriptReplyParser.prototype._packetEndOffset = function () {
169-
var offset = this._offset;
153+
var offset = this._offset,
154+
len = this._buffer.length - 1;
170155

171156
while (this._buffer[offset] !== 0x0d && this._buffer[offset + 1] !== 0x0a) {
172157
offset++;
173158

174-
if (offset >= this._buffer.length) {
159+
if (offset >= len) {
175160
throw new IncompleteReadBuffer('Did not see LF after NL reading multi bulk count (' + offset + ' => ' + this._buffer.length + ', ' + this._offset + ')');
176161
}
177162
}
178-
179163
return offset;
180164
};
181165

0 commit comments

Comments
 (0)