Skip to content

Commit 3142af5

Browse files
author
Ruben Bridgewater
committed
Update benchmark and diff output
1 parent d5628f4 commit 3142af5

File tree

2 files changed

+111
-89
lines changed

2 files changed

+111
-89
lines changed

‎benchmarks/diff_multi_bench_output.js

Lines changed: 52 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,50 +1,55 @@
1-
#!/usr/bin/env node
2-
31
'use strict';
42

5-
/* jshint -W079: Ignore redefinitions (before & after) */
6-
7-
var fs = require('fs'),
8-
metrics = require('metrics'),
3+
var fs = require('fs');
4+
var metrics = require('metrics');
5+
// `node diff_multi_bench_output.js beforeBench.txt afterBench.txt`
6+
var file1 = process.argv[2];
7+
var file2 = process.argv[3];
98

10-
// `node diff_multi_bench_output.js before.txt after.txt`
11-
before = process.argv[2],
12-
after = process.argv[3];
13-
14-
if (!before || !after) {
9+
if (!file1 || !file2) {
1510
console.log('Please supply two file arguments:');
1611
var n = __filename;
1712
n = n.substring(n.lastIndexOf('/', n.length));
18-
console.log(' ./' + n + ' multiBenchBefore.txt multiBenchAfter.txt');
19-
console.log('To generate multiBenchBefore.txt, run');
20-
console.log(' node multi_bench.js > multiBenchBefore.txt');
13+
console.log(' node .' + n + ' benchBefore.txt benchAfter.txt\n');
14+
console.log('To generate the benchmark files, run');
15+
console.log(' npm run benchmark > benchBefore.txt\n');
2116
console.log('Thank you for benchmarking responsibly.');
2217
return;
2318
}
2419

25-
var before_lines = fs.readFileSync(before, 'utf8').split('\n'),
26-
after_lines = fs.readFileSync(after, 'utf8').split('\n');
27-
28-
console.log('Comparing before,', before.green, '(', before_lines.length,
29-
'lines)', 'to after,', after.green, '(', after_lines.length, 'lines)');
30-
20+
var before_lines = fs.readFileSync(file1, 'utf8').split('\n');
21+
var after_lines = fs.readFileSync(file2, 'utf8').split('\n');
3122
var total_ops = new metrics.Histogram.createUniformHistogram();
3223

24+
console.log('Comparing before,', file1, '(', before_lines.length, 'lines)', 'to after,', file2, '(', after_lines.length, 'lines)');
25+
3326
function is_whitespace(s) {
3427
return !!s.trim();
3528
}
3629

37-
function parseInt10(s) {
38-
return parseInt(s, 10);
30+
function pad(input, len, chr, right) {
31+
var str = input.toString();
32+
chr = chr || ' ';
33+
34+
if (right) {
35+
while (str.length < len) {
36+
str += chr;
37+
}
38+
} else {
39+
while (str.length < len) {
40+
str = chr + str;
41+
}
42+
}
43+
return str;
3944
}
4045

4146
// green if greater than 0, red otherwise
42-
function humanize_diff(num, unit) {
47+
function humanize_diff(num, unit, toFixed) {
4348
unit = unit || '';
4449
if (num > 0) {
45-
return ('+' + num + unit).green;
50+
return ' +' + pad(num.toFixed(toFixed || 0) + unit, 7);
4651
}
47-
return ('' + num + unit).red;
52+
return ' -' + pad(Math.abs(num).toFixed(toFixed || 0) + unit, 7);
4853
}
4954

5055
function command_name(words) {
@@ -58,35 +63,33 @@ before_lines.forEach(function(b, i) {
5863
// console.log('#ignored#', '>'+a+'<', '>'+b+'<');
5964
return;
6065
}
61-
6266
var b_words = b.split(' ').filter(is_whitespace);
6367
var a_words = a.split(' ').filter(is_whitespace);
6468

65-
var ops =
66-
[b_words, a_words]
67-
.map(function(words) {
68-
// console.log(words);
69-
return parseInt10(words.slice(-2, -1));
70-
}).filter(function(num) {
71-
var isNaN = !num && num !== 0;
72-
return !isNaN;
73-
});
74-
if (ops.length !== 2) return;
75-
69+
var ops = [b_words, a_words].map(function(words) {
70+
// console.log(words);
71+
return words.slice(-2, -1) | 0;
72+
}).filter(function(num) {
73+
var isNaN = !num && num !== 0;
74+
return !isNaN;
75+
});
76+
if (ops.length !== 2) {
77+
return;
78+
}
7679
var delta = ops[1] - ops[0];
77-
var pct = ((delta / ops[0]) * 100).toPrecision(3);
78-
80+
var pct = +((delta / ops[0]) * 100);
81+
ops[0] = pad(ops[0], 6);
82+
ops[1] = pad(ops[1], 6);
7983
total_ops.update(delta);
80-
8184
delta = humanize_diff(delta);
82-
pct = humanize_diff(pct, '%');
83-
console.log(
84-
// name of test
85-
command_name(a_words) === command_name(b_words) ?
86-
command_name(a_words) + ':' :
87-
'404:',
88-
// results of test
89-
ops.join(' -> '), 'ops/sec (∆', delta, pct, ')');
85+
var small_delta = pct < 3 && pct > -3;
86+
// Let's mark differences above 20% bold
87+
var big_delta = pct > 20 || pct < -20 ? ';1' : '';
88+
pct = humanize_diff(pct, '', 2) + '%';
89+
var str = pad((command_name(a_words) === command_name(b_words) ? command_name(a_words) + ':' : '404:'), 14, false, true) +
90+
(pad(ops.join(' -> '), 15) + ' ops/sec (∆' + delta + pct + ')');
91+
str = (small_delta ? '' : (/-[^>]/.test(str) ? '\x1b[31' : '\x1b[32') + big_delta + 'm') + str + '\x1b[0m';
92+
console.log(str);
9093
});
9194

92-
console.log('Mean difference in ops/sec:', humanize_diff(total_ops.mean().toPrecision(6)));
95+
console.log('Mean difference in ops/sec:', humanize_diff(total_ops.mean(), '', 1));

‎benchmarks/multi_bench.js

Lines changed: 59 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -7,24 +7,33 @@ var client_nr = 0;
77
var redis = require('../index');
88
var totalTime = 0;
99
var metrics = require('metrics');
10-
var num_clients = parseInt(process.argv[2], 10) || 5;
11-
var num_requests = 50000;
1210
var tests = [];
1311
// var bluebird = require('bluebird');
1412
// bluebird.promisifyAll(redis.RedisClient.prototype);
1513
// bluebird.promisifyAll(redis.Multi.prototype);
14+
15+
function returnArg (name, def) {
16+
var matches = process.argv.filter(function(entry) {
17+
return entry.indexOf(name + '=') === 0;
18+
});
19+
if (matches.length) {
20+
return matches[0].substr(name.length + 1);
21+
}
22+
return def;
23+
}
24+
var num_clients = returnArg('clients', 1);
25+
var run_time = returnArg('time', 2500); // ms
1626
var versions_logged = false;
1727
var client_options = {
1828
return_buffers: false,
1929
max_attempts: 4,
20-
parser: process.argv.indexOf('parser=javascript') === -1 ? 'hiredis' : 'javascript'
30+
parser: returnArg('parser', 'hiredis')
2131
};
2232
var small_str, large_str, small_buf, large_buf, very_large_str, very_large_buf;
2333

2434
function lpad(input, len, chr) {
2535
var str = input.toString();
2636
chr = chr || ' ';
27-
2837
while (str.length < len) {
2938
str = chr + str;
3039
}
@@ -33,33 +42,27 @@ function lpad(input, len, chr) {
3342

3443
metrics.Histogram.prototype.print_line = function () {
3544
var obj = this.printObj();
36-
3745
return lpad(obj.min, 4) + '/' + lpad(obj.max, 4) + '/' + lpad(obj.mean.toFixed(2), 7) + '/' + lpad(obj.p95.toFixed(2), 7);
3846
};
3947

4048
function Test(args) {
4149
this.args = args;
42-
4350
this.callback = null;
4451
this.clients = [];
4552
this.clients_ready = 0;
4653
this.commands_sent = 0;
4754
this.commands_completed = 0;
48-
this.max_pipeline = this.args.pipeline || num_requests;
55+
this.max_pipeline = this.args.pipeline || 50;
4956
this.batch_pipeline = this.args.batch || 0;
5057
this.client_options = args.client_options || client_options;
51-
this.num_requests = args.reqs || num_requests;
52-
5358
this.connect_latency = new metrics.Histogram();
5459
this.ready_latency = new metrics.Histogram();
5560
this.command_latency = new metrics.Histogram();
5661
}
5762

5863
Test.prototype.run = function (callback) {
5964
var i;
60-
6165
this.callback = callback;
62-
6366
for (i = 0; i < num_clients ; i++) {
6467
this.new_client(i);
6568
}
@@ -112,27 +115,39 @@ Test.prototype.new_client = function (id) {
112115
Test.prototype.on_clients_ready = function () {
113116
process.stdout.write(lpad(this.args.descr, 13) + ', ' + (this.args.batch ? lpad('batch ' + this.args.batch, 9) : lpad(this.args.pipeline, 9)) + '/' + this.clients_ready + ' ');
114117
this.test_start = Date.now();
115-
116118
this.fill_pipeline();
117119
};
118120

119121
Test.prototype.fill_pipeline = function () {
120122
var pipeline = this.commands_sent - this.commands_completed;
121123

122-
if (this.batch_pipeline && this.commands_sent < this.num_requests) {
124+
if (this.test_start < Date.now() - run_time) {
125+
if (this.ended) {
126+
return;
127+
}
128+
this.ended = true;
129+
this.print_stats();
130+
this.stop_clients();
131+
return;
132+
}
133+
134+
if (this.clients[0].should_buffer) {
135+
var self = this;
136+
setTimeout(function() {
137+
self.fill_pipeline();
138+
}, 1);
139+
return;
140+
}
141+
142+
if (this.batch_pipeline) {
123143
this.batch();
124144
} else {
125-
while (this.commands_sent < this.num_requests && pipeline < this.max_pipeline) {
145+
while (pipeline < this.max_pipeline) {
126146
this.commands_sent++;
127147
pipeline++;
128148
this.send_next();
129149
}
130150
}
131-
132-
if (this.commands_completed === this.num_requests) {
133-
this.print_stats();
134-
this.stop_clients();
135-
}
136151
};
137152

138153
Test.prototype.batch = function () {
@@ -191,7 +206,7 @@ Test.prototype.print_stats = function () {
191206
totalTime += duration;
192207

193208
console.log('min/max/avg/p95: ' + this.command_latency.print_line() + ' ' + lpad(duration, 6) + 'ms total, ' +
194-
lpad((this.num_requests / (duration / 1000)).toFixed(2), 9) + ' ops/sec');
209+
lpad((this.commands_completed / (duration / 1000)).toFixed(2), 9) + ' ops/sec');
195210
};
196211

197212
small_str = '1234';
@@ -203,67 +218,71 @@ very_large_buf = new Buffer(very_large_str);
203218

204219
tests.push(new Test({descr: 'PING', command: 'ping', args: [], pipeline: 1}));
205220
tests.push(new Test({descr: 'PING', command: 'ping', args: [], pipeline: 50}));
206-
tests.push(new Test({descr: 'PING', command: 'ping', args: [], batch: 50, reqs: num_requests * 2}));
221+
tests.push(new Test({descr: 'PING', command: 'ping', args: [], batch: 50}));
207222

208223
tests.push(new Test({descr: 'SET 4B str', command: 'set', args: ['foo_rand000000000000', small_str], pipeline: 1}));
209224
tests.push(new Test({descr: 'SET 4B str', command: 'set', args: ['foo_rand000000000000', small_str], pipeline: 50}));
210-
tests.push(new Test({descr: 'SET 4B str', command: 'set', args: ['foo_rand000000000000', small_str], batch: 50, reqs: num_requests * 2}));
225+
tests.push(new Test({descr: 'SET 4B str', command: 'set', args: ['foo_rand000000000000', small_str], batch: 50}));
211226

212227
tests.push(new Test({descr: 'SET 4B buf', command: 'set', args: ['foo_rand000000000000', small_buf], pipeline: 1}));
213228
tests.push(new Test({descr: 'SET 4B buf', command: 'set', args: ['foo_rand000000000000', small_buf], pipeline: 50}));
214-
tests.push(new Test({descr: 'SET 4B buf', command: 'set', args: ['foo_rand000000000000', small_buf], batch: 50, reqs: num_requests * 2}));
229+
tests.push(new Test({descr: 'SET 4B buf', command: 'set', args: ['foo_rand000000000000', small_buf], batch: 50}));
215230

216231
tests.push(new Test({descr: 'GET 4B str', command: 'get', args: ['foo_rand000000000000'], pipeline: 1}));
217232
tests.push(new Test({descr: 'GET 4B str', command: 'get', args: ['foo_rand000000000000'], pipeline: 50}));
218-
tests.push(new Test({descr: 'GET 4B str', command: 'get', args: ['foo_rand000000000000'], batch: 50, reqs: num_requests * 2}));
233+
tests.push(new Test({descr: 'GET 4B str', command: 'get', args: ['foo_rand000000000000'], batch: 50}));
219234

220235
tests.push(new Test({descr: 'GET 4B buf', command: 'get', args: ['foo_rand000000000000'], pipeline: 1, client_opts: { return_buffers: true} }));
221236
tests.push(new Test({descr: 'GET 4B buf', command: 'get', args: ['foo_rand000000000000'], pipeline: 50, client_opts: { return_buffers: true} }));
222237
tests.push(new Test({descr: 'GET 4B buf', command: 'get', args: ['foo_rand000000000000'], batch: 50, client_opts: { return_buffers: true} }));
223238

224239
tests.push(new Test({descr: 'SET 4KiB str', command: 'set', args: ['foo_rand000000000001', large_str], pipeline: 1}));
225240
tests.push(new Test({descr: 'SET 4KiB str', command: 'set', args: ['foo_rand000000000001', large_str], pipeline: 50}));
226-
tests.push(new Test({descr: 'SET 4KiB str', command: 'set', args: ['foo_rand000000000001', large_str], batch: 50, reqs: num_requests * 2}));
241+
tests.push(new Test({descr: 'SET 4KiB str', command: 'set', args: ['foo_rand000000000001', large_str], batch: 50}));
227242

228243
tests.push(new Test({descr: 'SET 4KiB buf', command: 'set', args: ['foo_rand000000000001', large_buf], pipeline: 1}));
229244
tests.push(new Test({descr: 'SET 4KiB buf', command: 'set', args: ['foo_rand000000000001', large_buf], pipeline: 50}));
230-
tests.push(new Test({descr: 'SET 4KiB buf', command: 'set', args: ['foo_rand000000000001', large_buf], batch: 50, reqs: num_requests * 2}));
245+
tests.push(new Test({descr: 'SET 4KiB buf', command: 'set', args: ['foo_rand000000000001', large_buf], batch: 50}));
231246

232247
tests.push(new Test({descr: 'GET 4KiB str', command: 'get', args: ['foo_rand000000000001'], pipeline: 1}));
233248
tests.push(new Test({descr: 'GET 4KiB str', command: 'get', args: ['foo_rand000000000001'], pipeline: 50}));
234-
tests.push(new Test({descr: 'GET 4KiB str', command: 'get', args: ['foo_rand000000000001'], batch: 50, reqs: num_requests * 2}));
249+
tests.push(new Test({descr: 'GET 4KiB str', command: 'get', args: ['foo_rand000000000001'], batch: 50}));
235250

236251
tests.push(new Test({descr: 'GET 4KiB buf', command: 'get', args: ['foo_rand000000000001'], pipeline: 1, client_opts: { return_buffers: true} }));
237252
tests.push(new Test({descr: 'GET 4KiB buf', command: 'get', args: ['foo_rand000000000001'], pipeline: 50, client_opts: { return_buffers: true} }));
238253
tests.push(new Test({descr: 'GET 4KiB buf', command: 'get', args: ['foo_rand000000000001'], batch: 50, client_opts: { return_buffers: true} }));
239254

240255
tests.push(new Test({descr: 'INCR', command: 'incr', args: ['counter_rand000000000000'], pipeline: 1}));
241256
tests.push(new Test({descr: 'INCR', command: 'incr', args: ['counter_rand000000000000'], pipeline: 50}));
242-
tests.push(new Test({descr: 'INCR', command: 'incr', args: ['counter_rand000000000000'], batch: 50, reqs: num_requests * 2}));
257+
tests.push(new Test({descr: 'INCR', command: 'incr', args: ['counter_rand000000000000'], batch: 50}));
243258

244259
tests.push(new Test({descr: 'LPUSH', command: 'lpush', args: ['mylist', small_str], pipeline: 1}));
245260
tests.push(new Test({descr: 'LPUSH', command: 'lpush', args: ['mylist', small_str], pipeline: 50}));
246-
tests.push(new Test({descr: 'LPUSH', command: 'lpush', args: ['mylist', small_str], batch: 50, reqs: num_requests * 2}));
261+
tests.push(new Test({descr: 'LPUSH', command: 'lpush', args: ['mylist', small_str], batch: 50}));
247262

248263
tests.push(new Test({descr: 'LRANGE 10', command: 'lrange', args: ['mylist', '0', '9'], pipeline: 1}));
249264
tests.push(new Test({descr: 'LRANGE 10', command: 'lrange', args: ['mylist', '0', '9'], pipeline: 50}));
250-
tests.push(new Test({descr: 'LRANGE 10', command: 'lrange', args: ['mylist', '0', '9'], batch: 50, reqs: num_requests * 2}));
265+
tests.push(new Test({descr: 'LRANGE 10', command: 'lrange', args: ['mylist', '0', '9'], batch: 50}));
251266

252267
tests.push(new Test({descr: 'LRANGE 100', command: 'lrange', args: ['mylist', '0', '99'], pipeline: 1}));
253268
tests.push(new Test({descr: 'LRANGE 100', command: 'lrange', args: ['mylist', '0', '99'], pipeline: 50}));
254-
tests.push(new Test({descr: 'LRANGE 100', command: 'lrange', args: ['mylist', '0', '99'], batch: 50, reqs: num_requests * 2}));
269+
tests.push(new Test({descr: 'LRANGE 100', command: 'lrange', args: ['mylist', '0', '99'], batch: 50}));
270+
271+
tests.push(new Test({descr: 'SET 4MiB buf', command: 'set', args: ['foo_rand000000000002', very_large_buf], pipeline: 1}));
272+
tests.push(new Test({descr: 'SET 4MiB buf', command: 'set', args: ['foo_rand000000000002', very_large_buf], pipeline: 20}));
273+
tests.push(new Test({descr: 'SET 4MiB buf', command: 'set', args: ['foo_rand000000000002', very_large_buf], batch: 20}));
255274

256-
tests.push(new Test({descr: 'SET 4MiB buf', command: 'set', args: ['foo_rand000000000002', very_large_buf], pipeline: 1, reqs: 500}));
257-
tests.push(new Test({descr: 'SET 4MiB buf', command: 'set', args: ['foo_rand000000000002', very_large_buf], pipeline: 20, reqs: 500}));
258-
tests.push(new Test({descr: 'SET 4MiB buf', command: 'set', args: ['foo_rand000000000002', very_large_buf], batch: 20, reqs: 500}));
275+
tests.push(new Test({descr: 'SET 4MiB str', command: 'set', args: ['foo_rand000000000002', very_large_str], pipeline: 1}));
276+
tests.push(new Test({descr: 'SET 4MiB str', command: 'set', args: ['foo_rand000000000002', very_large_str], pipeline: 20}));
277+
tests.push(new Test({descr: 'SET 4MiB str', command: 'set', args: ['foo_rand000000000002', very_large_str], batch: 20}));
259278

260-
tests.push(new Test({descr: 'GET 4MiB str', command: 'get', args: ['foo_rand000000000002'], pipeline: 1, reqs: 100}));
261-
tests.push(new Test({descr: 'GET 4MiB str', command: 'get', args: ['foo_rand000000000002'], pipeline: 20, reqs: 100}));
262-
tests.push(new Test({descr: 'GET 4MiB str', command: 'get', args: ['foo_rand000000000002'], batch: 20, reqs: 100}));
279+
tests.push(new Test({descr: 'GET 4MiB str', command: 'get', args: ['foo_rand000000000002'], pipeline: 1}));
280+
tests.push(new Test({descr: 'GET 4MiB str', command: 'get', args: ['foo_rand000000000002'], pipeline: 20}));
281+
tests.push(new Test({descr: 'GET 4MiB str', command: 'get', args: ['foo_rand000000000002'], batch: 20}));
263282

264-
tests.push(new Test({descr: 'GET 4MiB buf', command: 'get', args: ['foo_rand000000000002'], pipeline: 1, reqs: 100, client_opts: { return_buffers: true} }));
265-
tests.push(new Test({descr: 'GET 4MiB buf', command: 'get', args: ['foo_rand000000000002'], pipeline: 20, reqs: 100, client_opts: { return_buffers: true} }));
266-
tests.push(new Test({descr: 'GET 4MiB buf', command: 'get', args: ['foo_rand000000000002'], batch: 20, reqs: 100, client_opts: { return_buffers: true} }));
283+
tests.push(new Test({descr: 'GET 4MiB buf', command: 'get', args: ['foo_rand000000000002'], pipeline: 1, client_opts: { return_buffers: true} }));
284+
tests.push(new Test({descr: 'GET 4MiB buf', command: 'get', args: ['foo_rand000000000002'], pipeline: 20, client_opts: { return_buffers: true} }));
285+
tests.push(new Test({descr: 'GET 4MiB buf', command: 'get', args: ['foo_rand000000000002'], batch: 20, client_opts: { return_buffers: true} }));
267286

268287
function next() {
269288
var test = tests.shift();

0 commit comments

Comments
 (0)