Skip to content

Commit dab611b

Browse files
sidoraressidorares
committed
fix 64 bit numbers calculation in text and binary protocol
1 parent 1fe5d73 commit dab611b

File tree

5 files changed

+255
-30
lines changed

5 files changed

+255
-30
lines changed

‎lib/compile_binary_parser.js

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ function compile (fields, options, config) {
7272
result.push(' if (nullBitmaskByte' + nullByteIndex + ' & ' + currentFieldNullBit + ')');
7373
result.push(' ' + lvalue + ' = null;');
7474
result.push(' else');
75-
result.push(' ' + lvalue + ' = ' + readCodeFor(fields[i], config));
75+
result.push(' ' + lvalue + ' = ' + readCodeFor(fields[i], config, options));
7676
// }
7777
currentFieldNullBit *= 2;
7878
if (currentFieldNullBit == 0x100) {
@@ -95,7 +95,10 @@ function compile (fields, options, config) {
9595
return vm.runInThisContext(src);
9696
}
9797

98-
function readCodeFor (field, config) {
98+
function readCodeFor (field, config, options) {
99+
var supportBigNumbers = options.supportBigNumbers || config.supportBigNumbers;
100+
var bigNumberStrings = options.bigNumberStrings || config.bigNumberStrings;
101+
99102
var unsigned = field.flags & FieldFlags.UNSIGNED;
100103
switch (field.columnType) {
101104
case Types.TINY:
@@ -133,8 +136,16 @@ function readCodeFor (field, config) {
133136
return 'packet.parseGeometryValue();';
134137
case Types.JSON:
135138
return 'JSON.parse(packet.readLengthCodedString());';
136-
case Types.LONGLONG: // TODO: 8 bytes. Implement as two 4 bytes read for now (it's out of JavaScript int precision!)
137-
return unsigned ? 'packet.readInt64();' : 'packet.readSInt64();';
139+
case Types.LONGLONG:
140+
if (!supportBigNumbers) {
141+
return unsigned ? 'packet.readInt64JSNumber();' : 'packet.readSInt64JSNumber();';
142+
} else {
143+
if (bigNumberStrings) {
144+
return unsigned ? 'packet.readInt64String();' : 'packet.readSInt64String();';
145+
} else {
146+
return unsigned ? 'packet.readInt64();' : 'packet.readSInt64();';
147+
}
148+
}
138149
default:
139150
if (field.characterSet == Charsets.BINARY) {
140151
return 'packet.readLengthCodedBuffer();';

‎lib/compile_text_parser.js

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ function compile (fields, options, config) {
7373
} else {
7474
lvalue = ' this[' + srcEscape(fields[i].name) + ']';
7575
}
76-
var readCode = readCodeFor(fields[i].columnType, fields[i].characterSet, config);
76+
var readCode = readCodeFor(fields[i].columnType, fields[i].characterSet, config, options);
7777
if (typeof options.typeCast === 'function') {
7878
result.push(lvalue + ' = options.typeCast(wrap(fields[' + i + '], ' + srcEscape(typeNames[fields[i].columnType]) + ', packet), function() { return ' + readCode + ';})');
7979
} else if (options.typeCast === false) {
@@ -98,19 +98,22 @@ function compile (fields, options, config) {
9898
return vm.runInThisContext(src);
9999
}
100100

101-
function readCodeFor (type, charset, config) {
101+
function readCodeFor (type, charset, config, options) {
102+
var supportBigNumbers = options.supportBigNumbers || config.supportBigNumbers;
103+
var bigNumberStrings = options.bigNumberStrings || config.bigNumberStrings;
104+
102105
switch (type) {
103106
case Types.TINY:
104107
case Types.SHORT:
105108
case Types.LONG:
106109
case Types.INT24:
107110
case Types.YEAR:
108-
return 'packet.parseLengthCodedInt()';
111+
return 'packet.parseLengthCodedIntNoBigCheck()';
109112
case Types.LONGLONG:
110-
if (config.supportBigNumbers && config.bigNumberStrings) {
113+
if (supportBigNumbers && bigNumberStrings) {
111114
return 'packet.parseLengthCodedIntString()';
112115
}
113-
return 'packet.parseLengthCodedInt()';
116+
return 'packet.parseLengthCodedInt(' + supportBigNumbers + ')';
114117
case Types.FLOAT:
115118
case Types.DOUBLE:
116119
return 'packet.parseLengthCodedFloat()';

‎lib/connection.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -382,7 +382,8 @@ Connection.prototype.resume = function resume () {
382382
};
383383

384384
Connection.prototype.keyFromFields = function keyFromFields (fields, options) {
385-
var res = (typeof options.nestTables) + '/' + options.nestTables + '/' + options.rowsAsArray;
385+
var res = (typeof options.nestTables) + '/' + options.nestTables + '/' + options.rowsAsArray +
386+
+ options.supportBigNumbers + '/' + options.bigNumberStrings;
386387
for (var i = 0; i < fields.length; ++i) {
387388
res += '/' + fields[i].name + ':' + fields[i].columnType + ':' + fields[i].flags;
388389
}

‎lib/packets/packet.js

Lines changed: 123 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -81,17 +81,63 @@ Packet.prototype.readSInt32 = function ()
8181
return this.buffer.readInt32LE(this.offset - 4, true);
8282
};
8383

84-
Packet.prototype.readInt64 = function () {
85-
return this.readInt32() + 0x100000000 * this.readInt32();
84+
Packet.prototype.readInt64JSNumber = function () {
85+
var word0 = this.readInt32();
86+
var word1 = this.readInt32();
87+
var l = new Long(word0, word1, true);
88+
return l.toNumber();
8689
};
8790

88-
Packet.prototype.readSInt64 = function () {
91+
Packet.prototype.readSInt64JSNumber = function () {
8992
var word0 = this.readInt32();
9093
var word1 = this.readInt32();
9194
if (!(word1 & 0x80000000)) {
9295
return word0 + 0x100000000 * word1;
9396
}
94-
return -((((~word1) >>> 0) * 0x100000000) + ((~word0) >>> 0) + 1);
97+
var l = new Long(word0, word1, false);
98+
return l.toNumber();
99+
};
100+
101+
Packet.prototype.readInt64String = function () {
102+
var word0 = this.readInt32();
103+
var word1 = this.readInt32();
104+
var res = new Long(word0, word1, true);
105+
return res.toString();
106+
};
107+
108+
Packet.prototype.readSInt64String = function () {
109+
var word0 = this.readInt32();
110+
var word1 = this.readInt32();
111+
var res = new Long(word0, word1, false);
112+
return res.toString();
113+
};
114+
115+
Packet.prototype.readInt64 = function () {
116+
var word0 = this.readInt32();
117+
var word1 = this.readInt32();
118+
var res = new Long(word0, word1, true);
119+
var resNumber = res.toNumber()
120+
, resString = res.toString();
121+
122+
res = resNumber.toString() === resString
123+
? resNumber
124+
: resString;
125+
126+
return res;
127+
};
128+
129+
Packet.prototype.readSInt64 = function () {
130+
var word0 = this.readInt32();
131+
var word1 = this.readInt32();
132+
var res = new Long(word0, word1, false);
133+
var resNumber = res.toNumber()
134+
, resString = res.toString();
135+
136+
res = resNumber.toString() === resString
137+
? resNumber
138+
: resString;
139+
140+
return res;
95141
};
96142

97143
Packet.prototype.isEOF = function () {
@@ -330,15 +376,29 @@ Packet.prototype.readString = function (len) {
330376

331377
var minus = '-'.charCodeAt(0);
332378
var plus = '+'.charCodeAt(0);
333-
// TODO: faster versions of parseInt for smaller types?
334-
// discard overflow checks if we know from type definition it's safe so
335-
Packet.prototype.parseInt = function (len) {
379+
380+
// The whole reason parse* function below exist
381+
// is because String creation is relatively expensive, and if we have
382+
// a buffer with "12345" content ideally we would like to bypass intermediate
383+
// "12345" string creation and directly build 12345 number out of
384+
// <Buffer 31 32 33 34 35> data.
385+
// In my benchmarks "parse" methods
386+
387+
388+
Packet.prototype.parseInt = function (len, supportBigNumbers) {
336389

337390
if (len === null) {
338391
return null;
339392
}
340393

394+
if (len >= 14 && !supportBigNumbers) {
395+
var s = this.buffer.toString('ascii', this.offset, this.offset + len);
396+
this.offset += len;
397+
return Number(s);
398+
}
399+
341400
var result = 0;
401+
var start = this.offset;
342402
var end = this.offset + len;
343403
var sign = 1;
344404
if (len === 0) {
@@ -351,23 +411,62 @@ Packet.prototype.parseInt = function (len) {
351411
}
352412

353413
// max precise int is 9007199254740992
354-
// note that we are here after handling optional +/-
355-
// treat everything above 9000000000000000 as potential overflow
356414
var str;
357415
var numDigits = end - this.offset;
358-
if (numDigits == 16 && (this.buffer[this.offset] - 48) > 8) {
359-
str = this.readString(end - this.offset);
360-
result = parseInt(str, 10);
361-
if (result.toString() == str) {
362-
return sign * result;
363-
} else {
416+
if (supportBigNumbers) {
417+
if (numDigits >= 15) {
418+
str = this.readString(end - this.offset);
419+
result = parseInt(str, 10);
420+
if (result.toString() == str) {
421+
return sign * result;
422+
} else {
423+
return sign == -1 ? '-' + str : str;
424+
}
425+
} else if (numDigits > 16) {
426+
str = this.readString(end - this.offset);
364427
return sign == -1 ? '-' + str : str;
365428
}
366-
} else if (numDigits > 16) {
367-
str = this.readString(end - this.offset);
368-
return sign == -1 ? '-' + str : str;
369429
}
370430

431+
if (this.buffer[this.offset] == plus) {
432+
this.offset++; // just ignore
433+
}
434+
while (this.offset < end) {
435+
result *= 10;
436+
result += this.buffer[this.offset] - 48;
437+
this.offset++;
438+
}
439+
var num = result * sign;
440+
if (!supportBigNumbers) {
441+
return num;
442+
}
443+
str = this.buffer.toString('ascii', start, end);
444+
if (num.toString() == str) {
445+
return num;
446+
} else {
447+
return str;
448+
}
449+
};
450+
451+
// note that if value of inputNumberAsString is bigger than MAX_SAFE_INTEGER
452+
// ( or smaller than MIN_SAFE_INTEGER ) the parseIntNoBigCheck result might be
453+
// different from what you would get from Number(inputNumberAsString)
454+
// String(parseIntNoBigCheck) <> String(Number(inputNumberAsString)) <> inputNumberAsString
455+
Packet.prototype.parseIntNoBigCheck = function (len) {
456+
if (len === null) {
457+
return null;
458+
}
459+
var result = 0;
460+
var end = this.offset + len;
461+
var sign = 1;
462+
if (len === 0) {
463+
return 0; // TODO: assert? exception?
464+
}
465+
466+
if (this.buffer[this.offset] == minus) {
467+
this.offset++;
468+
sign = -1;
469+
}
371470
if (this.buffer[this.offset] == plus) {
372471
this.offset++; // just ignore
373472
}
@@ -515,8 +614,12 @@ Packet.prototype.parseFloat = function (len) {
515614
return result / factor;
516615
};
517616

518-
Packet.prototype.parseLengthCodedInt = function () {
519-
return this.parseInt(this.readLengthCodedNumber());
617+
Packet.prototype.parseLengthCodedIntNoBigCheck = function () {
618+
return this.parseIntNoBigCheck(this.readLengthCodedNumber());
619+
};
620+
621+
Packet.prototype.parseLengthCodedInt = function (supportBigNumbers) {
622+
return this.parseInt(this.readLengthCodedNumber(), supportBigNumbers);
520623
};
521624

522625
Packet.prototype.parseLengthCodedIntString = function () {

‎test/integration/connection/test-binary-longlong.js

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
var assert = require('assert');
2+
3+
var FieldFlags = require('../../../lib/constants/field_flags.js');
4+
//var common = require('../../common');
5+
var mysql = require('../../../index.js')
6+
//var conn = common.createConnection();
7+
var conn = mysql.createConnection({
8+
password: 'mycause',
9+
user: 'mycause_dev',
10+
//supportBigNumbers: true,
11+
//bigNumberStrings: true,
12+
database: 'test',
13+
//debug: 1
14+
});
15+
16+
conn.query('CREATE TEMPORARY TABLE `tmp_longlong` ( ' +
17+
' `id` int(11) NOT NULL AUTO_INCREMENT, ' +
18+
' `ls` BIGINT SIGNED NOT NULL, ' +
19+
' `lu` BIGINT UNSIGNED NOT NULL, ' +
20+
' PRIMARY KEY (`id`) ' +
21+
' ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8');
22+
23+
var values = [
24+
['10', '10'],
25+
['-11', '11'],
26+
['965432100123456789', '1965432100123456789'],
27+
['-965432100123456789', '2965432100123456789']
28+
];
29+
30+
conn.connect(function (err) {
31+
if (err) {
32+
console.error(err);
33+
return;
34+
}
35+
36+
for (var i=0; i < values.length; ++i) {
37+
conn.query("INSERT INTO `tmp_longlong` VALUES (?, ?, ?)", [i+1, values[i][0], values[i][1]]);
38+
}
39+
40+
var bigNums_bnStringsFalse = [
41+
{ id: 1, ls: 10, lu: 10 },
42+
{ id: 2, ls: -11, lu: 11 },
43+
{ id: 3, ls: 965432100123456800, lu: 1965432100123456800 },
44+
{ id: 4, ls: -965432100123456800, lu: 2965432100123457000 }
45+
];
46+
47+
var bigNums_bnStringsTrueFalse = [
48+
{ id: 1, ls: 10, lu: 10 },
49+
{ id: 2, ls: -11, lu: 11 },
50+
{ id: 3, ls: '965432100123456789', lu: '1965432100123456789' },
51+
{ id: 4, ls: '-965432100123456789', lu: '2965432100123456789' }
52+
];
53+
54+
var bigNums_bnStringsTrueTrue = [
55+
{ id: 1, ls: 10, lu: 10 },
56+
{ id: 2, ls: -11, lu: 11 },
57+
{ id: 3, ls: '965432100123456789', lu: '1965432100123456789' },
58+
{ id: 4, ls: '-965432100123456789', lu: '2965432100123456789' }
59+
]
60+
61+
function check() {
62+
completed++;
63+
if (completed === started)
64+
conn.end();
65+
}
66+
67+
var completed = 0;
68+
var started = 0;
69+
70+
function testQuery(supportBigNumbers, bigNumberStrings, expectation) {
71+
started++;
72+
conn.query({
73+
sql: 'SELECT * from tmp_longlong',
74+
supportBigNumbers: supportBigNumbers,
75+
bigNumberStrings: bigNumberStrings
76+
}, function (err, rows, fields) {
77+
assert.ifError(err);
78+
assert.deepEqual(rows, expectation);
79+
completed++;
80+
if (completed === started)
81+
conn.end();
82+
});
83+
}
84+
85+
function testExecute(supportBigNumbers, bigNumberStrings, expectation) {
86+
started++;
87+
conn.execute({
88+
sql: 'SELECT * from tmp_longlong',
89+
supportBigNumbers: supportBigNumbers,
90+
bigNumberStrings: bigNumberStrings
91+
}, function (err, rows, fields) {
92+
assert.ifError(err);
93+
assert.deepEqual(rows, expectation);
94+
completed++;
95+
if (completed === started)
96+
conn.end();
97+
});
98+
}
99+
100+
testQuery(false, false, bigNums_bnStringsFalse);
101+
testQuery(true, false, bigNums_bnStringsTrueFalse);
102+
testQuery(true, true, bigNums_bnStringsTrueTrue);
103+
104+
testExecute(false, false, bigNums_bnStringsFalse);
105+
testExecute(true, false, bigNums_bnStringsTrueFalse);
106+
testExecute(true, true, bigNums_bnStringsTrueTrue);
107+
});

0 commit comments

Comments
 (0)