Skip to content

Commit 536041b

Browse files
authored
Merge pull request #366 from sidorares/longlong-precision-fixes
64 bit long fixes
2 parents de68ca1 + 77f933e commit 536041b

File tree

6 files changed

+255
-31
lines changed

6 files changed

+255
-31
lines changed

lib/compile_binary_parser.js

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ function compile (fields, options, config) {
7272
result.push(' if (nullBitmaskByte' + nullByteIndex + ' & ' + currentFieldNullBit + ')');
7373
result.push(' ' + lvalue + ' = null;');
7474
result.push(' else');
75-
result.push(' ' + lvalue + ' = ' + readCodeFor(fields[i], config));
75+
result.push(' ' + lvalue + ' = ' + readCodeFor(fields[i], config, options));
7676
// }
7777
currentFieldNullBit *= 2;
7878
if (currentFieldNullBit == 0x100) {
@@ -95,7 +95,10 @@ function compile (fields, options, config) {
9595
return vm.runInThisContext(src);
9696
}
9797

98-
function readCodeFor (field, config) {
98+
function readCodeFor (field, config, options) {
99+
var supportBigNumbers = options.supportBigNumbers || config.supportBigNumbers;
100+
var bigNumberStrings = options.bigNumberStrings || config.bigNumberStrings;
101+
99102
var unsigned = field.flags & FieldFlags.UNSIGNED;
100103
switch (field.columnType) {
101104
case Types.TINY:
@@ -133,8 +136,16 @@ function readCodeFor (field, config) {
133136
return 'packet.parseGeometryValue();';
134137
case Types.JSON:
135138
return 'JSON.parse(packet.readLengthCodedString());';
136-
case Types.LONGLONG: // TODO: 8 bytes. Implement as two 4 bytes read for now (it's out of JavaScript int precision!)
137-
return unsigned ? 'packet.readInt64();' : 'packet.readSInt64();';
139+
case Types.LONGLONG:
140+
if (!supportBigNumbers) {
141+
return unsigned ? 'packet.readInt64JSNumber();' : 'packet.readSInt64JSNumber();';
142+
} else {
143+
if (bigNumberStrings) {
144+
return unsigned ? 'packet.readInt64String();' : 'packet.readSInt64String();';
145+
} else {
146+
return unsigned ? 'packet.readInt64();' : 'packet.readSInt64();';
147+
}
148+
}
138149
default:
139150
if (field.characterSet == Charsets.BINARY) {
140151
return 'packet.readLengthCodedBuffer();';

lib/compile_text_parser.js

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ function compile (fields, options, config) {
7373
} else {
7474
lvalue = ' this[' + srcEscape(fields[i].name) + ']';
7575
}
76-
var readCode = readCodeFor(fields[i].columnType, fields[i].characterSet, config);
76+
var readCode = readCodeFor(fields[i].columnType, fields[i].characterSet, config, options);
7777
if (typeof options.typeCast === 'function') {
7878
result.push(lvalue + ' = options.typeCast(wrap(fields[' + i + '], ' + srcEscape(typeNames[fields[i].columnType]) + ', packet), function() { return ' + readCode + ';})');
7979
} else if (options.typeCast === false) {
@@ -98,19 +98,22 @@ function compile (fields, options, config) {
9898
return vm.runInThisContext(src);
9999
}
100100

101-
function readCodeFor (type, charset, config) {
101+
function readCodeFor (type, charset, config, options) {
102+
var supportBigNumbers = options.supportBigNumbers || config.supportBigNumbers;
103+
var bigNumberStrings = options.bigNumberStrings || config.bigNumberStrings;
104+
102105
switch (type) {
103106
case Types.TINY:
104107
case Types.SHORT:
105108
case Types.LONG:
106109
case Types.INT24:
107110
case Types.YEAR:
108-
return 'packet.parseLengthCodedInt()';
111+
return 'packet.parseLengthCodedIntNoBigCheck()';
109112
case Types.LONGLONG:
110-
if (config.supportBigNumbers && config.bigNumberStrings) {
113+
if (supportBigNumbers && bigNumberStrings) {
111114
return 'packet.parseLengthCodedIntString()';
112115
}
113-
return 'packet.parseLengthCodedInt()';
116+
return 'packet.parseLengthCodedInt(' + supportBigNumbers + ')';
114117
case Types.FLOAT:
115118
case Types.DOUBLE:
116119
return 'packet.parseLengthCodedFloat()';

lib/connection.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -382,7 +382,8 @@ Connection.prototype.resume = function resume () {
382382
};
383383

384384
Connection.prototype.keyFromFields = function keyFromFields (fields, options) {
385-
var res = (typeof options.nestTables) + '/' + options.nestTables + '/' + options.rowsAsArray;
385+
var res = (typeof options.nestTables) + '/' + options.nestTables + '/' + options.rowsAsArray +
386+
+ options.supportBigNumbers + '/' + options.bigNumberStrings;
386387
for (var i = 0; i < fields.length; ++i) {
387388
res += '/' + fields[i].name + ':' + fields[i].columnType + ':' + fields[i].flags;
388389
}

lib/packets/packet.js

Lines changed: 127 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -81,17 +81,63 @@ Packet.prototype.readSInt32 = function ()
8181
return this.buffer.readInt32LE(this.offset - 4, true);
8282
};
8383

84-
Packet.prototype.readInt64 = function () {
85-
return this.readInt32() + 0x100000000 * this.readInt32();
84+
Packet.prototype.readInt64JSNumber = function () {
85+
var word0 = this.readInt32();
86+
var word1 = this.readInt32();
87+
var l = new Long(word0, word1, true);
88+
return l.toNumber();
8689
};
8790

88-
Packet.prototype.readSInt64 = function () {
91+
Packet.prototype.readSInt64JSNumber = function () {
8992
var word0 = this.readInt32();
9093
var word1 = this.readInt32();
9194
if (!(word1 & 0x80000000)) {
9295
return word0 + 0x100000000 * word1;
9396
}
94-
return -((((~word1) >>> 0) * 0x100000000) + ((~word0) >>> 0) + 1);
97+
var l = new Long(word0, word1, false);
98+
return l.toNumber();
99+
};
100+
101+
Packet.prototype.readInt64String = function () {
102+
var word0 = this.readInt32();
103+
var word1 = this.readInt32();
104+
var res = new Long(word0, word1, true);
105+
return res.toString();
106+
};
107+
108+
Packet.prototype.readSInt64String = function () {
109+
var word0 = this.readInt32();
110+
var word1 = this.readInt32();
111+
var res = new Long(word0, word1, false);
112+
return res.toString();
113+
};
114+
115+
Packet.prototype.readInt64 = function () {
116+
var word0 = this.readInt32();
117+
var word1 = this.readInt32();
118+
var res = new Long(word0, word1, true);
119+
var resNumber = res.toNumber()
120+
, resString = res.toString();
121+
122+
res = resNumber.toString() === resString
123+
? resNumber
124+
: resString;
125+
126+
return res;
127+
};
128+
129+
Packet.prototype.readSInt64 = function () {
130+
var word0 = this.readInt32();
131+
var word1 = this.readInt32();
132+
var res = new Long(word0, word1, false);
133+
var resNumber = res.toNumber()
134+
, resString = res.toString();
135+
136+
res = resNumber.toString() === resString
137+
? resNumber
138+
: resString;
139+
140+
return res;
95141
};
96142

97143
Packet.prototype.isEOF = function () {
@@ -328,17 +374,35 @@ Packet.prototype.readString = function (len) {
328374
return this.buffer.utf8Slice(this.offset - len, this.offset);
329375
};
330376

377+
// The whole reason parse* function below exist
378+
// is because String creation is relatively expensive (at least with V8), and if we have
379+
// a buffer with "12345" content ideally we would like to bypass intermediate
380+
// "12345" string creation and directly build 12345 number out of
381+
// <Buffer 31 32 33 34 35> data.
382+
// In my benchmarks the difference is ~25M 8-digit numbers per second vs
383+
// 4.5 M using Number(packet.readLengthCodedString())
384+
// not used when size is close to max precision as series of *10 accumulate error
385+
// and approximate result mihgt be diffreent from (approximate as well) Number(bigNumStringValue))
386+
// In the futire node version if speed difference is smaller parse* functions might be removed
387+
// don't consider them as Packet public API
388+
331389
var minus = '-'.charCodeAt(0);
332390
var plus = '+'.charCodeAt(0);
333-
// TODO: faster versions of parseInt for smaller types?
334-
// discard overflow checks if we know from type definition it's safe so
335-
Packet.prototype.parseInt = function (len) {
391+
392+
Packet.prototype.parseInt = function (len, supportBigNumbers) {
336393

337394
if (len === null) {
338395
return null;
339396
}
340397

398+
if (len >= 14 && !supportBigNumbers) {
399+
var s = this.buffer.toString('ascii', this.offset, this.offset + len);
400+
this.offset += len;
401+
return Number(s);
402+
}
403+
341404
var result = 0;
405+
var start = this.offset;
342406
var end = this.offset + len;
343407
var sign = 1;
344408
if (len === 0) {
@@ -351,23 +415,62 @@ Packet.prototype.parseInt = function (len) {
351415
}
352416

353417
// max precise int is 9007199254740992
354-
// note that we are here after handling optional +/-
355-
// treat everything above 9000000000000000 as potential overflow
356418
var str;
357419
var numDigits = end - this.offset;
358-
if (numDigits == 16 && (this.buffer[this.offset] - 48) > 8) {
359-
str = this.readString(end - this.offset);
360-
result = parseInt(str, 10);
361-
if (result.toString() == str) {
362-
return sign * result;
363-
} else {
420+
if (supportBigNumbers) {
421+
if (numDigits >= 15) {
422+
str = this.readString(end - this.offset);
423+
result = parseInt(str, 10);
424+
if (result.toString() == str) {
425+
return sign * result;
426+
} else {
427+
return sign == -1 ? '-' + str : str;
428+
}
429+
} else if (numDigits > 16) {
430+
str = this.readString(end - this.offset);
364431
return sign == -1 ? '-' + str : str;
365432
}
366-
} else if (numDigits > 16) {
367-
str = this.readString(end - this.offset);
368-
return sign == -1 ? '-' + str : str;
369433
}
370434

435+
if (this.buffer[this.offset] == plus) {
436+
this.offset++; // just ignore
437+
}
438+
while (this.offset < end) {
439+
result *= 10;
440+
result += this.buffer[this.offset] - 48;
441+
this.offset++;
442+
}
443+
var num = result * sign;
444+
if (!supportBigNumbers) {
445+
return num;
446+
}
447+
str = this.buffer.toString('ascii', start, end);
448+
if (num.toString() == str) {
449+
return num;
450+
} else {
451+
return str;
452+
}
453+
};
454+
455+
// note that if value of inputNumberAsString is bigger than MAX_SAFE_INTEGER
456+
// ( or smaller than MIN_SAFE_INTEGER ) the parseIntNoBigCheck result might be
457+
// different from what you would get from Number(inputNumberAsString)
458+
// String(parseIntNoBigCheck) <> String(Number(inputNumberAsString)) <> inputNumberAsString
459+
Packet.prototype.parseIntNoBigCheck = function (len) {
460+
if (len === null) {
461+
return null;
462+
}
463+
var result = 0;
464+
var end = this.offset + len;
465+
var sign = 1;
466+
if (len === 0) {
467+
return 0; // TODO: assert? exception?
468+
}
469+
470+
if (this.buffer[this.offset] == minus) {
471+
this.offset++;
472+
sign = -1;
473+
}
371474
if (this.buffer[this.offset] == plus) {
372475
this.offset++; // just ignore
373476
}
@@ -515,8 +618,12 @@ Packet.prototype.parseFloat = function (len) {
515618
return result / factor;
516619
};
517620

518-
Packet.prototype.parseLengthCodedInt = function () {
519-
return this.parseInt(this.readLengthCodedNumber());
621+
Packet.prototype.parseLengthCodedIntNoBigCheck = function () {
622+
return this.parseIntNoBigCheck(this.readLengthCodedNumber());
623+
};
624+
625+
Packet.prototype.parseLengthCodedInt = function (supportBigNumbers) {
626+
return this.parseInt(this.readLengthCodedNumber(), supportBigNumbers);
520627
};
521628

522629
Packet.prototype.parseLengthCodedIntString = function () {
Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
var assert = require('assert');
2+
3+
var FieldFlags = require('../../../lib/constants/field_flags.js');
4+
var common = require('../../common');
5+
var conn = common.createConnection();
6+
7+
conn.query('CREATE TEMPORARY TABLE `tmp_longlong` ( ' +
8+
' `id` int(11) NOT NULL AUTO_INCREMENT, ' +
9+
' `ls` BIGINT SIGNED NOT NULL, ' +
10+
' `lu` BIGINT UNSIGNED NOT NULL, ' +
11+
' PRIMARY KEY (`id`) ' +
12+
' ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8');
13+
14+
var values = [
15+
['10', '10'],
16+
['-11', '11'],
17+
['965432100123456789', '1965432100123456789'],
18+
['-965432100123456789', '2965432100123456789']
19+
];
20+
21+
conn.connect(function (err) {
22+
if (err) {
23+
console.error(err);
24+
return;
25+
}
26+
27+
for (var i=0; i < values.length; ++i) {
28+
conn.query("INSERT INTO `tmp_longlong` VALUES (?, ?, ?)", [i+1, values[i][0], values[i][1]]);
29+
}
30+
31+
var bigNums_bnStringsFalse = [
32+
{ id: 1, ls: 10, lu: 10 },
33+
{ id: 2, ls: -11, lu: 11 },
34+
{ id: 3, ls: 965432100123456800, lu: 1965432100123456800 },
35+
{ id: 4, ls: -965432100123456800, lu: 2965432100123457000 }
36+
];
37+
38+
var bigNums_bnStringsTrueFalse = [
39+
{ id: 1, ls: 10, lu: 10 },
40+
{ id: 2, ls: -11, lu: 11 },
41+
{ id: 3, ls: '965432100123456789', lu: '1965432100123456789' },
42+
{ id: 4, ls: '-965432100123456789', lu: '2965432100123456789' }
43+
];
44+
45+
var bigNums_bnStringsTrueTrue = [
46+
{ id: 1, ls: 10, lu: 10 },
47+
{ id: 2, ls: -11, lu: 11 },
48+
{ id: 3, ls: '965432100123456789', lu: '1965432100123456789' },
49+
{ id: 4, ls: '-965432100123456789', lu: '2965432100123456789' }
50+
]
51+
52+
function check() {
53+
completed++;
54+
if (completed === started)
55+
conn.end();
56+
}
57+
58+
var completed = 0;
59+
var started = 0;
60+
61+
function testQuery(supportBigNumbers, bigNumberStrings, expectation) {
62+
started++;
63+
conn.query({
64+
sql: 'SELECT * from tmp_longlong',
65+
supportBigNumbers: supportBigNumbers,
66+
bigNumberStrings: bigNumberStrings
67+
}, function (err, rows, fields) {
68+
assert.ifError(err);
69+
assert.deepEqual(rows, expectation);
70+
completed++;
71+
if (completed === started)
72+
conn.end();
73+
});
74+
}
75+
76+
function testExecute(supportBigNumbers, bigNumberStrings, expectation) {
77+
started++;
78+
conn.execute({
79+
sql: 'SELECT * from tmp_longlong',
80+
supportBigNumbers: supportBigNumbers,
81+
bigNumberStrings: bigNumberStrings
82+
}, function (err, rows, fields) {
83+
assert.ifError(err);
84+
assert.deepEqual(rows, expectation);
85+
completed++;
86+
if (completed === started)
87+
conn.end();
88+
});
89+
}
90+
91+
testQuery(false, false, bigNums_bnStringsFalse);
92+
testQuery(true, false, bigNums_bnStringsTrueFalse);
93+
testQuery(true, true, bigNums_bnStringsTrueTrue);
94+
95+
testExecute(false, false, bigNums_bnStringsFalse);
96+
testExecute(true, false, bigNums_bnStringsTrueFalse);
97+
testExecute(true, true, bigNums_bnStringsTrueTrue);
98+
});

test/integration/connection/test-insert-bigint.js

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,11 @@ connection.query('INSERT INTO bigs SET title=\'test1\'', function (err, result)
3030
connection.query('INSERT INTO bigs SET title=\'test\', id=90071992547409924');
3131
connection.query('INSERT INTO bigs SET title=\'test4\'', function (err, result) {
3232
assert.strictEqual((Long.fromString('90071992547409925')).compare(result.insertId), 0);
33-
connection.query('select * from bigs', function (err, result) {
33+
connection.query({
34+
sql: 'select * from bigs',
35+
supportBigNumbers: true,
36+
bigNumberString: false,
37+
}, function (err, result) {
3438
assert.strictEqual(result[0].id, 123);
3539
assert.strictEqual(result[1].id, 124);
3640
assert.strictEqual(result[2].id, 123456789);

0 commit comments

Comments
 (0)