Handle 64 bit numbers

Author: zachasme

README.md

@@ -73,12 +73,12 @@ var data = new Pbf(buffer).readFields(readData, {});
 
 function readData(tag, data, pbf) {
     if (tag === 1) data.name = pbf.readString();
-    else if (tag === 2) data.version = pbf.readVarint();
+    else if (tag === 2) data.version = pbf.readVarint32();
     else if (tag === 3) data.layer = pbf.readMessage(readLayer, {});
 }
 function readLayer(tag, layer, pbf) {
     if (tag === 1) layer.name = pbf.readString();
-    else if (tag === 3) layer.size = pbf.readVarint();
+    else if (tag === 3) layer.size = pbf.readVarint32();
 }
 ```
 
@@ -143,7 +143,7 @@ Read a sequence of fields:
 
 ```js
 pbf.readFields(function (tag) {
-    if (tag === 1) pbf.readVarint();
+    if (tag === 1) pbf.readVarint32();
     else if (tag === 2) pbf.readString();
     else ...
 });
@@ -156,7 +156,7 @@ and also passes the `Pbf` object as a third argument:
 var result = pbf.readFields(callback, {})
 
 function callback(tag, result, pbf) {
-    if (tag === 1) result.id = pbf.readVarint();
+    if (tag === 1) result.id = pbf.readVarint32();
 }
 ```
 
@@ -165,9 +165,9 @@ To read an embedded message, use `pbf.readMessage(fn[, obj])` (in the same way a
 Read values:
 
 ```js
-var value = pbf.readVarint();
+var value = pbf.readVarint32();
 var str = pbf.readString();
-var numbers = pbf.readPackedVarint();
+var numbers = pbf.readPackedVarint32();
 ```
 
 For lazy or partial decoding, simply save the position instead of reading a value,
@@ -185,8 +185,8 @@ pbf.readMessage(readFoo);
 
 Scalar reading methods:
 
-* `readVarint(isSigned)` (pass `true` if you expect negative varints)
-* `readSVarint()`
+* `readVarint32(isSigned)` (pass `true` if you expect negative varints)
+* `readSVarint32()`
 * `readFixed32()`
 * `readFixed64()`
 * `readSFixed32()`
@@ -200,8 +200,8 @@ Scalar reading methods:
 
 Packed reading methods:
 
-* `readPackedVarint(arr, isSigned)` (appends read items to `arr`)
-* `readPackedSVarint(arr)`
+* `readPackedVarint32(arr, isSigned)` (appends read items to `arr`)
+* `readPackedSVarint32(arr)`
 * `readPackedFixed32(arr)`
 * `readPackedFixed64(arr)`
 * `readPackedSFixed32(arr)`
@@ -215,7 +215,7 @@ Packed reading methods:
 Write values:
 
 ```js
-pbf.writeVarint(123);
+pbf.writeVarint32(123);
 pbf.writeString("Hello world");
 ```
 
@@ -246,8 +246,8 @@ Field writing methods:
 
 Packed field writing methods:
 
-* `writePackedVarint(tag, val)`
-* `writePackedSVarint(tag, val)`
+* `writePackedVarint32(tag, val)`
+* `writePackedSVarint32(tag, val)`
 * `writePackedSFixed32(tag, val)`
 * `writePackedSFixed64(tag, val)`
 * `writePackedBoolean(tag, val)`
@@ -256,8 +256,8 @@ Packed field writing methods:
 
 Scalar writing methods:
 
-* `writeVarint(val)`
-* `writeSVarint(val)`
+* `writeVarint32(val)`
+* `writeSVarint32(val)`
 * `writeSFixed32(val)`
 * `writeSFixed64(val)`
 * `writeBoolean(val)`
@@ -345,7 +345,7 @@ This release include tons of compatibility/robustness fixes, and a more reliable
 - Fixed `writeVarint` to write `0` when given `NaN` or other non-number to avoid producing a broken Protobuf message.
 - Changed `readPacked*` methods signature to accept an optional `arr` argument to append the results to (to support messages with repeated fields that mix packed/non-packed encoding).
 - Added an optional `isSigned` argument to `readVarint` that enables proper reading of negative varints.
-- Deprecated `readVarint64()` (it still works, but it's recommended to be changed to `readVarint(true)`).
+- Deprecated `readVarint64()` (it still works, but it's recommended to be changed to `readVarint32(true)`).
 - Faster string encoding.
 
 ##### Proto compiler

compile.js

@@ -144,9 +144,9 @@ function compileFieldRead(ctx, field) {
     case 'bool':     return prefix + 'Boolean' + suffix;
     case 'enum':
     case 'uint32':
+    case 'int32':    return prefix + 'Varint' + suffix;
     case 'uint64':
-    case 'int32':
-    case 'int64':    return prefix + 'Varint' + suffix;
+    case 'int64':    return prefix + 'Varint64' + suffix;
     case 'sint32':
     case 'sint64':   return prefix + 'SVarint' + suffix;
     case 'fixed32':  return prefix + 'Fixed32' + suffix;
@@ -178,9 +178,9 @@ function compileFieldWrite(ctx, field, name) {
     case 'bool':     return prefix + 'Boolean' + postfix;
     case 'enum':
     case 'uint32':
+    case 'int32':    return prefix + 'Varint' + postfix;
     case 'uint64':
-    case 'int32':
-    case 'int64':    return prefix + 'Varint' + postfix;
+    case 'int64':    return prefix + 'Varint64' + postfix;
     case 'sint32':
     case 'sint64':   return prefix + 'SVarint' + postfix;
     case 'fixed32':  return prefix + 'Fixed32' + postfix;

index.js

@@ -1,3 +1,4 @@
+/* eslint-disable new-cap, no-undef */
 'use strict';
 
 module.exports = Pbf;
@@ -90,7 +91,7 @@ Pbf.prototype = {
         return val;
     },
 
-    readVarint: function(isSigned) {
+    readVarint: function(isSigned, is64) {
         var buf = this.buf,
             val, b;
 
@@ -100,11 +101,11 @@ Pbf.prototype = {
         b = buf[this.pos++]; val |= (b & 0x7f) << 21; if (b < 0x80) return val;
         b = buf[this.pos];   val |= (b & 0x0f) << 28;
 
-        return readVarintRemainder(val, isSigned, this);
+        return readVarintRemainder(val, isSigned, is64, this);
     },
 
-    readVarint64: function() { // for compatibility with v2.0.1
-        return this.readVarint(true);
+    readVarint64: function(isSigned) {
+        return this.readVarint(isSigned, true);
     },
 
     readSVarint: function() {
@@ -266,7 +267,7 @@ Pbf.prototype = {
         val = +val || 0;
 
         if (val > 0xfffffff || val < 0) {
-            writeBigVarint(val, this);
+            writeBigVarint(BigInt(val), this);
             return;
         }
 
@@ -282,6 +283,24 @@ Pbf.prototype = {
         this.writeVarint(val < 0 ? -val * 2 - 1 : val * 2);
     },
 
+    writeVarint64: function(val) {
+        val = BigInt(val);
+
+        if (val > 0xfffffff || val < 0) {
+            writeBigVarint(val, this);
+            return;
+        }
+
+        val = Number(val);
+
+        this.realloc(4);
+
+        this.buf[this.pos++] =           val & 0x7f  | (val > 0x7f ? 0x80 : 0); if (val <= 0x7f) return;
+        this.buf[this.pos++] = ((val >>>= 7) & 0x7f) | (val > 0x7f ? 0x80 : 0); if (val <= 0x7f) return;
+        this.buf[this.pos++] = ((val >>>= 7) & 0x7f) | (val > 0x7f ? 0x80 : 0); if (val <= 0x7f) return;
+        this.buf[this.pos++] =   (val >>> 7) & 0x7f;
+    },
+
     writeBoolean: function(val) {
         this.writeVarint(Boolean(val));
     },
@@ -379,6 +398,10 @@ Pbf.prototype = {
         this.writeTag(tag, Pbf.Varint);
         this.writeVarint(val);
     },
+    writeVarint64Field: function(tag, val) {
+        this.writeTag(tag, Pbf.Varint);
+        this.writeVarint64(val);
+    },
     writeSVarintField: function(tag, val) {
         this.writeTag(tag, Pbf.Varint);
         this.writeSVarint(val);
@@ -400,16 +423,16 @@ Pbf.prototype = {
     }
 };
 
-function readVarintRemainder(l, s, p) {
+function readVarintRemainder(l, s, is64, p) {
     var buf = p.buf,
         h, b;
 
-    b = buf[p.pos++]; h  = (b & 0x70) >> 4;  if (b < 0x80) return toNum(l, h, s);
-    b = buf[p.pos++]; h |= (b & 0x7f) << 3;  if (b < 0x80) return toNum(l, h, s);
-    b = buf[p.pos++]; h |= (b & 0x7f) << 10; if (b < 0x80) return toNum(l, h, s);
-    b = buf[p.pos++]; h |= (b & 0x7f) << 17; if (b < 0x80) return toNum(l, h, s);
-    b = buf[p.pos++]; h |= (b & 0x7f) << 24; if (b < 0x80) return toNum(l, h, s);
-    b = buf[p.pos++]; h |= (b & 0x01) << 31; if (b < 0x80) return toNum(l, h, s);
+    b = buf[p.pos++]; h  = (b & 0x70) >> 4;  if (b < 0x80) return toNum(l, h, s, is64);
+    b = buf[p.pos++]; h |= (b & 0x7f) << 3;  if (b < 0x80) return toNum(l, h, s, is64);
+    b = buf[p.pos++]; h |= (b & 0x7f) << 10; if (b < 0x80) return toNum(l, h, s, is64);
+    b = buf[p.pos++]; h |= (b & 0x7f) << 17; if (b < 0x80) return toNum(l, h, s, is64);
+    b = buf[p.pos++]; h |= (b & 0x7f) << 24; if (b < 0x80) return toNum(l, h, s, is64);
+    b = buf[p.pos++]; h |= (b & 0x01) << 31; if (b < 0x80) return toNum(l, h, s, is64);
 
     throw new Error('Expected varint not more than 10 bytes');
 }
@@ -419,7 +442,18 @@ function readPackedEnd(pbf) {
         pbf.readVarint() + pbf.pos : pbf.pos + 1;
 }
 
-function toNum(low, high, isSigned) {
+function toNum(low, high, isSigned, is64) {
+    if (is64) {
+        var int;
+        if (isSigned) {
+            int = BigInt(high) * BigInt(0x100000000) + BigInt(low >>> 0);
+            return int.toString();
+        }
+
+        int = (BigInt(high >>> 0) * BigInt(0x100000000)) + BigInt(low >>> 0);
+        return int.toString();
+    }
+
     if (isSigned) {
         return high * 0x100000000 + (low >>> 0);
     }
@@ -431,11 +465,11 @@ function writeBigVarint(val, pbf) {
     var low, high;
 
     if (val >= 0) {
-        low  = (val % 0x100000000) | 0;
-        high = (val / 0x100000000) | 0;
+        low  = Number(val % BigInt(0x100000000)) | 0;
+        high = Number(val / BigInt(0x100000000)) | 0;
     } else {
-        low  = ~(-val % 0x100000000);
-        high = ~(-val / 0x100000000);
+        low  = ~Number(-val % BigInt(0x100000000));
+        high = ~Number(-val / BigInt(0x100000000));
 
         if (low ^ 0xffffffff) {
             low = (low + 1) | 0;

test/compile.test.js

@@ -259,14 +259,14 @@ test('handles negative varint', function(t) {
 
     Envelope.write({
         int: -5,
-        long: -10
+        long: '-10'
     }, pbf);
 
     var buf = pbf.finish();
     var data = Envelope.read(new Pbf(buf));
 
     t.equals(data.int, -5);
-    t.equals(data.long, -10);
+    t.equals(data.long, '-10');
 
     t.end();
 });
@@ -278,14 +278,14 @@ test('handles unsigned varint', function(t) {
 
     Envelope.write({
         uint: Math.pow(2, 31),
-        ulong: Math.pow(2, 63)
+        ulong: '18446744073709551615'
     }, pbf);
 
     var buf = pbf.finish();
     var data = Envelope.read(new Pbf(buf));
 
     t.equals(data.uint, Math.pow(2, 31));
-    t.equals(data.ulong, Math.pow(2, 63));
+    t.equals(data.ulong, '18446744073709551615');
 
     t.end();
 });

test/pbf.test.js

@@ -95,7 +95,7 @@ test('readVarint signed', function(t) {
 test('readVarint64 (compatibility)', function(t) {
     var bytes = [0xc8,0xe8,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x01];
     var buf = new Pbf(new Buffer(bytes));
-    t.equal(buf.readVarint64(), -3000);
+    t.equal(buf.readVarint64(true), '-3000');
     t.end();
 });