feat: big number support for cbor

bigint support in cbor

Author: kuhe

.changeset/real-pens-smile.md

@@ -0,0 +1,5 @@
+---
+"@smithy/core": minor
+---
+
+support BigInt in cbor

packages/core/src/submodules/cbor/SmithyRpcV2CborProtocol.spec.ts

@@ -155,7 +155,7 @@ describe(SmithyRpcV2CborProtocol.name, () => {
           {
             name: "dummy",
             input: testCase.schema,
-            output: void 0,
+            output: "unit",
             traits: {},
           },
           testCase.input,
@@ -257,7 +257,7 @@ describe(SmithyRpcV2CborProtocol.name, () => {
         const output = await protocol.deserializeResponse(
           {
             name: "dummy",
-            input: void 0,
+            input: "unit",
             output: testCase.schema,
             traits: {},
           },

packages/core/src/submodules/cbor/byte-printer.ts

@@ -0,0 +1,7 @@
+/**
+ * Prints bytes as binary string with numbers.
+ * @param bytes
+ */
+export function printBytes(bytes: Uint8Array) {
+  return [...bytes].map((n) => ("0".repeat(8) + n.toString(2)).slice(-8) + ` (${n})`);
+}

packages/core/src/submodules/cbor/cbor-decode.ts

@@ -119,11 +119,25 @@ export function decode(at: Uint32, to: Uint32): CborValueType {
         _offset = offset;
         return castBigInt(negativeInt);
       } else {
-        const value = decode(at + offset, to);
-        const valueOffset = _offset;
+        if (minor === 2 || minor === 3) {
+          const length = decodeCount(at + offset, to);
 
-        _offset = offset + valueOffset;
-        return tag({ tag: castBigInt(unsignedInt), value });
+          let b = BigInt(0);
+          const start = at + offset + _offset;
+          for (let i = start; i < start + length; ++i) {
+            b = (b << BigInt(8)) | BigInt(payload[i]);
+          }
+
+          _offset = offset + length;
+          return minor === 3 ? -b - BigInt(1) : b;
+        } else {
+          const value = decode(at + offset, to);
+          const valueOffset = _offset;
+
+          _offset = offset + valueOffset;
+
+          return tag({ tag: castBigInt(unsignedInt), value });
+        }
       }
     case majorUtf8String:
     case majorMap:

packages/core/src/submodules/cbor/cbor-encode.ts

@@ -1,6 +1,7 @@
 import { fromUtf8 } from "@smithy/util-utf8";
 
 import {
+  alloc,
   CborMajorType,
   extendedFloat16,
   extendedFloat32,
@@ -19,7 +20,6 @@ import {
   tagSymbol,
   Uint64,
 } from "./cbor-types";
-import { alloc } from "./cbor-types";
 
 const USE_BUFFER = typeof Buffer !== "undefined";
 
@@ -152,10 +152,30 @@ export function encode(_input: any): void {
         data[cursor++] = (major << 5) | extendedFloat32;
         dataView.setUint32(cursor, n);
         cursor += 4;
-      } else {
+      } else if (value < BigInt("18446744073709551616")) {
         data[cursor++] = (major << 5) | extendedFloat64;
         dataView.setBigUint64(cursor, value);
         cursor += 8;
+      } else {
+        // refer to https://www.rfc-editor.org/rfc/rfc8949.html#name-bignums
+        const binaryBigInt = value.toString(2);
+        const bigIntBytes = new Uint8Array(Math.ceil(binaryBigInt.length / 8));
+        let b = value;
+        let i = 0;
+        while (bigIntBytes.byteLength - ++i >= 0) {
+          bigIntBytes[bigIntBytes.byteLength - i] = Number(b & BigInt(255));
+          b >>= BigInt(8);
+        }
+        ensureSpace(bigIntBytes.byteLength * 2);
+        data[cursor++] = nonNegative ? 0b110_00010 : 0b110_00011;
+
+        if (USE_BUFFER) {
+          encodeHeader(majorUnstructuredByteString, Buffer.byteLength(bigIntBytes));
+        } else {
+          encodeHeader(majorUnstructuredByteString, bigIntBytes.byteLength);
+        }
+        data.set(bigIntBytes, cursor);
+        cursor += bigIntBytes.byteLength;
       }
       continue;
     } else if (input === null) {

packages/core/src/submodules/cbor/cbor.spec.ts

@@ -88,12 +88,12 @@ describe("cbor", () => {
     {
       name: "negative float",
       data: -3015135.135135135,
-      cbor: allocByteArray([0b111_11011, +193, +71, +0, +239, +145, +76, +27, +173]),
+      cbor: allocByteArray([0b111_11011, 193, 71, 0, 239, 145, 76, 27, 173]),
     },
     {
       name: "positive float",
       data: 3015135.135135135,
-      cbor: allocByteArray([0b111_11011, +65, +71, +0, +239, +145, +76, +27, +173]),
+      cbor: allocByteArray([0b111_11011, 65, 71, 0, 239, 145, 76, 27, 173]),
     },
     {
       name: "various numbers",
@@ -214,6 +214,18 @@ describe("cbor", () => {
         65, 109, 110, 101, 115, 116, 101, 100, 32, 105, 116, 101, 109, 32, 66,
       ]),
     },
+    {
+      name: "object containing big numbers",
+      data: {
+        map: {
+          items: [BigInt(1e80)],
+        },
+      },
+      cbor: allocByteArray([
+        161, 99, 109, 97, 112, 161, 101, 105, 116, 101, 109, 115, 129, 194, 88, 34, 3, 95, 157, 234, 62, 31, 107, 224,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      ]),
+    },
   ];
 
   const toBytes = (hex: string) => {
@@ -226,6 +238,48 @@ describe("cbor", () => {
   };
 
   describe("locally curated scenarios", () => {
+    it("should round-trip bigInteger to major 6 with tag 2", () => {
+      const bigInt = BigInt("1267650600228229401496703205376");
+      const serialized = cbor.serialize(bigInt);
+
+      const major = serialized[0] >> 5;
+      expect(major).toEqual(0b110); // 6
+
+      const tag = serialized[0] & 0b11111;
+      expect(tag).toEqual(0b010); // 2
+
+      const byteStringCount = serialized[1];
+      expect(byteStringCount).toEqual(0b010_01101); // major 2, 13 bytes
+
+      const byteString = serialized.slice(2);
+      expect(byteString).toEqual(allocByteArray([0b000_10000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]));
+
+      const deserialized = cbor.deserialize(serialized);
+      expect(deserialized).toEqual(bigInt);
+    });
+
+    it("should round-trip negative bigInteger to major 6 with tag 3", () => {
+      const bigInt = BigInt("-1267650600228229401496703205377");
+      const serialized = cbor.serialize(bigInt);
+
+      const major = serialized[0] >> 5;
+      expect(major).toEqual(0b110); // 6
+
+      const tag = serialized[0] & 0b11111;
+      expect(tag).toEqual(0b011); // 3
+
+      const byteStringCount = serialized[1];
+      expect(byteStringCount).toEqual(0b010_01101); // major 2, 13 bytes
+
+      const byteString = serialized.slice(2);
+      expect(byteString).toEqual(allocByteArray([0b000_10000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]));
+
+      const deserialized = cbor.deserialize(serialized);
+      expect(deserialized).toEqual(bigInt);
+    });
+
+    it("should round-trip NumericValue to major 6 with tag 4", () => {});
+
     it("should throw an error if serializing a tag with missing properties", () => {
       expect(() =>
         cbor.serialize({

packages/core/src/submodules/protocols/HttpBindingProtocol.spec.ts

@@ -5,6 +5,7 @@ import {
   CodecSettings,
   HandlerExecutionContext,
   HttpResponse as IHttpResponse,
+  MetadataBearer,
   OperationSchema,
   ResponseMetadata,
   ShapeDeserializer,
@@ -64,7 +65,7 @@ describe(HttpBindingProtocol.name, () => {
     });
 
     const protocol = new StringRestProtocol();
-    const output = await protocol.deserializeResponse(
+    const output = (await protocol.deserializeResponse(
       op(
         "",
         "",
@@ -87,7 +88,7 @@ describe(HttpBindingProtocol.name, () => {
       ),
       {} as any,
       response
-    );
+    )) as Partial<MetadataBearer>;
     delete output.$metadata;
     expect(output).toEqual({
       timestampList: [new Date("2019-12-16T23:48:18.000Z"), new Date("2019-12-16T23:48:18.000Z")],
@@ -104,7 +105,7 @@ describe(HttpBindingProtocol.name, () => {
     });
 
     const protocol = new StringRestProtocol();
-    const output = await protocol.deserializeResponse(
+    const output = (await protocol.deserializeResponse(
       op(
         "",
         "",
@@ -127,7 +128,7 @@ describe(HttpBindingProtocol.name, () => {
       ),
       {} as any,
       response
-    );
+    )) as Partial<MetadataBearer>;
     delete output.$metadata;
     expect(output).toEqual({
       httpPrefixHeaders: {

packages/core/src/submodules/serde/parse-utils.spec.ts

@@ -22,6 +22,8 @@ import {
 } from "./parse-utils";
 import { expectBoolean, expectNumber, expectString } from "./parse-utils";
 
+logger.warn = () => {};
+
 describe("parseBoolean", () => {
   it('Returns true for "true"', () => {
     expect(parseBoolean("true")).toEqual(true);

packages/core/vitest.config.ts

@@ -5,5 +5,6 @@ export default defineConfig({
     exclude: ["**/*.{integ,e2e,browser}.spec.ts"],
     include: ["**/*.spec.ts"],
     environment: "node",
+    hideSkippedTests: true,
   },
 });