Skip to content

Commit 10a0534

Browse files
kuhekuhe
authored
feat: big number support for cbor (#1603)
* feat: big number support for cbor bigint support in cbor * chore: add local model to test codegen for big numbers * chore: fix datasource variable
1 parent 1308a1e commit 10a0534

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+1520
-19
lines changed

‎.changeset/real-pens-smile.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@smithy/core": minor
3+
---
4+
5+
support BigInt in cbor

‎Makefile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,14 @@ generate-protocol-tests:
1111
./gradlew :smithy-typescript-protocol-test-codegen:build
1212
rm -rf ./private/smithy-rpcv2-cbor
1313
rm -rf ./private/smithy-rpcv2-cbor-schema
14+
rm -rf ./private/my-local-model
1415
cp -r ./smithy-typescript-protocol-test-codegen/build/smithyprojections/smithy-typescript-protocol-test-codegen/smithy-rpcv2-cbor/typescript-codegen ./private/smithy-rpcv2-cbor
1516
cp -r ./smithy-typescript-protocol-test-codegen/build/smithyprojections/smithy-typescript-protocol-test-codegen/smithy-rpcv2-cbor-schema/typescript-codegen ./private/smithy-rpcv2-cbor-schema
17+
cp -r ./smithy-typescript-protocol-test-codegen/build/smithyprojections/smithy-typescript-protocol-test-codegen/my-local-model/typescript-client-codegen/ ./private/my-local-model
1618
node ./scripts/post-protocol-test-codegen
1719
npx prettier --write ./private/smithy-rpcv2-cbor
1820
npx prettier --write ./private/smithy-rpcv2-cbor-schema
21+
npx prettier --write ./private/my-local-model
1922
yarn
2023

2124
test-protocols:

‎README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,7 @@ By default, the Smithy TypeScript code generators provide the code generation fr
194194
| `protocol` | No | The Shape ID of the protocol used to generate serialization and deserialization. If not provided, the code generator will attempt to resolve the highest priority service protocol supported in code generation (registered through `TypeScriptIntegration`). If no protocols are found, code generation will use serialization and deserialization error stubs. |
195195
| `private` | No | Whether the package is `private` in `package.json`. The default value is `false`. |
196196
| `requiredMemberMode` | No | **NOT RECOMMENDED DUE TO BACKWARD COMPATIBILITY CONCERNS.** Sets whether members marked with the `@required` trait are allowed to be `undefined`. See more details on the risks in `TypeScriptSettings.RequiredMemberMode`. The default value is `nullable`. |
197+
| `bigNumberMode` | No | use `"native"` to serialize and deserialize Smithy BigInteger and BigDecimal to `bigint` and `@smithy/core/serde`'s `NumericValue`. Otherwise, use `"big.js"` to serialize and deserialize with that numeric library. |
197198
| `createDefaultReadme` | No | Whether to generate a default `README.md` for the package. The default value is `false`. |
198199
| `useLegacyAuth` | No | **NOT RECOMMENDED, AVAILABLE ONLY FOR BACKWARD COMPATIBILITY CONCERNS.** Flag that enables using legacy auth. When in doubt, use the default identity and auth behavior (not configuring `useLegacyAuth`) as the golden path. |
199200
| `serviceProtocolPriority` | No | Map of service `ShapeId` strings to lists of protocol `ShapeId` strings. Used to override protocol selection behavior. |
@@ -451,7 +452,7 @@ For documentation of `typescript-server-codegen` artifacts and implementation, s
451452
| `private` | No | Whether the package is `private` in `package.json`. The default value is `false`. |
452453
| `requiredMemberMode` | No | **NOT RECOMMENDED DUE TO BACKWARD COMPATIBILITY CONCERNS.** Sets whether members marked with the `@required` trait are allowed to be `undefined`. See more details on the risks in `TypeScriptSettings.RequiredMemberMode`. The default value is `nullable`. |
453454
| `createDefaultReadme` | No | Whether to generate a default `README.md` for the package. The default value is `false`. |
454-
| `disableDefaultValidation` | No | Whether or not default validation is disabled. See [the documentation for Smithy TypeScript SSDK validation](https://smithy.io/2.0/languages/typescript/ts-ssdk/validation.html) to learn more. The default value is `false`. |
455+
| `disableDefaultValidation` | No | Whether or not default validation is disabled. See [the documentation for Smithy TypeScript SSDK validation](https://smithy.io/2.0/languages/typescript/ts-ssdk/validation.html) to learn more. The default value is `false`. |
455456

456457
### Adding customizations to Smithy TypeScript
457458

‎packages/core/src/submodules/cbor/byte-printer.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
/**
2+
* Prints bytes as binary string with numbers.
3+
* @param bytes - to print.
4+
* @deprecated for testing only, do not use in runtime.
5+
*/
6+
export function printBytes(bytes: Uint8Array) {
7+
return [...bytes].map((n) => ("0".repeat(8) + n.toString(2)).slice(-8) + ` (${n})`);
8+
}

‎packages/core/src/submodules/cbor/cbor-decode.ts‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { NumericValue } from "@smithy/core/serde";
12
import { toUtf8 } from "@smithy/util-utf8";
23

34
import {
@@ -119,11 +120,33 @@ export function decode(at: Uint32, to: Uint32): CborValueType {
119120
_offset = offset;
120121
return castBigInt(negativeInt);
121122
} else {
122-
const value = decode(at + offset, to);
123-
const valueOffset = _offset;
123+
/* major === majorTag */
124+
if (minor === 2 || minor === 3) {
125+
const length = decodeCount(at + offset, to);
126+
127+
let b = BigInt(0);
128+
const start = at + offset + _offset;
129+
for (let i = start; i < start + length; ++i) {
130+
b = (b << BigInt(8)) | BigInt(payload[i]);
131+
}
132+
133+
_offset = offset + length;
134+
return minor === 3 ? -b - BigInt(1) : b;
135+
} else if (minor === 4) {
136+
const decimalFraction = decode(at + offset, to);
137+
const [exponent, mantissa] = decimalFraction;
138+
const s = mantissa.toString();
139+
const numericString = exponent === 0 ? s : s.slice(0, s.length + exponent) + "." + s.slice(exponent);
140+
141+
return new NumericValue(numericString, "bigDecimal");
142+
} else {
143+
const value = decode(at + offset, to);
144+
const valueOffset = _offset;
145+
146+
_offset = offset + valueOffset;
124147

125-
_offset = offset + valueOffset;
126-
return tag({ tag: castBigInt(unsignedInt), value });
148+
return tag({ tag: castBigInt(unsignedInt), value });
149+
}
127150
}
128151
case majorUtf8String:
129152
case majorMap:

‎packages/core/src/submodules/cbor/cbor-encode.ts‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
1+
import { NumericValue } from "@smithy/core/serde";
12
import { fromUtf8 } from "@smithy/util-utf8";
23

34
import {
5+
alloc,
46
CborMajorType,
57
extendedFloat16,
68
extendedFloat32,
@@ -19,7 +21,6 @@ import {
1921
tagSymbol,
2022
Uint64,
2123
} from "./cbor-types";
22-
import { alloc } from "./cbor-types";
2324

2425
const USE_BUFFER = typeof Buffer !== "undefined";
2526

@@ -152,10 +153,30 @@ export function encode(_input: any): void {
152153
data[cursor++] = (major << 5) | extendedFloat32;
153154
dataView.setUint32(cursor, n);
154155
cursor += 4;
155-
} else {
156+
} else if (value < BigInt("18446744073709551616")) {
156157
data[cursor++] = (major << 5) | extendedFloat64;
157158
dataView.setBigUint64(cursor, value);
158159
cursor += 8;
160+
} else {
161+
// refer to https://www.rfc-editor.org/rfc/rfc8949.html#name-bignums
162+
const binaryBigInt = value.toString(2);
163+
const bigIntBytes = new Uint8Array(Math.ceil(binaryBigInt.length / 8));
164+
let b = value;
165+
let i = 0;
166+
while (bigIntBytes.byteLength - ++i >= 0) {
167+
bigIntBytes[bigIntBytes.byteLength - i] = Number(b & BigInt(255));
168+
b >>= BigInt(8);
169+
}
170+
ensureSpace(bigIntBytes.byteLength * 2);
171+
data[cursor++] = nonNegative ? 0b110_00010 : 0b110_00011;
172+
173+
if (USE_BUFFER) {
174+
encodeHeader(majorUnstructuredByteString, Buffer.byteLength(bigIntBytes));
175+
} else {
176+
encodeHeader(majorUnstructuredByteString, bigIntBytes.byteLength);
177+
}
178+
data.set(bigIntBytes, cursor);
179+
cursor += bigIntBytes.byteLength;
159180
}
160181
continue;
161182
} else if (input === null) {
@@ -181,6 +202,18 @@ export function encode(_input: any): void {
181202
cursor += input.byteLength;
182203
continue;
183204
} else if (typeof input === "object") {
205+
if (input instanceof NumericValue) {
206+
const decimalIndex = input.string.indexOf(".");
207+
const exponent = decimalIndex === -1 ? 0 : decimalIndex - input.string.length + 1;
208+
const mantissa = BigInt(input.string.replace(".", ""));
209+
210+
data[cursor++] = 0b110_00100; // major 6, tag 4.
211+
212+
encodeStack.push(mantissa);
213+
encodeStack.push(exponent);
214+
encodeHeader(majorList, 2);
215+
continue;
216+
}
184217
if (input[tagSymbol]) {
185218
if ("tag" in input && "value" in input) {
186219
encodeStack.push(input.value);

‎packages/core/src/submodules/cbor/cbor.spec.ts‎

Lines changed: 81 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { NumericValue } from "@smithy/core/serde";
12
import * as fs from "fs";
23
// @ts-ignore
34
import JSONbig from "json-bigint";
@@ -88,12 +89,12 @@ describe("cbor", () => {
8889
{
8990
name: "negative float",
9091
data: -3015135.135135135,
91-
cbor: allocByteArray([0b111_11011, +193, +71, +0, +239, +145, +76, +27, +173]),
92+
cbor: allocByteArray([0b111_11011, 193, 71, 0, 239, 145, 76, 27, 173]),
9293
},
9394
{
9495
name: "positive float",
9596
data: 3015135.135135135,
96-
cbor: allocByteArray([0b111_11011, +65, +71, +0, +239, +145, +76, +27, +173]),
97+
cbor: allocByteArray([0b111_11011, 65, 71, 0, 239, 145, 76, 27, 173]),
9798
},
9899
{
99100
name: "various numbers",
@@ -214,6 +215,18 @@ describe("cbor", () => {
214215
65, 109, 110, 101, 115, 116, 101, 100, 32, 105, 116, 101, 109, 32, 66,
215216
]),
216217
},
218+
{
219+
name: "object containing big numbers",
220+
data: {
221+
map: {
222+
items: [BigInt(1e80)],
223+
},
224+
},
225+
cbor: allocByteArray([
226+
161, 99, 109, 97, 112, 161, 101, 105, 116, 101, 109, 115, 129, 194, 88, 34, 3, 95, 157, 234, 62, 31, 107, 224,
227+
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
228+
]),
229+
},
217230
];
218231

219232
const toBytes = (hex: string) => {
@@ -226,6 +239,72 @@ describe("cbor", () => {
226239
};
227240

228241
describe("locally curated scenarios", () => {
242+
it("should round-trip bigInteger to major 6 with tag 2", () => {
243+
const bigInt = BigInt("1267650600228229401496703205376");
244+
const serialized = cbor.serialize(bigInt);
245+
246+
const major = serialized[0] >> 5;
247+
expect(major).toEqual(0b110); // 6
248+
249+
const tag = serialized[0] & 0b11111;
250+
expect(tag).toEqual(0b010); // 2
251+
252+
const byteStringCount = serialized[1];
253+
expect(byteStringCount).toEqual(0b010_01101); // major 2, 13 bytes
254+
255+
const byteString = serialized.slice(2);
256+
expect(byteString).toEqual(allocByteArray([0b000_10000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]));
257+
258+
const deserialized = cbor.deserialize(serialized);
259+
expect(deserialized).toEqual(bigInt);
260+
});
261+
262+
it("should round-trip negative bigInteger to major 6 with tag 3", () => {
263+
const bigInt = BigInt("-1267650600228229401496703205377");
264+
const serialized = cbor.serialize(bigInt);
265+
266+
const major = serialized[0] >> 5;
267+
expect(major).toEqual(0b110); // 6
268+
269+
const tag = serialized[0] & 0b11111;
270+
expect(tag).toEqual(0b011); // 3
271+
272+
const byteStringCount = serialized[1];
273+
expect(byteStringCount).toEqual(0b010_01101); // major 2, 13 bytes
274+
275+
const byteString = serialized.slice(2);
276+
expect(byteString).toEqual(allocByteArray([0b000_10000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]));
277+
278+
const deserialized = cbor.deserialize(serialized);
279+
expect(deserialized).toEqual(bigInt);
280+
});
281+
282+
it("should round-trip NumericValue to major 6 with tag 4", () => {
283+
for (const bigDecimal of [
284+
"10000000000000000000000054.321",
285+
"1000000000000000000000000000000000054.134134321",
286+
"100000000000000000000000000000000000054.0000000000000001",
287+
"100000000000000000000000000000000000054.00510351095130000",
288+
"-10000000000000000000000054.321",
289+
"-1000000000000000000000000000000000054.134134321",
290+
"-100000000000000000000000000000000000054.0000000000000001",
291+
"-100000000000000000000000000000000000054.00510351095130000",
292+
]) {
293+
const nv = new NumericValue(bigDecimal, "bigDecimal");
294+
const serialized = cbor.serialize(nv);
295+
296+
const major = serialized[0] >> 5;
297+
expect(major).toEqual(0b110); // 6
298+
299+
const tag = serialized[0] & 0b11111;
300+
expect(tag).toEqual(0b0100); // 4
301+
302+
const deserialized = cbor.deserialize(serialized);
303+
expect(deserialized).toEqual(nv);
304+
expect(deserialized.string).toEqual(nv.string);
305+
}
306+
});
307+
229308
it("should throw an error if serializing a tag with missing properties", () => {
230309
expect(() =>
231310
cbor.serialize({

‎packages/core/src/submodules/serde/parse-utils.spec.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ import {
2222
} from "./parse-utils";
2323
import { expectBoolean, expectNumber, expectString } from "./parse-utils";
2424

25+
logger.warn = () => {};
26+
2527
describe("parseBoolean", () => {
2628
it('Returns true for "true"', () => {
2729
expect(parseBoolean("true")).toEqual(true);

‎packages/core/src/submodules/serde/value/NumericValue.spec.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,11 @@ describe(NumericValue.name, () => {
99
expect(num.string).toEqual("1.0");
1010
expect(num.type).toEqual("bigDecimal");
1111
});
12+
13+
it("allows only numeric digits and at most one decimal point", () => {
14+
expect(() => nv("a")).toThrow();
15+
expect(() => nv("1.0.1")).toThrow();
16+
expect(() => nv("-10.1")).not.toThrow();
17+
expect(() => nv("-.101")).not.toThrow();
18+
});
1219
});

‎packages/core/src/submodules/serde/value/NumericValue.ts‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,51 @@ export class NumericValue {
2424
public constructor(
2525
public readonly string: string,
2626
public readonly type: NumericType
27-
) {}
27+
) {
28+
let dot = 0;
29+
for (let i = 0; i < string.length; ++i) {
30+
const char = string.charCodeAt(i);
31+
if (i === 0 && char === 45) {
32+
// negation prefix "-"
33+
continue;
34+
}
35+
if (char === 46) {
36+
// decimal point "."
37+
if (dot) {
38+
throw new Error("@smithy/core/serde - NumericValue must contain at most one decimal point.");
39+
}
40+
dot = 1;
41+
continue;
42+
}
43+
if (char < 48 || char > 57) {
44+
// not in 0 through 9
45+
throw new Error(
46+
`@smithy/core/serde - NumericValue must only contain [0-9], at most one decimal point ".", and an optional negation prefix "-".`
47+
);
48+
}
49+
}
50+
}
51+
52+
public toString() {
53+
return this.string;
54+
}
55+
56+
public [Symbol.hasInstance](object: unknown) {
57+
if (!object || typeof object !== "object") {
58+
return false;
59+
}
60+
const _nv = object as NumericValue;
61+
if (typeof _nv.string === "string" && typeof _nv.type === "string" && _nv.constructor?.name === "NumericValue") {
62+
return true;
63+
}
64+
return false;
65+
}
2866
}
2967

3068
/**
3169
* Serde shortcut.
3270
* @internal
3371
*/
34-
export function nv(string: string): NumericValue {
35-
return new NumericValue(string, "bigDecimal");
72+
export function nv(input: string | unknown): NumericValue {
73+
return new NumericValue(String(input), "bigDecimal");
3674
}

0 commit comments

Comments
 (0)