Skip to content

Commit fd00602

Browse files
kuhekuhe
authored
chore(core/cbor): idempotencyToken handling for CborCodec (#1660)
1 parent 64e033f commit fd00602

File tree

12 files changed

+217
-82
lines changed

12 files changed

+217
-82
lines changed

‎.changeset/gentle-dogs-protect.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@smithy/middleware-retry": patch
3+
---
4+
5+
update uuid types version

‎.changeset/quiet-yaks-bathe.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@smithy/core": minor
3+
---
4+
5+
handle idempotency token generation for CBOR protocol

‎packages/core/package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,9 @@
7878
"@smithy/util-middleware": "workspace:^",
7979
"@smithy/util-stream": "workspace:^",
8080
"@smithy/util-utf8": "workspace:^",
81-
"tslib": "^2.6.2"
81+
"@types/uuid": "^9.0.1",
82+
"tslib": "^2.6.2",
83+
"uuid": "^9.0.1"
8284
},
8385
"engines": {
8486
"node": ">=18.0.0"

‎packages/core/src/submodules/cbor/CborCodec.spec.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
import { NormalizedSchema, sim, struct } from "@smithy/core/schema";
2+
import { describe, expect, it } from "vitest";
3+
4+
import { cbor } from "./cbor";
5+
import { CborCodec, CborShapeSerializer } from "./CborCodec";
6+
7+
describe(CborShapeSerializer.name, () => {
8+
const codec = new CborCodec();
9+
10+
const UUID_V4 = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
11+
12+
const idempotencyTokenSchemas = [
13+
NormalizedSchema.of(sim("", "StringWithTraits", 0, 0b0100)),
14+
NormalizedSchema.of(sim("", "StringWithTraits", 0, { idempotencyToken: 1 })),
15+
];
16+
17+
const plainSchemas = [
18+
NormalizedSchema.of(0),
19+
NormalizedSchema.of(sim("", "StringWithTraits", 0, 0)),
20+
NormalizedSchema.of(sim("", "StringWithTraits", 0, {})),
21+
];
22+
23+
const serializer = codec.createSerializer();
24+
25+
describe("serialization", () => {
26+
it("should generate an idempotency token when the input for such a member is undefined", () => {
27+
for (const idempotencyTokenSchema of idempotencyTokenSchemas) {
28+
for (const plainSchema of plainSchemas) {
29+
const objectSchema = struct(
30+
"ns",
31+
"StructWithIdempotencyToken",
32+
0,
33+
["idempotencyToken", "plainString"],
34+
[idempotencyTokenSchema, plainSchema]
35+
);
36+
37+
serializer.write(objectSchema, {
38+
idempotencyToken: undefined,
39+
plainString: undefined,
40+
});
41+
expect(cbor.deserialize(serializer.flush())).toMatchObject({
42+
idempotencyToken: UUID_V4,
43+
});
44+
45+
serializer.write(objectSchema, {
46+
idempotencyToken: undefined,
47+
plainString: "abc",
48+
});
49+
expect(cbor.deserialize(serializer.flush())).toMatchObject({
50+
idempotencyToken: UUID_V4,
51+
plainString: /^abc$/,
52+
});
53+
}
54+
}
55+
});
56+
});
57+
});

‎packages/core/src/submodules/cbor/CborCodec.ts‎

Lines changed: 76 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { NormalizedSchema } from "@smithy/core/schema";
2-
import { parseEpochTimestamp } from "@smithy/core/serde";
2+
import { generateIdempotencyToken, parseEpochTimestamp } from "@smithy/core/serde";
33
import { Codec, Schema, SerdeFunctions, ShapeDeserializer, ShapeSerializer } from "@smithy/types";
4+
import { fromBase64 } from "@smithy/util-base64";
45

56
import { cbor } from "./cbor";
67
import { dateToTag } from "./parseCborBody";
@@ -50,70 +51,70 @@ export class CborShapeSerializer implements ShapeSerializer {
5051
public serialize(schema: Schema, source: unknown): any {
5152
const ns = NormalizedSchema.of(schema);
5253

53-
switch (typeof source) {
54-
case "undefined":
55-
return null;
56-
case "boolean":
57-
case "number":
58-
case "string":
59-
case "bigint":
60-
case "symbol":
61-
return source;
62-
case "function":
63-
case "object":
64-
if (source === null) {
65-
return null;
66-
}
54+
if (source == null) {
55+
if (ns.isIdempotencyToken()) {
56+
return generateIdempotencyToken();
57+
}
58+
return source as null | undefined;
59+
}
6760

68-
const sourceObject = source as Record<string, unknown>;
69-
const sparse = !!ns.getMergedTraits().sparse;
61+
if (ns.isBlobSchema()) {
62+
if (typeof source === "string") {
63+
return (this.serdeContext?.base64Decoder ?? fromBase64)(source);
64+
}
65+
return source as Uint8Array;
66+
}
67+
68+
if (ns.isTimestampSchema()) {
69+
if (typeof source === "number" || typeof source === "bigint") {
70+
return dateToTag(new Date((Number(source) / 1000) | 0));
71+
}
72+
return dateToTag(source as Date);
73+
}
7074

71-
if (ns.isListSchema() && Array.isArray(sourceObject)) {
72-
const newArray = [];
73-
let i = 0;
74-
for (const item of sourceObject) {
75-
const value = this.serialize(ns.getValueSchema(), item);
76-
if (value != null || sparse) {
77-
newArray[i++] = value;
78-
}
75+
if (typeof source === "function" || typeof source === "object") {
76+
const sourceObject = source as Record<string, unknown>;
77+
78+
if (ns.isListSchema() && Array.isArray(sourceObject)) {
79+
const sparse = !!ns.getMergedTraits().sparse;
80+
const newArray = [];
81+
let i = 0;
82+
for (const item of sourceObject) {
83+
const value = this.serialize(ns.getValueSchema(), item);
84+
if (value != null || sparse) {
85+
newArray[i++] = value;
7986
}
80-
return newArray;
81-
}
82-
if (sourceObject instanceof Uint8Array) {
83-
const newBytes = new Uint8Array(sourceObject.byteLength);
84-
newBytes.set(sourceObject, 0);
85-
return newBytes;
86-
}
87-
if (sourceObject instanceof Date) {
88-
return dateToTag(sourceObject);
8987
}
90-
const newObject = {} as any;
91-
if (ns.isMapSchema()) {
92-
for (const key of Object.keys(sourceObject)) {
93-
const value = this.serialize(ns.getValueSchema(), sourceObject[key]);
94-
if (value != null || sparse) {
95-
newObject[key] = value;
96-
}
97-
}
98-
} else if (ns.isStructSchema()) {
99-
for (const [key, memberSchema] of ns.structIterator()) {
100-
const value = this.serialize(memberSchema, sourceObject[key]);
101-
if (value != null) {
102-
newObject[key] = value;
103-
}
88+
return newArray;
89+
}
90+
if (sourceObject instanceof Date) {
91+
return dateToTag(sourceObject);
92+
}
93+
const newObject = {} as any;
94+
if (ns.isMapSchema()) {
95+
const sparse = !!ns.getMergedTraits().sparse;
96+
for (const key of Object.keys(sourceObject)) {
97+
const value = this.serialize(ns.getValueSchema(), sourceObject[key]);
98+
if (value != null || sparse) {
99+
newObject[key] = value;
104100
}
105-
} else if (ns.isDocumentSchema()) {
106-
for (const key of Object.keys(sourceObject)) {
107-
const value = this.serialize(ns.getValueSchema(), sourceObject[key]);
108-
if (value != null) {
109-
newObject[key] = value;
110-
}
101+
}
102+
} else if (ns.isStructSchema()) {
103+
for (const [key, memberSchema] of ns.structIterator()) {
104+
const value = this.serialize(memberSchema, sourceObject[key]);
105+
if (value != null) {
106+
newObject[key] = value;
111107
}
112108
}
113-
return newObject;
114-
default:
115-
return source;
109+
} else if (ns.isDocumentSchema()) {
110+
for (const key of Object.keys(sourceObject)) {
111+
newObject[key] = this.serialize(ns.getValueSchema(), sourceObject[key]);
112+
}
113+
}
114+
return newObject;
116115
}
116+
117+
return source;
117118
}
118119

119120
public flush(): Uint8Array {
@@ -140,16 +141,17 @@ export class CborShapeDeserializer implements ShapeDeserializer {
140141

141142
private readValue(_schema: Schema, value: any): any {
142143
const ns = NormalizedSchema.of(_schema);
143-
const schema = ns.getSchema();
144144

145-
if (typeof schema === "number") {
146-
if (ns.isTimestampSchema()) {
147-
// format is ignored.
148-
return parseEpochTimestamp(value);
149-
}
150-
if (ns.isBlobSchema()) {
151-
return value;
145+
if (ns.isTimestampSchema() && typeof value === "number") {
146+
// format is ignored.
147+
return parseEpochTimestamp(value);
148+
}
149+
150+
if (ns.isBlobSchema()) {
151+
if (typeof value === "string") {
152+
return (this.serdeContext?.base64Decoder ?? fromBase64)(value);
152153
}
154+
return value as Uint8Array | undefined;
153155
}
154156

155157
if (
@@ -176,14 +178,14 @@ export class CborShapeDeserializer implements ShapeDeserializer {
176178
}
177179

178180
if (ns.isListSchema()) {
179-
const newArray = [];
181+
const newArray = [] as any[];
180182
const memberSchema = ns.getValueSchema();
181-
const sparse = ns.isListSchema() && !!ns.getMergedTraits().sparse;
183+
const sparse = !!ns.getMergedTraits().sparse;
182184

183185
for (const item of value) {
184-
newArray.push(this.readValue(memberSchema, item));
185-
if (!sparse && newArray[newArray.length - 1] == null) {
186-
newArray.pop();
186+
const itemValue = this.readValue(memberSchema, item);
187+
if (itemValue != null || sparse) {
188+
newArray.push(itemValue);
187189
}
188190
}
189191
return newArray;
@@ -192,14 +194,13 @@ export class CborShapeDeserializer implements ShapeDeserializer {
192194
const newObject = {} as any;
193195

194196
if (ns.isMapSchema()) {
195-
const sparse = ns.getMergedTraits().sparse;
197+
const sparse = !!ns.getMergedTraits().sparse;
196198
const targetSchema = ns.getValueSchema();
197199

198200
for (const key of Object.keys(value)) {
199-
newObject[key] = this.readValue(targetSchema, value[key]);
200-
201-
if (newObject[key] == null && !sparse) {
202-
delete newObject[key];
201+
const itemValue = this.readValue(targetSchema, value[key]);
202+
if (itemValue != null || sparse) {
203+
newObject[key] = itemValue;
203204
}
204205
}
205206
} else if (ns.isStructSchema()) {

‎packages/core/src/submodules/schema/schemas/NormalizedSchema.spec.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,4 +214,29 @@ describe(NormalizedSchema.name, () => {
214214
});
215215
});
216216
});
217+
218+
describe("idempotency token detection", () => {
219+
const idempotencyTokenSchemas = [
220+
NormalizedSchema.of(sim("", "StringWithTraits", 0, 0b0100)),
221+
NormalizedSchema.of(sim("", "StringWithTraits", 0, { idempotencyToken: 1 })),
222+
];
223+
224+
const plainSchemas = [
225+
NormalizedSchema.of(0),
226+
NormalizedSchema.of(sim("", "StringWithTraits", 0, 0)),
227+
NormalizedSchema.of(sim("", "StringWithTraits", 0, {})),
228+
];
229+
230+
it("has a consistent shortcut method for idempotencyToken detection", () => {
231+
for (const schema of idempotencyTokenSchemas) {
232+
expect(schema.isIdempotencyToken()).toBe(true);
233+
expect(schema.getMergedTraits().idempotencyToken).toBe(1);
234+
}
235+
236+
for (const schema of plainSchemas) {
237+
expect(schema.isIdempotencyToken()).toBe(false);
238+
expect(schema.getMergedTraits().idempotencyToken).toBe(undefined);
239+
}
240+
});
241+
});
217242
});

‎packages/core/src/submodules/schema/schemas/NormalizedSchema.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,19 @@ export class NormalizedSchema implements INormalizedSchema {
273273
return this.getSchema() === SCHEMA.STREAMING_BLOB;
274274
}
275275

276+
/**
277+
* This is a shortcut to avoid calling `getMergedTraits().idempotencyToken` on every string.
278+
* @returns whether the schema has the idempotencyToken trait.
279+
*/
280+
public isIdempotencyToken(): boolean {
281+
if (typeof this.traits === "number") {
282+
return (this.traits & 0b0100) === 0b0100;
283+
} else if (typeof this.traits === "object") {
284+
return !!this.traits.idempotencyToken;
285+
}
286+
return false;
287+
}
288+
276289
/**
277290
* @returns own traits merged with member traits, where member traits of the same trait key take priority.
278291
* This method is cached.

‎packages/core/src/submodules/serde/generateIdempotencyToken.spec.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import { describe, expect, test as it } from "vitest";
2+
3+
import { generateIdempotencyToken } from "./generateIdempotencyToken";
4+
5+
describe("generateIdempotencyToken", () => {
6+
// This test is not meaningful when using uuid v4 as an external package, but
7+
// will become useful if replacing the uuid implementation in the future.
8+
const tokens = {} as Record<string, boolean>;
9+
10+
it("should repeatedly generate uuid v4 strings", () => {
11+
for (let i = 0; i < 1000; ++i) {
12+
const token = generateIdempotencyToken();
13+
tokens[token] = true;
14+
expect(generateIdempotencyToken()).toMatch(
15+
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i
16+
);
17+
}
18+
19+
expect(Object.keys(tokens)).toHaveLength(1000);
20+
});
21+
});

‎packages/core/src/submodules/serde/generateIdempotencyToken.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
import { v4 as generateIdempotencyToken } from "uuid";
2+
3+
export { generateIdempotencyToken };

‎packages/core/src/submodules/serde/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
export * from "./copyDocumentWithTransform";
22
export * from "./date-utils";
3+
export * from "./generateIdempotencyToken";
34
export * from "./lazy-json";
45
export * from "./parse-utils";
56
export * from "./quote-header";

0 commit comments

Comments
 (0)