From 9a82f60b01a6880ddd43c8a74f14778c941ca9b3 Mon Sep 17 00:00:00 2001 From: George Fu Date: Mon, 29 Sep 2025 12:30:25 -0400 Subject: [PATCH 1/5] fix(lib-storage): use input parameter object size information if available # Conflicts: # lib/lib-storage/src/Upload.ts --- lib/lib-storage/src/Upload.spec.ts | 79 +++++++++++++++++++++++------- lib/lib-storage/src/Upload.ts | 14 ++++-- 2 files changed, 73 insertions(+), 20 deletions(-) diff --git a/lib/lib-storage/src/Upload.spec.ts b/lib/lib-storage/src/Upload.spec.ts index 15c9b9a2f4ce..96e77e0f9369 100644 --- a/lib/lib-storage/src/Upload.spec.ts +++ b/lib/lib-storage/src/Upload.spec.ts @@ -1,11 +1,23 @@ +import { + CompleteMultipartUploadCommand, + CompleteMultipartUploadCommandOutput, + CreateMultipartUploadCommand, + PutObjectCommand, + PutObjectTaggingCommand, + S3, + S3Client, + UploadPartCommand, +} from "@aws-sdk/client-s3"; +import { AbortController } from "@smithy/abort-controller"; +import { EventEmitter, Readable } from "stream"; import { afterAll, afterEach, beforeEach, describe, expect, test as it, vi } from "vitest"; +import { Progress, Upload } from "./index"; + /* eslint-disable no-var */ var hostname = "s3.region.amazonaws.com"; var port: number | undefined; -import { EventEmitter, Readable } from "stream"; - vi.mock("@aws-sdk/client-s3", async () => { const sendMock = vi.fn().mockImplementation(async (x) => x); const endpointMock = vi.fn().mockImplementation(() => ({ @@ -61,24 +73,20 @@ vi.mock("@aws-sdk/client-s3", async () => { }; }); -import { - CompleteMultipartUploadCommand, - CompleteMultipartUploadCommandOutput, - CreateMultipartUploadCommand, - PutObjectCommand, - PutObjectTaggingCommand, - S3, - S3Client, - UploadPartCommand, -} from "@aws-sdk/client-s3"; -import { AbortController } from "@smithy/abort-controller"; - -import { Progress, Upload } from "./index"; - const DEFAULT_PART_SIZE = 1024 * 1024 * 5; +type Expose = { + totalBytes: number | undefined; +}; +type VisibleForTesting = Omit & Expose; + describe(Upload.name, () => { - const s3MockInstance = new S3Client(); + const s3MockInstance = new S3Client({ + credentials: { + accessKeyId: "UNIT", + secretAccessKey: "UNIT", + }, + }); beforeEach(() => { vi.clearAllMocks(); @@ -107,6 +115,43 @@ describe(Upload.name, () => { Body: "this-is-a-sample-payload", }; + it("uses the input parameters for object length if provided", async () => { + const falseFileStream = Object.assign(Readable.from("abcd"), { + path: "/dev/null", + }); + let upload = new Upload({ + params: { + Bucket: "", + Key: "", + Body: falseFileStream, + MpuObjectSize: 6 * 1024 * 1024, + }, + client: s3MockInstance, + }) as unknown as VisibleForTesting; + expect(upload.totalBytes).toEqual(6 * 1024 * 1024); + + upload = new Upload({ + params: { + Bucket: "", + Key: "", + Body: falseFileStream, + ContentLength: 6 * 1024 * 1024, + }, + client: s3MockInstance, + }) as unknown as VisibleForTesting; + expect(upload.totalBytes).toEqual(6 * 1024 * 1024); + + upload = new Upload({ + params: { + Bucket: "", + Key: "", + Body: falseFileStream, + }, + client: s3MockInstance, + }) as unknown as VisibleForTesting; + expect(upload.totalBytes).toEqual(0); + }); + it("correctly exposes the event emitter API", () => { const upload = new Upload({ params, diff --git a/lib/lib-storage/src/Upload.ts b/lib/lib-storage/src/Upload.ts index cbc847d15c33..d780038c3f12 100644 --- a/lib/lib-storage/src/Upload.ts +++ b/lib/lib-storage/src/Upload.ts @@ -3,8 +3,10 @@ import { ChecksumAlgorithm, CompletedPart, CompleteMultipartUploadCommand, + CompleteMultipartUploadCommandInput, CompleteMultipartUploadCommandOutput, CreateMultipartUploadCommand, + CreateMultipartUploadCommandInput, CreateMultipartUploadCommandOutput, PutObjectCommand, PutObjectCommandInput, @@ -12,6 +14,7 @@ import { S3Client, Tag, UploadPartCommand, + UploadPartCommandInput, } from "@aws-sdk/client-s3"; import { AbortController } from "@smithy/abort-controller"; import { @@ -52,7 +55,8 @@ export class Upload extends EventEmitter { private readonly tags: Tag[] = []; private readonly client: S3Client; - private readonly params: PutObjectCommandInput; + private readonly params: PutObjectCommandInput & + Partial; // used for reporting progress. private totalBytes?: number; @@ -93,13 +97,17 @@ export class Upload extends EventEmitter { } // set progress defaults - this.totalBytes = byteLength(this.params.Body); + this.totalBytes = this.params.MpuObjectSize ?? this.params.ContentLength ?? byteLength(this.params.Body); this.bytesUploadedSoFar = 0; this.abortController = options.abortController ?? new AbortController(); this.partSize = options.partSize || Math.max(Upload.MIN_PART_SIZE, Math.floor((this.totalBytes || 0) / this.MAX_PARTS)); - this.expectedPartsCount = this.totalBytes !== undefined ? Math.ceil(this.totalBytes / this.partSize) : undefined; + + if (this.totalBytes !== undefined) { + this.expectedPartsCount = Math.ceil(this.totalBytes / this.partSize); + } + this.__validateInput(); } From d45ea1308bb2fb196bb94341e9796aad26012f39 Mon Sep 17 00:00:00 2001 From: George Fu Date: Mon, 29 Sep 2025 13:09:01 -0400 Subject: [PATCH 2/5] fix(lib-storage): use only ContentLength --- lib/lib-storage/src/Upload.spec.ts | 20 +++----------------- lib/lib-storage/src/Upload.ts | 2 +- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/lib/lib-storage/src/Upload.spec.ts b/lib/lib-storage/src/Upload.spec.ts index 96e77e0f9369..85804f18873a 100644 --- a/lib/lib-storage/src/Upload.spec.ts +++ b/lib/lib-storage/src/Upload.spec.ts @@ -116,25 +116,11 @@ describe(Upload.name, () => { }; it("uses the input parameters for object length if provided", async () => { - const falseFileStream = Object.assign(Readable.from("abcd"), { - path: "/dev/null", - }); let upload = new Upload({ params: { Bucket: "", Key: "", - Body: falseFileStream, - MpuObjectSize: 6 * 1024 * 1024, - }, - client: s3MockInstance, - }) as unknown as VisibleForTesting; - expect(upload.totalBytes).toEqual(6 * 1024 * 1024); - - upload = new Upload({ - params: { - Bucket: "", - Key: "", - Body: falseFileStream, + Body: Buffer.from("a".repeat(256)), ContentLength: 6 * 1024 * 1024, }, client: s3MockInstance, @@ -145,11 +131,11 @@ describe(Upload.name, () => { params: { Bucket: "", Key: "", - Body: falseFileStream, + Body: Buffer.from("a".repeat(256)), }, client: s3MockInstance, }) as unknown as VisibleForTesting; - expect(upload.totalBytes).toEqual(0); + expect(upload.totalBytes).toEqual(256); }); it("correctly exposes the event emitter API", () => { diff --git a/lib/lib-storage/src/Upload.ts b/lib/lib-storage/src/Upload.ts index d780038c3f12..d6deb8c40c91 100644 --- a/lib/lib-storage/src/Upload.ts +++ b/lib/lib-storage/src/Upload.ts @@ -97,7 +97,7 @@ export class Upload extends EventEmitter { } // set progress defaults - this.totalBytes = this.params.MpuObjectSize ?? this.params.ContentLength ?? byteLength(this.params.Body); + this.totalBytes = this.params.ContentLength ?? byteLength(this.params.Body); this.bytesUploadedSoFar = 0; this.abortController = options.abortController ?? new AbortController(); From b16f3022c5d2939fc78039f2e0cf14fb1c683fcc Mon Sep 17 00:00:00 2001 From: George Fu Date: Mon, 29 Sep 2025 13:13:19 -0400 Subject: [PATCH 3/5] fix: import ordering --- lib/lib-storage/src/Upload.spec.ts | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/lib-storage/src/Upload.spec.ts b/lib/lib-storage/src/Upload.spec.ts index 85804f18873a..91c075c1f050 100644 --- a/lib/lib-storage/src/Upload.spec.ts +++ b/lib/lib-storage/src/Upload.spec.ts @@ -1,14 +1,3 @@ -import { - CompleteMultipartUploadCommand, - CompleteMultipartUploadCommandOutput, - CreateMultipartUploadCommand, - PutObjectCommand, - PutObjectTaggingCommand, - S3, - S3Client, - UploadPartCommand, -} from "@aws-sdk/client-s3"; -import { AbortController } from "@smithy/abort-controller"; import { EventEmitter, Readable } from "stream"; import { afterAll, afterEach, beforeEach, describe, expect, test as it, vi } from "vitest"; @@ -73,6 +62,18 @@ vi.mock("@aws-sdk/client-s3", async () => { }; }); +import { + CompleteMultipartUploadCommand, + CompleteMultipartUploadCommandOutput, + CreateMultipartUploadCommand, + PutObjectCommand, + PutObjectTaggingCommand, + S3, + S3Client, + UploadPartCommand, +} from "@aws-sdk/client-s3"; +import { AbortController } from "@smithy/abort-controller"; + const DEFAULT_PART_SIZE = 1024 * 1024 * 5; type Expose = { From a71e654fff04f22f1ef5dcb846025c9b35fa0470 Mon Sep 17 00:00:00 2001 From: George Fu Date: Mon, 29 Sep 2025 15:49:31 -0400 Subject: [PATCH 4/5] fix: byteLengthSource --- lib/lib-storage/src/Upload.spec.ts | 2 +- lib/lib-storage/src/Upload.ts | 18 +++++-- lib/lib-storage/src/byteLength.ts | 41 ++++++++++++++ lib/lib-storage/src/byteLengthSource.ts | 54 +++++++++++++++++++ lib/lib-storage/src/bytelength.ts | 26 --------- lib/lib-storage/src/lib-storage.e2e.spec.ts | 56 ++++++++++++++++++++ lib/lib-storage/src/runtimeConfig.browser.ts | 6 +-- lib/lib-storage/src/runtimeConfig.native.ts | 6 +-- lib/lib-storage/src/runtimeConfig.shared.ts | 2 +- lib/lib-storage/src/runtimeConfig.ts | 6 +-- 10 files changed, 175 insertions(+), 42 deletions(-) create mode 100644 lib/lib-storage/src/byteLength.ts create mode 100644 lib/lib-storage/src/byteLengthSource.ts delete mode 100644 lib/lib-storage/src/bytelength.ts diff --git a/lib/lib-storage/src/Upload.spec.ts b/lib/lib-storage/src/Upload.spec.ts index 91c075c1f050..7f524ce23843 100644 --- a/lib/lib-storage/src/Upload.spec.ts +++ b/lib/lib-storage/src/Upload.spec.ts @@ -969,7 +969,7 @@ describe(Upload.name, () => { partSize: 1024, // Too small client: new S3({}), }); - }).toThrow(/EntityTooSmall: Your proposed upload partsize/); + }).toThrow(/EntityTooSmall: Your proposed upload part size/); }); it("should validate minimum queueSize", () => { diff --git a/lib/lib-storage/src/Upload.ts b/lib/lib-storage/src/Upload.ts index d6deb8c40c91..0ff6b1468ab3 100644 --- a/lib/lib-storage/src/Upload.ts +++ b/lib/lib-storage/src/Upload.ts @@ -27,7 +27,8 @@ import { extendedEncodeURIComponent } from "@smithy/smithy-client"; import type { AbortController as IAbortController, AbortSignal as IAbortSignal, Endpoint } from "@smithy/types"; import { EventEmitter } from "events"; -import { byteLength } from "./bytelength"; +import { byteLength } from "./byteLength"; +import { BYTE_LENGTH_SOURCE, byteLengthSource } from "./byteLengthSource"; import { getChunk } from "./chunker"; import { BodyDataTypes, Options, Progress } from "./types"; @@ -60,6 +61,7 @@ export class Upload extends EventEmitter { // used for reporting progress. private totalBytes?: number; + private readonly totalBytesSource?: BYTE_LENGTH_SOURCE; private bytesUploadedSoFar: number; // used in the upload. @@ -98,6 +100,7 @@ export class Upload extends EventEmitter { // set progress defaults this.totalBytes = this.params.ContentLength ?? byteLength(this.params.Body); + this.totalBytesSource = byteLengthSource(this.params.Body, this.params.ContentLength); this.bytesUploadedSoFar = 0; this.abortController = options.abortController ?? new AbortController(); @@ -381,9 +384,14 @@ export class Upload extends EventEmitter { let result; if (this.isMultiPart) { - const { expectedPartsCount, uploadedParts } = this; - if (expectedPartsCount !== undefined && uploadedParts.length !== expectedPartsCount) { - throw new Error(`Expected ${expectedPartsCount} part(s) but uploaded ${uploadedParts.length} part(s).`); + const { expectedPartsCount, uploadedParts, totalBytes, totalBytesSource } = this; + if (totalBytes !== undefined && expectedPartsCount !== undefined && uploadedParts.length !== expectedPartsCount) { + throw new Error(`Expected ${expectedPartsCount} part(s) but uploaded ${uploadedParts.length} part(s). +The expected part count is based on the byte-count of the input.params.Body, +which was read from ${totalBytesSource} and is ${totalBytes}. +If this is not correct, provide an override value by setting a number +to input.params.ContentLength in bytes. +`); } this.uploadedParts.sort((a, b) => a.PartNumber! - b.PartNumber!); @@ -478,7 +486,7 @@ export class Upload extends EventEmitter { if (this.partSize < Upload.MIN_PART_SIZE) { throw new Error( - `EntityTooSmall: Your proposed upload partsize [${this.partSize}] is smaller than the minimum allowed size [${Upload.MIN_PART_SIZE}] (5MB)` + `EntityTooSmall: Your proposed upload part size [${this.partSize}] is smaller than the minimum allowed size [${Upload.MIN_PART_SIZE}] (5MB)` ); } diff --git a/lib/lib-storage/src/byteLength.ts b/lib/lib-storage/src/byteLength.ts new file mode 100644 index 000000000000..fdc860501a01 --- /dev/null +++ b/lib/lib-storage/src/byteLength.ts @@ -0,0 +1,41 @@ +import { Buffer } from "buffer"; // do not remove this import: Node.js buffer or buffer NPM module for browser. + +import { runtimeConfig } from "./runtimeConfig"; + +/** + * Clients use util-body-length-[node|browser] instead. + * @internal + * @param input - to examine. + * @returns byte count of input or undefined if indeterminable. + */ +export const byteLength = (input: any): number | undefined => { + if (input == null) { + return 0; + } + + if (typeof input === "string") { + return Buffer.byteLength(input); + } + + if (typeof input.byteLength === "number") { + // Uint8Array, ArrayBuffer, Buffer, and ArrayBufferView + return input.byteLength; + } else if (typeof input.length === "number") { + // todo: unclear in what cases this is a valid byte count. + return input.length; + } else if (typeof input.size === "number") { + // todo: unclear in what cases this is a valid byte count. + return input.size; + } else if (typeof input.start === "number" && typeof input.end === "number") { + // file read stream with range. + return input.end + 1 - input.start; + } else if (typeof input.path === "string") { + // file read stream with path. + try { + return runtimeConfig.lstatSync(input.path).size; + } catch (error) { + return undefined; + } + } + return undefined; +}; diff --git a/lib/lib-storage/src/byteLengthSource.ts b/lib/lib-storage/src/byteLengthSource.ts new file mode 100644 index 000000000000..5a47864039e6 --- /dev/null +++ b/lib/lib-storage/src/byteLengthSource.ts @@ -0,0 +1,54 @@ +import { runtimeConfig } from "./runtimeConfig"; + +/** + * @internal + */ +export enum BYTE_LENGTH_SOURCE { + EMPTY_INPUT = "a null or undefined Body", + CONTENT_LENGTH = "the ContentLength property of the params set by the caller", + STRING_LENGTH = "the encoded byte length of the Body string", + TYPED_ARRAY = "the byteLength of a typed byte array such as Uint8Array", + LENGTH = "the value of Body.length", + SIZE = "the value of Body.size", + START_END_DIFF = "the numeric difference between Body.start and Body.end", + LSTAT = "the size of the file given by Body.path on disk as reported by lstatSync", +} + +/** + * The returned value should complete the sentence, "The byte count of the data was determined by ...". + * @internal + * @param input - to examine. + * @param override - manually specified value. + * @returns source of byte count information. + */ +export const byteLengthSource = (input: any, override?: number): BYTE_LENGTH_SOURCE | undefined => { + if (override != null) { + return BYTE_LENGTH_SOURCE.CONTENT_LENGTH; + } + + if (input == null) { + return BYTE_LENGTH_SOURCE.EMPTY_INPUT; + } + + if (typeof input === "string") { + return BYTE_LENGTH_SOURCE.STRING_LENGTH; + } + + if (typeof input.byteLength === "number") { + return BYTE_LENGTH_SOURCE.TYPED_ARRAY; + } else if (typeof input.length === "number") { + return BYTE_LENGTH_SOURCE.LENGTH; + } else if (typeof input.size === "number") { + return BYTE_LENGTH_SOURCE.SIZE; + } else if (typeof input.start === "number" && typeof input.end === "number") { + return BYTE_LENGTH_SOURCE.START_END_DIFF; + } else if (typeof input.path === "string") { + try { + runtimeConfig.lstatSync(input.path).size; + return BYTE_LENGTH_SOURCE.LSTAT; + } catch (error) { + return undefined; + } + } + return undefined; +}; diff --git a/lib/lib-storage/src/bytelength.ts b/lib/lib-storage/src/bytelength.ts deleted file mode 100644 index caefb943d0a8..000000000000 --- a/lib/lib-storage/src/bytelength.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { Buffer } from "buffer"; // do not remove this import: Node.js buffer or buffer NPM module for browser. - -import { ClientDefaultValues } from "./runtimeConfig"; - -export const byteLength = (input: any) => { - if (input === null || input === undefined) return 0; - - if (typeof input === "string") { - return Buffer.byteLength(input); - } - - if (typeof input.byteLength === "number") { - return input.byteLength; - } else if (typeof input.length === "number") { - return input.length; - } else if (typeof input.size === "number") { - return input.size; - } else if (typeof input.path === "string") { - try { - return ClientDefaultValues.lstatSync(input.path).size; - } catch (error) { - return undefined; - } - } - return undefined; -}; diff --git a/lib/lib-storage/src/lib-storage.e2e.spec.ts b/lib/lib-storage/src/lib-storage.e2e.spec.ts index e615e615f49a..8329aad83ba4 100644 --- a/lib/lib-storage/src/lib-storage.e2e.spec.ts +++ b/lib/lib-storage/src/lib-storage.e2e.spec.ts @@ -2,6 +2,7 @@ import { getE2eTestResources } from "@aws-sdk/aws-util-test/src"; import { ChecksumAlgorithm, S3 } from "@aws-sdk/client-s3"; import { Upload } from "@aws-sdk/lib-storage"; import { randomBytes } from "crypto"; +import fs from "node:fs"; import { Readable } from "stream"; import { afterAll, beforeAll, describe, expect, test as it } from "vitest"; @@ -179,4 +180,59 @@ describe("@aws-sdk/lib-storage", () => { }); } ); + + describe("inferring the byte length of the input", () => { + beforeAll(async () => { + const e2eTestResourcesEnv = await getE2eTestResources(); + Object.assign(process.env, e2eTestResourcesEnv); + }); + + it("should throw an informative error about the correct override if the SDK infers the byte count incorrectly", async () => { + const s3 = new S3({ + region: process.env.AWS_SMOKE_TEST_REGION, + }); + + const pseudoFileReadStream = fs.createReadStream("/dev/urandom", { end: 6 * 1024 * 1024 }); + + const upload = new Upload({ + client: s3, + params: { + Key: `/dev/urandom`, + Bucket: process.env.AWS_SMOKE_TEST_BUCKET, + Body: pseudoFileReadStream, + }, + }); + + const error = await upload.done().catch((e) => e); + + expect(error).toBeInstanceOf(Error); + expect(error.message).toEqual(`Expected 0 part(s) but uploaded 2 part(s). +The expected part count is based on the byte-count of the input.params.Body, +which was read from the size of the file given by Body.path on disk as reported by lstatSync and is 0. +If this is not correct, provide an override value by setting a number +to input.params.ContentLength in bytes. +`); + }); + + it("should use the input ContentLength as the total byte count if supplied by the caller", async () => { + const s3 = new S3({ + region: process.env.AWS_SMOKE_TEST_REGION, + }); + + const pseudoFileReadStream = fs.createReadStream("/dev/urandom", { end: 6 * 1024 * 1024 }); + + const upload = new Upload({ + client: s3, + params: { + Key: `/dev/urandom`, + Bucket: process.env.AWS_SMOKE_TEST_BUCKET, + Body: pseudoFileReadStream, + ContentLength: 6 * 1024 * 1024, + }, + }); + + await upload.done(); + // no thrown error is sufficient. + }); + }); }, 60_000); diff --git a/lib/lib-storage/src/runtimeConfig.browser.ts b/lib/lib-storage/src/runtimeConfig.browser.ts index 34e08a46f008..a0798ce2d6ce 100644 --- a/lib/lib-storage/src/runtimeConfig.browser.ts +++ b/lib/lib-storage/src/runtimeConfig.browser.ts @@ -1,9 +1,9 @@ -import { ClientSharedValues } from "./runtimeConfig.shared"; +import { runtimeConfigShared as shared } from "./runtimeConfig.shared"; /** * @internal */ -export const ClientDefaultValues = { - ...ClientSharedValues, +export const runtimeConfig = { + ...shared, runtime: "browser", }; diff --git a/lib/lib-storage/src/runtimeConfig.native.ts b/lib/lib-storage/src/runtimeConfig.native.ts index 8026ee2fd70c..1c5968aa213a 100644 --- a/lib/lib-storage/src/runtimeConfig.native.ts +++ b/lib/lib-storage/src/runtimeConfig.native.ts @@ -1,9 +1,9 @@ -import { ClientDefaultValues as BrowserDefaults } from "./runtimeConfig.browser"; +import { runtimeConfig as browserConfig } from "./runtimeConfig.browser"; /** * @internal */ -export const ClientDefaultValues = { - ...BrowserDefaults, +export const runtimeConfig = { + ...browserConfig, runtime: "react-native", }; diff --git a/lib/lib-storage/src/runtimeConfig.shared.ts b/lib/lib-storage/src/runtimeConfig.shared.ts index 30bc8ddecaac..4cdcdcf9e11b 100644 --- a/lib/lib-storage/src/runtimeConfig.shared.ts +++ b/lib/lib-storage/src/runtimeConfig.shared.ts @@ -1,6 +1,6 @@ /** * @internal */ -export const ClientSharedValues = { +export const runtimeConfigShared = { lstatSync: () => {}, }; diff --git a/lib/lib-storage/src/runtimeConfig.ts b/lib/lib-storage/src/runtimeConfig.ts index ea4c0a6841f6..3b773d9daa7b 100644 --- a/lib/lib-storage/src/runtimeConfig.ts +++ b/lib/lib-storage/src/runtimeConfig.ts @@ -1,12 +1,12 @@ import { lstatSync } from "fs"; -import { ClientSharedValues } from "./runtimeConfig.shared"; +import { runtimeConfigShared as shared } from "./runtimeConfig.shared"; /** * @internal */ -export const ClientDefaultValues = { - ...ClientSharedValues, +export const runtimeConfig = { + ...shared, runtime: "node", lstatSync, }; From 437f05c5a358fca7bb5d83e29c85ece0e43ef52a Mon Sep 17 00:00:00 2001 From: George Fu Date: Mon, 29 Sep 2025 16:20:14 -0400 Subject: [PATCH 5/5] fix: add error message --- lib/lib-storage/src/byteLength.spec.ts | 70 +++++++++++++++++++ .../src/chunks/getChunkUint8Array.spec.ts | 8 ++- .../src/chunks/getDataReadable.spec.ts | 2 +- .../src/chunks/getDataReadableStream.spec.ts | 2 +- 4 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 lib/lib-storage/src/byteLength.spec.ts diff --git a/lib/lib-storage/src/byteLength.spec.ts b/lib/lib-storage/src/byteLength.spec.ts new file mode 100644 index 000000000000..a0d19647f0e5 --- /dev/null +++ b/lib/lib-storage/src/byteLength.spec.ts @@ -0,0 +1,70 @@ +import { describe, expect, it, vi } from "vitest"; + +import { BYTE_LENGTH_SOURCE, byteLengthSource } from "./byteLengthSource"; +import { runtimeConfig } from "./runtimeConfig"; + +vi.mock("./runtimeConfig", () => ({ + runtimeConfig: { + lstatSync: vi.fn(), + }, +})); + +describe("byteLengthSource", () => { + it("should return CONTENT_LENGTH when override is provided", () => { + expect(byteLengthSource({}, 100)).toBe(BYTE_LENGTH_SOURCE.CONTENT_LENGTH); + }); + + it("should return EMPTY_INPUT for null input", () => { + expect(byteLengthSource(null)).toBe(BYTE_LENGTH_SOURCE.EMPTY_INPUT); + }); + + it("should return EMPTY_INPUT for undefined input", () => { + expect(byteLengthSource(undefined)).toBe(BYTE_LENGTH_SOURCE.EMPTY_INPUT); + }); + + it("should return STRING_LENGTH for string input", () => { + expect(byteLengthSource("test")).toBe(BYTE_LENGTH_SOURCE.STRING_LENGTH); + }); + + it("should return TYPED_ARRAY for input with byteLength", () => { + const input = new Uint8Array(10); + expect(byteLengthSource(input)).toBe(BYTE_LENGTH_SOURCE.TYPED_ARRAY); + }); + + it("should return LENGTH for input with length property", () => { + const input = { length: 10 }; + expect(byteLengthSource(input)).toBe(BYTE_LENGTH_SOURCE.LENGTH); + }); + + it("should return SIZE for input with size property", () => { + const input = { size: 10 }; + expect(byteLengthSource(input)).toBe(BYTE_LENGTH_SOURCE.SIZE); + }); + + it("should return START_END_DIFF for input with start and end properties", () => { + const input = { start: 0, end: 10 }; + expect(byteLengthSource(input)).toBe(BYTE_LENGTH_SOURCE.START_END_DIFF); + }); + + it("should return LSTAT for input with path that exists", () => { + const input = { path: "/test/path" }; + vi.mocked(runtimeConfig.lstatSync).mockReturnValue({ size: 100 } as any); + + expect(byteLengthSource(input)).toBe(BYTE_LENGTH_SOURCE.LSTAT); + expect(runtimeConfig.lstatSync).toHaveBeenCalledWith("/test/path"); + }); + + it("should return undefined for input with path that throws error", () => { + const input = { path: "/test/path" }; + vi.mocked(runtimeConfig.lstatSync).mockImplementation(() => { + throw new Error("File not found"); + }); + + expect(byteLengthSource(input)).toBeUndefined(); + }); + + it("should return undefined for input with no matching properties", () => { + const input = { foo: "bar" }; + expect(byteLengthSource(input)).toBeUndefined(); + }); +}); diff --git a/lib/lib-storage/src/chunks/getChunkUint8Array.spec.ts b/lib/lib-storage/src/chunks/getChunkUint8Array.spec.ts index 664ee546de7c..81a716562421 100644 --- a/lib/lib-storage/src/chunks/getChunkUint8Array.spec.ts +++ b/lib/lib-storage/src/chunks/getChunkUint8Array.spec.ts @@ -1,6 +1,7 @@ import { describe, expect, test as it } from "vitest"; -import { byteLength } from "../bytelength"; +import { byteLength } from "../byteLength"; +import { BYTE_LENGTH_SOURCE, byteLengthSource } from "../byteLengthSource"; import { RawDataPart } from "../Upload"; import { getChunkUint8Array } from "./getChunkUint8Array"; @@ -21,6 +22,7 @@ describe(getChunkUint8Array.name, () => { for await (const chunk of chunker) { chunkNum += 1; expect(byteLength(chunk.data)).toEqual(chunklength); + expect(byteLengthSource(chunk.data)).toEqual(BYTE_LENGTH_SOURCE.TYPED_ARRAY); expect(chunk.partNumber).toEqual(chunkNum); if (chunkNum < expectedNumberOfChunks) { expect(chunk.lastPart).toBe(undefined); @@ -45,10 +47,13 @@ describe(getChunkUint8Array.name, () => { expect(chunks.length).toEqual(3); expect(byteLength(chunks[0].data)).toBe(chunklength); + expect(byteLengthSource(chunks[0].data)).toEqual(BYTE_LENGTH_SOURCE.TYPED_ARRAY); expect(chunks[0].lastPart).toBe(undefined); expect(byteLength(chunks[1].data)).toBe(chunklength); + expect(byteLengthSource(chunks[1].data)).toEqual(BYTE_LENGTH_SOURCE.TYPED_ARRAY); expect(chunks[1].lastPart).toBe(undefined); expect(byteLength(chunks[2].data)).toBe(totalLength % chunklength); + expect(byteLengthSource(chunks[2].data)).toEqual(BYTE_LENGTH_SOURCE.TYPED_ARRAY); expect(chunks[2].lastPart).toBe(true); }); @@ -65,6 +70,7 @@ describe(getChunkUint8Array.name, () => { expect(chunks.length).toEqual(1); expect(byteLength(chunks[0].data)).toBe(totalLength % chunklength); + expect(byteLengthSource(chunks[0].data)).toEqual(BYTE_LENGTH_SOURCE.TYPED_ARRAY); expect(chunks[0].lastPart).toBe(true); }); }); diff --git a/lib/lib-storage/src/chunks/getDataReadable.spec.ts b/lib/lib-storage/src/chunks/getDataReadable.spec.ts index 6c57baafc066..a3817b712742 100644 --- a/lib/lib-storage/src/chunks/getDataReadable.spec.ts +++ b/lib/lib-storage/src/chunks/getDataReadable.spec.ts @@ -1,7 +1,7 @@ import { Readable } from "stream"; import { describe, expect, test as it } from "vitest"; -import { byteLength } from "../bytelength"; +import { byteLength } from "../byteLength"; import { RawDataPart as DataPart } from "../Upload"; import { getChunkStream as chunkFromReadable } from "./getChunkStream"; import { getDataReadable } from "./getDataReadable"; diff --git a/lib/lib-storage/src/chunks/getDataReadableStream.spec.ts b/lib/lib-storage/src/chunks/getDataReadableStream.spec.ts index 6cd11135bbb8..d023bf1be6c1 100644 --- a/lib/lib-storage/src/chunks/getDataReadableStream.spec.ts +++ b/lib/lib-storage/src/chunks/getDataReadableStream.spec.ts @@ -2,7 +2,7 @@ import { describe, expect, test as it } from "vitest"; // polyfill exposes the same ReadableStream API as web, allowing easy testing import { ReadableStream } from "web-streams-polyfill"; -import { byteLength } from "../bytelength"; +import { byteLength } from "../byteLength"; import { RawDataPart as DataPart } from "../Upload"; import { getChunkStream as chunkFromReadable } from "./getChunkStream"; import { getDataReadableStream } from "./getDataReadableStream";