fix: byteLengthSource

Author: kuhe

lib/lib-storage/src/Upload.spec.ts

@@ -969,7 +969,7 @@ describe(Upload.name, () => {
           partSize: 1024, // Too small
           client: new S3({}),
         });
-      }).toThrow(/EntityTooSmall: Your proposed upload partsize/);
+      }).toThrow(/EntityTooSmall: Your proposed upload part size/);
     });
 
     it("should validate minimum queueSize", () => {

lib/lib-storage/src/Upload.ts

@@ -27,7 +27,8 @@ import { extendedEncodeURIComponent } from "@smithy/smithy-client";
 import type { AbortController as IAbortController, AbortSignal as IAbortSignal, Endpoint } from "@smithy/types";
 import { EventEmitter } from "events";
 
-import { byteLength } from "./bytelength";
+import { byteLength } from "./byteLength";
+import { BYTE_LENGTH_SOURCE, byteLengthSource } from "./byteLengthSource";
 import { getChunk } from "./chunker";
 import { BodyDataTypes, Options, Progress } from "./types";
 
@@ -60,6 +61,7 @@ export class Upload extends EventEmitter {
 
   // used for reporting progress.
   private totalBytes?: number;
+  private readonly totalBytesSource?: BYTE_LENGTH_SOURCE;
   private bytesUploadedSoFar: number;
 
   // used in the upload.
@@ -98,6 +100,7 @@ export class Upload extends EventEmitter {
 
     // set progress defaults
     this.totalBytes = this.params.ContentLength ?? byteLength(this.params.Body);
+    this.totalBytesSource = byteLengthSource(this.params.Body, this.params.ContentLength);
     this.bytesUploadedSoFar = 0;
     this.abortController = options.abortController ?? new AbortController();
 
@@ -381,9 +384,14 @@ export class Upload extends EventEmitter {
 
     let result;
     if (this.isMultiPart) {
-      const { expectedPartsCount, uploadedParts } = this;
-      if (expectedPartsCount !== undefined && uploadedParts.length !== expectedPartsCount) {
-        throw new Error(`Expected ${expectedPartsCount} part(s) but uploaded ${uploadedParts.length} part(s).`);
+      const { expectedPartsCount, uploadedParts, totalBytes, totalBytesSource } = this;
+      if (totalBytes !== undefined && expectedPartsCount !== undefined && uploadedParts.length !== expectedPartsCount) {
+        throw new Error(`Expected ${expectedPartsCount} part(s) but uploaded ${uploadedParts.length} part(s).
+The expected part count is based on the byte-count of the input.params.Body,
+which was read from ${totalBytesSource} and is ${totalBytes}.
+If this is not correct, provide an override value by setting a number
+to input.params.ContentLength in bytes.
+`);
       }
 
       this.uploadedParts.sort((a, b) => a.PartNumber! - b.PartNumber!);
@@ -478,7 +486,7 @@ export class Upload extends EventEmitter {
 
     if (this.partSize < Upload.MIN_PART_SIZE) {
       throw new Error(
-        `EntityTooSmall: Your proposed upload partsize [${this.partSize}] is smaller than the minimum allowed size [${Upload.MIN_PART_SIZE}] (5MB)`
+        `EntityTooSmall: Your proposed upload part size [${this.partSize}] is smaller than the minimum allowed size [${Upload.MIN_PART_SIZE}] (5MB)`
       );
     }
 

lib/lib-storage/src/byteLength.ts

@@ -0,0 +1,41 @@
+import { Buffer } from "buffer"; // do not remove this import: Node.js buffer or buffer NPM module for browser.
+
+import { runtimeConfig } from "./runtimeConfig";
+
+/**
+ * Clients use util-body-length-[node|browser] instead.
+ * @internal
+ * @param input - to examine.
+ * @returns byte count of input or undefined if indeterminable.
+ */
+export const byteLength = (input: any): number | undefined => {
+  if (input == null) {
+    return 0;
+  }
+
+  if (typeof input === "string") {
+    return Buffer.byteLength(input);
+  }
+
+  if (typeof input.byteLength === "number") {
+    // Uint8Array, ArrayBuffer, Buffer, and ArrayBufferView
+    return input.byteLength;
+  } else if (typeof input.length === "number") {
+    // todo: unclear in what cases this is a valid byte count.
+    return input.length;
+  } else if (typeof input.size === "number") {
+    // todo: unclear in what cases this is a valid byte count.
+    return input.size;
+  } else if (typeof input.start === "number" && typeof input.end === "number") {
+    // file read stream with range.
+    return input.end + 1 - input.start;
+  } else if (typeof input.path === "string") {
+    // file read stream with path.
+    try {
+      return runtimeConfig.lstatSync(input.path).size;
+    } catch (error) {
+      return undefined;
+    }
+  }
+  return undefined;
+};

lib/lib-storage/src/byteLengthSource.ts

@@ -0,0 +1,54 @@
+import { runtimeConfig } from "./runtimeConfig";
+
+/**
+ * @internal
+ */
+export enum BYTE_LENGTH_SOURCE {
+  EMPTY_INPUT = "a null or undefined Body",
+  CONTENT_LENGTH = "the ContentLength property of the params set by the caller",
+  STRING_LENGTH = "the encoded byte length of the Body string",
+  TYPED_ARRAY = "the byteLength of a typed byte array such as Uint8Array",
+  LENGTH = "the value of Body.length",
+  SIZE = "the value of Body.size",
+  START_END_DIFF = "the numeric difference between Body.start and Body.end",
+  LSTAT = "the size of the file given by Body.path on disk as reported by lstatSync",
+}
+
+/**
+ * The returned value should complete the sentence, "The byte count of the data was determined by ...".
+ * @internal
+ * @param input - to examine.
+ * @param override - manually specified value.
+ * @returns source of byte count information.
+ */
+export const byteLengthSource = (input: any, override?: number): BYTE_LENGTH_SOURCE | undefined => {
+  if (override != null) {
+    return BYTE_LENGTH_SOURCE.CONTENT_LENGTH;
+  }
+
+  if (input == null) {
+    return BYTE_LENGTH_SOURCE.EMPTY_INPUT;
+  }
+
+  if (typeof input === "string") {
+    return BYTE_LENGTH_SOURCE.STRING_LENGTH;
+  }
+
+  if (typeof input.byteLength === "number") {
+    return BYTE_LENGTH_SOURCE.TYPED_ARRAY;
+  } else if (typeof input.length === "number") {
+    return BYTE_LENGTH_SOURCE.LENGTH;
+  } else if (typeof input.size === "number") {
+    return BYTE_LENGTH_SOURCE.SIZE;
+  } else if (typeof input.start === "number" && typeof input.end === "number") {
+    return BYTE_LENGTH_SOURCE.START_END_DIFF;
+  } else if (typeof input.path === "string") {
+    try {
+      runtimeConfig.lstatSync(input.path).size;
+      return BYTE_LENGTH_SOURCE.LSTAT;
+    } catch (error) {
+      return undefined;
+    }
+  }
+  return undefined;
+};

lib/lib-storage/src/bytelength.ts

@@ -2,6 +2,7 @@ import { getE2eTestResources } from "@aws-sdk/aws-util-test/src";
 import { ChecksumAlgorithm, S3 } from "@aws-sdk/client-s3";
 import { Upload } from "@aws-sdk/lib-storage";
 import { randomBytes } from "crypto";
+import fs from "node:fs";
 import { Readable } from "stream";
 import { afterAll, beforeAll, describe, expect, test as it } from "vitest";
 
@@ -179,4 +180,59 @@ describe("@aws-sdk/lib-storage", () => {
       });
     }
   );
+
+  describe("inferring the byte length of the input", () => {
+    beforeAll(async () => {
+      const e2eTestResourcesEnv = await getE2eTestResources();
+      Object.assign(process.env, e2eTestResourcesEnv);
+    });
+
+    it("should throw an informative error about the correct override if the SDK infers the byte count incorrectly", async () => {
+      const s3 = new S3({
+        region: process.env.AWS_SMOKE_TEST_REGION,
+      });
+
+      const pseudoFileReadStream = fs.createReadStream("/dev/urandom", { end: 6 * 1024 * 1024 });
+
+      const upload = new Upload({
+        client: s3,
+        params: {
+          Key: `/dev/urandom`,
+          Bucket: process.env.AWS_SMOKE_TEST_BUCKET,
+          Body: pseudoFileReadStream,
+        },
+      });
+
+      const error = await upload.done().catch((e) => e);
+
+      expect(error).toBeInstanceOf(Error);
+      expect(error.message).toEqual(`Expected 0 part(s) but uploaded 2 part(s).
+The expected part count is based on the byte-count of the input.params.Body,
+which was read from the size of the file given by Body.path on disk as reported by lstatSync and is 0.
+If this is not correct, provide an override value by setting a number
+to input.params.ContentLength in bytes.
+`);
+    });
+
+    it("should use the input ContentLength as the total byte count if supplied by the caller", async () => {
+      const s3 = new S3({
+        region: process.env.AWS_SMOKE_TEST_REGION,
+      });
+
+      const pseudoFileReadStream = fs.createReadStream("/dev/urandom", { end: 6 * 1024 * 1024 });
+
+      const upload = new Upload({
+        client: s3,
+        params: {
+          Key: `/dev/urandom`,
+          Bucket: process.env.AWS_SMOKE_TEST_BUCKET,
+          Body: pseudoFileReadStream,
+          ContentLength: 6 * 1024 * 1024,
+        },
+      });
+
+      await upload.done();
+      // no thrown error is sufficient.
+    });
+  });
 }, 60_000);

lib/lib-storage/src/lib-storage.e2e.spec.ts

@@ -1,9 +1,9 @@
-import { ClientSharedValues } from "./runtimeConfig.shared";
+import { runtimeConfigShared as shared } from "./runtimeConfig.shared";
 
 /**
  * @internal
  */
-export const ClientDefaultValues = {
-  ...ClientSharedValues,
+export const runtimeConfig = {
+  ...shared,
   runtime: "browser",
 };

lib/lib-storage/src/runtimeConfig.browser.ts

@@ -1,9 +1,9 @@
-import { ClientDefaultValues as BrowserDefaults } from "./runtimeConfig.browser";
+import { runtimeConfig as browserConfig } from "./runtimeConfig.browser";
 
 /**
  * @internal
  */
-export const ClientDefaultValues = {
-  ...BrowserDefaults,
+export const runtimeConfig = {
+  ...browserConfig,
   runtime: "react-native",
 };

lib/lib-storage/src/runtimeConfig.native.ts

@@ -1,6 +1,6 @@
 /**
  * @internal
  */
-export const ClientSharedValues = {
+export const runtimeConfigShared = {
   lstatSync: () => {},
 };

lib/lib-storage/src/runtimeConfig.shared.ts

@@ -1,12 +1,12 @@
 import { lstatSync } from "fs";
 
-import { ClientSharedValues } from "./runtimeConfig.shared";
+import { runtimeConfigShared as shared } from "./runtimeConfig.shared";
 
 /**
  * @internal
  */
-export const ClientDefaultValues = {
-  ...ClientSharedValues,
+export const runtimeConfig = {
+  ...shared,
   runtime: "node",
   lstatSync,
 };