chore(lib-storage): support passing of ChecksumAlgorithm

Author: trivikr

lib/lib-storage/src/Upload.ts

@@ -204,7 +204,7 @@ export class Upload extends EventEmitter {
     if (!this.createMultiPartPromise) {
       const createCommandParams = { ...this.params, Body: undefined };
       if (requestChecksumCalculation === "WHEN_SUPPORTED") {
-        createCommandParams.ChecksumAlgorithm = ChecksumAlgorithm.CRC32;
+        createCommandParams.ChecksumAlgorithm = this.params.ChecksumAlgorithm || ChecksumAlgorithm.CRC32;
       }
       this.createMultiPartPromise = this.client
         .send(new CreateMultipartUploadCommand(createCommandParams))

lib/lib-storage/src/lib-storage.e2e.spec.ts

@@ -1,4 +1,4 @@
-import { S3 } from "@aws-sdk/client-s3";
+import { ChecksumAlgorithm, S3 } from "@aws-sdk/client-s3";
 import { Upload } from "@aws-sdk/lib-storage";
 import { randomBytes } from "crypto";
 import { Readable } from "stream";
@@ -10,128 +10,126 @@ describe("@aws-sdk/lib-storage", () => {
   describe.each([undefined, "WHEN_REQUIRED", "WHEN_SUPPORTED"])(
     "requestChecksumCalculation: %s",
     (requestChecksumCalculation) => {
-      let Key: string;
-      let client: S3;
-      let data: Uint8Array;
-      let dataString: string;
-      let Bucket: string;
-      let region: string;
-
-      beforeAll(async () => {
-        const integTestResourcesEnv = await getIntegTestResources();
-        Object.assign(process.env, integTestResourcesEnv);
-
-        region = process?.env?.AWS_SMOKE_TEST_REGION as string;
-        Bucket = process?.env?.AWS_SMOKE_TEST_BUCKET as string;
-
-        Key = ``;
-        data = randomBytes(20_240_000);
-        dataString = data.toString();
-
-        // @ts-expect-error: Types of property 'requestChecksumCalculation' are incompatible
-        client = new S3({
-          region,
-          requestChecksumCalculation,
+      describe.each([
+        undefined,
+        ChecksumAlgorithm.SHA1,
+        ChecksumAlgorithm.SHA256,
+        ChecksumAlgorithm.CRC32,
+        ChecksumAlgorithm.CRC32C,
+      ])("ChecksumAlgorithm: %s", (ChecksumAlgorithm) => {
+        let Key: string;
+        let client: S3;
+        let data: Uint8Array;
+        let dataString: string;
+        let Bucket: string;
+        let region: string;
+
+        beforeAll(async () => {
+          const integTestResourcesEnv = await getIntegTestResources();
+          Object.assign(process.env, integTestResourcesEnv);
+
+          region = process?.env?.AWS_SMOKE_TEST_REGION as string;
+          Bucket = process?.env?.AWS_SMOKE_TEST_BUCKET as string;
+
+          Key = ``;
+          data = randomBytes(20_240_000);
+          dataString = data.toString();
+
+          // @ts-expect-error: Types of property 'requestChecksumCalculation' are incompatible
+          client = new S3({
+            region,
+            requestChecksumCalculation,
+          });
+          Key = `multi-part-file-${requestChecksumCalculation}-${ChecksumAlgorithm}-${Date.now()}`;
         });
-        Key = `multi-part-file-${requestChecksumCalculation}-${Date.now()}`;
-      });
-
-      afterAll(async () => {
-        await client.deleteObject({ Bucket, Key });
-      });
 
-      it("should upload in parts for input type bytes", async () => {
-        const s3Upload = new Upload({
-          client,
-          params: { Bucket, Key, Body: data },
+        afterAll(async () => {
+          await client.deleteObject({ Bucket, Key });
         });
-        await s3Upload.done();
 
-        const object = await client.getObject({ Bucket, Key });
-        expect(await object.Body?.transformToString()).toEqual(dataString);
-      });
+        it("should upload in parts for input type bytes", async () => {
+          const s3Upload = new Upload({
+            client,
+            params: { Bucket, Key, Body: data, ChecksumAlgorithm },
+          });
+          await s3Upload.done();
 
-      it("should upload in parts for input type string", async () => {
-        const s3Upload = new Upload({
-          client,
-          params: { Bucket, Key, Body: dataString },
+          const object = await client.getObject({ Bucket, Key });
+          expect(await object.Body?.transformToString()).toEqual(dataString);
         });
-        await s3Upload.done();
 
-        const object = await client.getObject({ Bucket, Key });
-        expect(await object.Body?.transformToString()).toEqual(dataString);
-      });
+        it("should upload in parts for input type string", async () => {
+          const s3Upload = new Upload({
+            client,
+            params: { Bucket, Key, Body: dataString, ChecksumAlgorithm },
+          });
+          await s3Upload.done();
 
-      it("should upload in parts for input type Readable", async () => {
-        const s3Upload = new Upload({
-          client,
-          params: { Bucket, Key, Body: Readable.from(data) },
+          const object = await client.getObject({ Bucket, Key });
+          expect(await object.Body?.transformToString()).toEqual(dataString);
         });
-        await s3Upload.done();
-
-        const object = await client.getObject({ Bucket, Key });
-        expect(await object.Body?.transformToString()).toEqual(dataString);
-      });
 
-      it("should call AbortMultipartUpload if unable to complete a multipart upload.", async () => {
-        class MockFailureS3 extends S3 {
-          public counter = 0;
-          async send(command: any, ...rest: any[]) {
-            if (command?.constructor?.name === "UploadPartCommand" && this.counter++ % 3 === 0) {
-              throw new Error("simulated upload part error");
-            }
-            return super.send(command, ...rest);
-          }
-        }
+        it("should upload in parts for input type Readable", async () => {
+          const s3Upload = new Upload({
+            client,
+            params: { Bucket, Key, Body: Readable.from(data), ChecksumAlgorithm },
+          });
+          await s3Upload.done();
 
-        const client = new MockFailureS3({
-          region,
+          const object = await client.getObject({ Bucket, Key });
+          expect(await object.Body?.transformToString()).toEqual(dataString);
         });
 
-        const requestLog = [] as string[];
-
-        client.middlewareStack.add(
-          (next, context) => async (args) => {
-            const result = await next(args);
-            requestLog.push(
-              [context.clientName, context.commandName, result.output.$metadata.httpStatusCode].join(" ")
-            );
-            return result;
-          },
-          {
-            name: "E2eRequestLog",
-            step: "build",
-            override: true,
+        it("should call AbortMultipartUpload if unable to complete a multipart upload.", async () => {
+          class MockFailureS3 extends S3 {
+            public counter = 0;
+            async send(command: any, ...rest: any[]) {
+              if (command?.constructor?.name === "UploadPartCommand" && this.counter++ % 3 === 0) {
+                throw new Error("simulated upload part error");
+              }
+              return super.send(command, ...rest);
+            }
           }
-        );
-
-        const s3Upload = new Upload({
-          client,
-          params: {
-            Bucket,
-            Key,
-            Body: data,
-          },
+
+          const client = new MockFailureS3({ region });
+
+          const requestLog = [] as string[];
+
+          client.middlewareStack.add(
+            (next, context) => async (args) => {
+              const result = await next(args);
+              requestLog.push(
+                [context.clientName, context.commandName, result.output.$metadata.httpStatusCode].join(" ")
+              );
+              return result;
+            },
+            {
+              name: "E2eRequestLog",
+              step: "build",
+              override: true,
+            }
+          );
+
+          const s3Upload = new Upload({
+            client,
+            params: { Bucket, Key, Body: data, ChecksumAlgorithm },
+          });
+          // eslint-disable-next-line @typescript-eslint/no-unused-vars
+          await s3Upload.done().catch((ignored) => {});
+
+          const uploadStatus = await client
+            .listParts({ Bucket, Key, UploadId: s3Upload.uploadId })
+            .then((listParts) => listParts.$metadata.httpStatusCode)
+            .catch((err) => err.toString());
+
+          expect(uploadStatus).toMatch(/NoSuchUpload:(.*?)aborted or completed\./);
+          expect(requestLog).toEqual([
+            "S3Client CreateMultipartUploadCommand 200",
+            "S3Client UploadPartCommand 200",
+            "S3Client UploadPartCommand 200",
+            "S3Client AbortMultipartUploadCommand 204",
+          ]);
         });
-        // eslint-disable-next-line @typescript-eslint/no-unused-vars
-        await s3Upload.done().catch((ignored) => {});
-
-        const uploadStatus = await client
-          .listParts({
-            Bucket,
-            Key,
-            UploadId: s3Upload.uploadId,
-          })
-          .then((listParts) => listParts.$metadata.httpStatusCode)
-          .catch((err) => err.toString());
-
-        expect(uploadStatus).toMatch(/NoSuchUpload:(.*?)aborted or completed\./);
-        expect(requestLog).toEqual([
-          "S3Client CreateMultipartUploadCommand 200",
-          "S3Client UploadPartCommand 200",
-          "S3Client UploadPartCommand 200",
-          "S3Client AbortMultipartUploadCommand 204",
-        ]);
       });
     }
   );