feat(client-s3): make expect continue header configurable

Author: kuhe

packages/middleware-expect-continue/src/index.spec.ts

@@ -11,7 +11,10 @@ describe("addExpectContinueMiddleware", () => {
   });
 
   it("sets the Expect header to 100-continue if there is a request body in node runtime", async () => {
-    const handler = addExpectContinueMiddleware({ runtime: "node" })(mockNextHandler, {} as any);
+    const handler = addExpectContinueMiddleware({ runtime: "node", expectContinueHeader: true })(
+      mockNextHandler,
+      {} as any
+    );
     await handler({
       input: {},
       request: new HttpRequest({
@@ -27,7 +30,10 @@ describe("addExpectContinueMiddleware", () => {
   });
 
   it("does not set the Expect header to 100-continue if there is no request body in node runtime", async () => {
-    const handler = addExpectContinueMiddleware({ runtime: "node" })(mockNextHandler, {} as any);
+    const handler = addExpectContinueMiddleware({ runtime: "node", expectContinueHeader: true })(
+      mockNextHandler,
+      {} as any
+    );
     await handler({
       input: {},
       request: new HttpRequest({
@@ -42,7 +48,10 @@ describe("addExpectContinueMiddleware", () => {
   });
 
   it("does not set the Expect header to 100-continue for browser runtime", async () => {
-    const handler = addExpectContinueMiddleware({ runtime: "browser" })(mockNextHandler, {} as any);
+    const handler = addExpectContinueMiddleware({ runtime: "browser", expectContinueHeader: true })(
+      mockNextHandler,
+      {} as any
+    );
     await handler({
       input: {},
       request: new HttpRequest({

packages/middleware-expect-continue/src/index.ts

@@ -1,5 +1,6 @@
 import { HttpHandler, HttpRequest } from "@smithy/protocol-http";
-import {
+import type {
+  BodyLengthCalculator,
   BuildHandler,
   BuildHandlerArguments,
   BuildHandlerOptions,
@@ -13,20 +14,43 @@ import {
 interface PreviouslyResolved {
   runtime: string;
   requestHandler?: RequestHandler<any, any, any> | HttpHandler<any>;
+  bodyLengthChecker?: BodyLengthCalculator;
+  expectContinueHeader?: boolean | number;
 }
 
 export function addExpectContinueMiddleware(options: PreviouslyResolved): BuildMiddleware<any, any> {
   return <Output extends MetadataBearer>(next: BuildHandler<any, Output>): BuildHandler<any, Output> =>
     async (args: BuildHandlerArguments<any>): Promise<BuildHandlerOutput<Output>> => {
       const { request } = args;
-      if (HttpRequest.isInstance(request) && request.body && options.runtime === "node") {
-        if (options.requestHandler?.constructor?.name !== "FetchHttpHandler") {
-          request.headers = {
-            ...request.headers,
-            Expect: "100-continue",
-          };
+
+      if (
+        options.expectContinueHeader !== false &&
+        HttpRequest.isInstance(request) &&
+        request.body &&
+        options.runtime === "node" &&
+        options.requestHandler?.constructor?.name !== "FetchHttpHandler"
+      ) {
+        let sendHeader = true;
+        if (typeof options.expectContinueHeader === "number") {
+          try {
+            const bodyLength =
+              Number(request.headers?.["content-length"]) ?? options.bodyLengthChecker?.(request.body) ?? Infinity;
+            sendHeader = bodyLength >= options.expectContinueHeader;
+            console.log({
+              sendHeader,
+              bodyLength,
+              threshold: options.expectContinueHeader,
+            });
+          } catch (e) {}
+        } else {
+          sendHeader = !!options.expectContinueHeader;
+        }
+
+        if (sendHeader) {
+          request.headers.Expect = "100-continue";
         }
       }
+
       return next({
         ...args,
         request,

packages/middleware-expect-continue/src/middleware-expect-continue.integ.spec.ts

@@ -5,35 +5,88 @@ import { describe, expect, test as it } from "vitest";
 describe("middleware-expect-continue", () => {
   describe(S3.name, () => {
     it("should not set expect header if there is no body", async () => {
-      const client = new S3({ region: "us-west-2" });
-
+      const client = new S3({ region: "us-west-2", expectContinueHeader: true });
       requireRequestsFrom(client).toMatch({
         headers: {
           Expect: /undefined/,
         },
       });
-
       await client.listBuckets({});
-
       expect.assertions(1);
     });
 
     it("should set expect header if there is a body", async () => {
-      const client = new S3({ region: "us-west-2" });
-
+      const client = new S3({ region: "us-west-2", expectContinueHeader: 4 });
       requireRequestsFrom(client).toMatch({
         headers: {
           Expect: /100-continue/,
         },
       });
-
       await client.putObject({
         Bucket: "b",
         Key: "k",
         Body: Buffer.from("abcd"),
       });
-
       expect.assertions(1);
     });
+
+    describe("should set or omit expect header based on configurations", () => {
+      it("false", async () => {
+        const client = new S3({ region: "us-west-2", expectContinueHeader: false });
+        requireRequestsFrom(client).toMatch({
+          headers: {
+            Expect: /undefined/,
+          },
+        });
+        await client.putObject({
+          Bucket: "b",
+          Key: "k",
+          Body: Buffer.from("abcd"),
+        });
+        expect.assertions(1);
+      });
+      it("5", async () => {
+        const client = new S3({ region: "us-west-2", expectContinueHeader: 5 });
+        requireRequestsFrom(client).toMatch({
+          headers: {
+            Expect: /undefined/,
+          },
+        });
+        await client.putObject({
+          Bucket: "b",
+          Key: "k",
+          Body: Buffer.from("abcd"),
+        });
+        expect.assertions(1);
+      });
+      it("true", async () => {
+        const client = new S3({ region: "us-west-2", expectContinueHeader: true });
+        requireRequestsFrom(client).toMatch({
+          headers: {
+            Expect: /100-continue/,
+          },
+        });
+        await client.putObject({
+          Bucket: "b",
+          Key: "k",
+          Body: Buffer.from("abcd"),
+        });
+        expect.assertions(1);
+      });
+      it("4", async () => {
+        const client = new S3({ region: "us-west-2", expectContinueHeader: 4 });
+        requireRequestsFrom(client).toMatch({
+          headers: {
+            Expect: /100-continue/,
+          },
+        });
+        await client.putObject({
+          Bucket: "b",
+          Key: "k",
+          Body: Buffer.from("abcd"),
+        });
+        expect.assertions(1);
+      });
+    });
   });
 });

packages/middleware-sdk-s3/src/s3Configuration.spec.ts

@@ -11,4 +11,37 @@ describe(resolveS3Config.name, () => {
       })
     ).toBe(input);
   });
+
+  it("accepts bool/num for expectContinueHeader and defaults to 2mb", () => {
+    expect(
+      resolveS3Config(
+        {
+          expectContinueHeader: 1,
+        },
+        {
+          session: [() => null, vi.fn()],
+        }
+      ).expectContinueHeader
+    ).toEqual(1);
+
+    expect(
+      resolveS3Config(
+        {
+          expectContinueHeader: false,
+        },
+        {
+          session: [() => null, vi.fn()],
+        }
+      ).expectContinueHeader
+    ).toEqual(false);
+
+    expect(
+      resolveS3Config(
+        {},
+        {
+          session: [() => null, vi.fn()],
+        }
+      ).expectContinueHeader
+    ).toEqual(2 * 1024 * 1024);
+  });
 });

packages/middleware-sdk-s3/src/s3Configuration.ts

@@ -35,6 +35,22 @@ export interface S3InputConfig {
    * Whether to use the bucket name as the endpoint for this client.
    */
   bucketEndpoint?: boolean;
+  /**
+   * This field configures the SDK's behavior around setting the `expect: 100-continue` header.
+   *
+   * Default: 2_097_152 (2 MB)
+   *
+   * When given as a boolean - always send or omit the header.
+   * When given as a number - minimum byte threshold of the payload before setting the header.
+   *                          Unmeasurable payload sizes (streams) will set the header too.
+   *
+   * The `expect: 100-continue` header is used to allow the server a chance to validate the PUT request
+   * headers before the client begins to send the object payload. This avoids wasteful data transmission for a
+   * request that is rejected.
+   *
+   * However, there is a trade-off where the request will take longer to complete.
+   */
+  expectContinueHeader?: boolean | number;
 }
 
 /**
@@ -58,6 +74,7 @@ export interface S3ResolvedConfig {
   followRegionRedirects: boolean;
   s3ExpressIdentityProvider: S3ExpressIdentityProvider;
   bucketEndpoint: boolean;
+  expectContinueHeader: boolean | number;
 }
 
 export const resolveS3Config = <T>(
@@ -76,6 +93,7 @@ export const resolveS3Config = <T>(
     followRegionRedirects,
     s3ExpressIdentityProvider,
     bucketEndpoint,
+    expectContinueHeader,
   } = input;
 
   return Object.assign(input, {
@@ -93,5 +111,6 @@ export const resolveS3Config = <T>(
         )
       ),
     bucketEndpoint: bucketEndpoint ?? false,
+    expectContinueHeader: expectContinueHeader ?? 2_097_152,
   });
 };

private/aws-client-api-test/src/client-interface-tests/client-s3/impl/initializeWithMaximalConfiguration.ts

@@ -132,6 +132,7 @@ export const initializeWithMaximalConfiguration = () => {
     responseChecksumValidation: DEFAULT_RESPONSE_CHECKSUM_VALIDATION,
     userAgentAppId: "testApp",
     requestStreamBufferSize: 8 * 1024,
+    expectContinueHeader: 8 * 1024 * 1024,
   };
 
   const s3 = new S3Client(config);