SDK maxAttempts now configurable through env, plugin logs now debug (#310)

* Double chunk DDB batch writes to not overwhelm DDB on load

Author: dkershner6

README.md

@@ -54,6 +54,8 @@ Vercel link: https://open-next.vercel.app
 
 ### Environment variables
 
+- AWS_SDK_DYNAMODB_MAX_ATTEMPTS: The maximum number of times requests that encounter retryable failures should be attempted for DynamoDB. Defaults to 3.
+- AWS_SDK_S3_MAX_ATTEMPTS: The maximum number of times requests that encounter retryable failures should be attempted for S3. Defaults to 3.
 - DYNAMO_BATCH_WRITE_COMMAND_CONCURRENCY: The number of concurrent batch write commands to DynamoDB. Defaults to 4 in an effort to leave plenty of DynamoDB write request capacity for the production load.
 
 ## Contribute

packages/open-next/src/adapters/server-adapter.ts

@@ -1,21 +1,20 @@
-/* eslint-disable unused-imports/no-unused-imports */
-import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
-import { S3Client } from "@aws-sdk/client-s3";
+import { DynamoDBClient, DynamoDBClientConfig } from "@aws-sdk/client-dynamodb";
+import { S3Client, S3ClientConfig } from "@aws-sdk/client-s3";
 
-// We load every config here so that they are only loaded once
-// and during cold starts
 import { BuildId } from "./config/index.js";
 import { awsLogger } from "./logger.js";
 import { lambdaHandler } from "./plugins/lambdaHandler.js";
 import { setNodeEnv } from "./util.js";
 
+// We load every config here so that they are only loaded once
+// and during cold starts
 setNodeEnv();
 setBuildIdEnv();
 setNextjsServerWorkingDirectory();
 
-///////////////////////
-// AWS global client //
-///////////////////////
+////////////////////////
+// AWS global clients //
+////////////////////////
 
 declare global {
   var S3Client: S3Client;
@@ -24,20 +23,40 @@ declare global {
 
 const CACHE_BUCKET_REGION = process.env.CACHE_BUCKET_REGION;
 
+function parseS3ClientConfigFromEnv(): S3ClientConfig {
+  return {
+    region: CACHE_BUCKET_REGION,
+    logger: awsLogger,
+    maxAttempts: parseNumberFromEnv(process.env.AWS_SDK_S3_MAX_ATTEMPTS),
+  };
+}
+
+function parseDynamoClientConfigFromEnv(): DynamoDBClientConfig {
+  return {
+    region: CACHE_BUCKET_REGION,
+    logger: awsLogger,
+    maxAttempts: parseNumberFromEnv(process.env.AWS_SDK_DYNAMODB_MAX_ATTEMPTS),
+  };
+}
+
+function parseNumberFromEnv(envValue: string | undefined): number | undefined {
+  if (typeof envValue !== "string") {
+    return envValue;
+  }
+
+  const parsedValue = parseInt(envValue);
+
+  return isNaN(parsedValue) ? undefined : parsedValue;
+}
+
 // Cache clients using global variables
 // Note: The clients are used in `cache.ts`. The incremental cache is recreated on
 //       every request and required on every request (And the require cache is also
 //       cleared). It was causing some file to stay open which after enough time
 //       would cause the function to crash with error "EMFILE too many open". It
 //       was also making the memory grow out of control.
-globalThis.S3Client = new S3Client({
-  region: CACHE_BUCKET_REGION,
-  logger: awsLogger,
-});
-globalThis.dynamoClient = new DynamoDBClient({
-  region: CACHE_BUCKET_REGION,
-  logger: awsLogger,
-});
+globalThis.S3Client = new S3Client(parseS3ClientConfigFromEnv());
+globalThis.dynamoClient = new DynamoDBClient(parseDynamoClientConfigFromEnv());
 
 /////////////
 // Handler //