Fix : incremental cache fixes (#257)

* potential fix for EMFILE, too many open file

* fix dynamo provider not generated when no fetch cache

* Add flags to disable dynamodb cache and incremental cache

Author: conico974

packages/open-next/src/adapters/cache.ts

@@ -16,8 +16,7 @@ import {
 // import { getDerivedTags } from "next/dist/server/lib/incremental-cache/utils";
 import path from "path";
 
-import { loadBuildId } from "./config/util.js";
-import { awsLogger, debug, error } from "./logger.js";
+import { debug, error } from "./logger.js";
 
 // TODO: Remove this, temporary only to run some tests
 const getDerivedTags = (tags: string[]) => tags;
@@ -105,30 +104,32 @@ type Extension =
 const {
   CACHE_BUCKET_NAME,
   CACHE_BUCKET_KEY_PREFIX,
-  CACHE_BUCKET_REGION,
   CACHE_DYNAMO_TABLE,
+  NEXT_BUILD_ID,
 } = process.env;
 
+declare global {
+  var S3Client: S3Client;
+  var dynamoClient: DynamoDBClient;
+  var disableDynamoDBCache: boolean;
+  var disableIncrementalCache: boolean;
+}
+
 export default class S3Cache {
   private client: S3Client;
   private dynamoClient: DynamoDBClient;
   private buildId: string;
 
   constructor(_ctx: CacheHandlerContext) {
-    this.client = new S3Client({
-      region: CACHE_BUCKET_REGION,
-      logger: awsLogger,
-    });
-    this.dynamoClient = new DynamoDBClient({
-      region: CACHE_BUCKET_REGION,
-      logger: awsLogger,
-    });
-    this.buildId = loadBuildId(
-      path.dirname(_ctx.serverDistDir ?? ".next/server"),
-    );
+    this.client = globalThis.S3Client;
+    this.dynamoClient = globalThis.dynamoClient;
+    this.buildId = NEXT_BUILD_ID!;
   }
 
   public async get(key: string, options?: boolean | { fetchCache?: boolean }) {
+    if (globalThis.disableIncrementalCache) {
+      return null;
+    }
     const isFetchCache =
       typeof options === "object" ? options.fetchCache : options;
     const keys = await this.listS3Object(key);
@@ -257,6 +258,9 @@ export default class S3Cache {
   }
 
   async set(key: string, data?: IncrementalCacheValue): Promise<void> {
+    if (globalThis.disableIncrementalCache) {
+      return;
+    }
     if (data?.kind === "ROUTE") {
       const { body, status, headers } = data;
       await Promise.all([
@@ -317,6 +321,9 @@ export default class S3Cache {
   }
 
   public async revalidateTag(tag: string) {
+    if (globalThis.disableDynamoDBCache || globalThis.disableIncrementalCache) {
+      return;
+    }
     debug("revalidateTag", tag);
     // Find all keys with the given tag
     const paths = await this.getByTag(tag);
@@ -334,6 +341,7 @@ export default class S3Cache {
 
   private async getTagsByPath(path: string) {
     try {
+      if (disableDynamoDBCache) return [];
       const result = await this.dynamoClient.send(
         new QueryCommand({
           TableName: CACHE_DYNAMO_TABLE,
@@ -359,6 +367,7 @@ export default class S3Cache {
   //TODO: Figure out a better name for this function since it returns the lastModified
   private async getHasRevalidatedTags(key: string, lastModified?: number) {
     try {
+      if (disableDynamoDBCache) return lastModified ?? Date.now();
       const result = await this.dynamoClient.send(
         new QueryCommand({
           TableName: CACHE_DYNAMO_TABLE,
@@ -387,6 +396,7 @@ export default class S3Cache {
 
   private async getByTag(tag: string) {
     try {
+      if (disableDynamoDBCache) return [];
       const { Items } = await this.dynamoClient.send(
         new QueryCommand({
           TableName: CACHE_DYNAMO_TABLE,
@@ -413,6 +423,7 @@ export default class S3Cache {
 
   private async batchWriteDynamoItem(req: { path: string; tag: string }[]) {
     try {
+      if (disableDynamoDBCache) return;
       await Promise.all(
         this.chunkArray(req, 25).map((Items) => {
           return this.dynamoClient.send(

packages/open-next/src/adapters/server-adapter.ts

@@ -1,4 +1,7 @@
 /* eslint-disable unused-imports/no-unused-imports */
+import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
+import { S3Client } from "@aws-sdk/client-s3";
+
 // We load every config here so that they are only loaded once
 // and during cold starts
 import {
@@ -13,13 +16,34 @@ import {
   PublicAssets,
   RoutesManifest,
 } from "./config/index.js";
+import { awsLogger } from "./logger.js";
 import { lambdaHandler } from "./plugins/lambdaHandler.js";
 import { setNodeEnv } from "./util.js";
 
 setNodeEnv();
 setBuildIdEnv();
 setNextjsServerWorkingDirectory();
 
+///////////////////////
+// AWS global client //
+///////////////////////
+
+declare global {
+  var S3Client: S3Client;
+  var dynamoClient: DynamoDBClient;
+}
+
+const CACHE_BUCKET_REGION = process.env.CACHE_BUCKET_REGION;
+
+globalThis.S3Client = new S3Client({
+  region: CACHE_BUCKET_REGION,
+  logger: awsLogger,
+});
+globalThis.dynamoClient = new DynamoDBClient({
+  region: CACHE_BUCKET_REGION,
+  logger: awsLogger,
+});
+
 /////////////
 // Handler //
 /////////////

packages/open-next/src/build.ts

@@ -13,6 +13,19 @@ import {
 import { minifyAll } from "./minimize-js.js";
 import openNextPlugin from "./plugin.js";
 
+interface DangerousOptions {
+  /**
+   * The dynamo db cache is used for revalidateTags and revalidatePath.
+   * @default false
+   */
+  disableDynamoDBCache?: boolean;
+  /**
+   * The incremental cache is used for ISR and SSG.
+   * Disable this only if you use only SSR
+   * @default false
+   */
+  disableIncrementalCache?: boolean;
+}
 interface BuildOptions {
   /**
    * Minify the server bundle.
@@ -39,6 +52,10 @@ interface BuildOptions {
    * });
    * ```
    */
+  /**
+   * Dangerous options. This break some functionnality but can be useful in some cases.
+   */
+  dangerous?: DangerousOptions;
   buildCommand?: string;
   /**
    * The path to the target folder of build output from the `buildCommand` option (the path which will contain the `.next` and `.open-next` folders). This path is relative from the current process.cwd().
@@ -82,8 +99,10 @@ export async function build(opts: BuildOptions = {}) {
   printHeader("Generating bundle");
   initOutputDir();
   createStaticAssets();
-  createCacheAssets(monorepoRoot);
-  await createServerBundle(monorepoRoot, opts.streaming);
+  if (!options.dangerous?.disableIncrementalCache) {
+    createCacheAssets(monorepoRoot, options.dangerous?.disableDynamoDBCache);
+  }
+  await createServerBundle(monorepoRoot, options.streaming);
   createRevalidationBundle();
   createImageOptimizationBundle();
   createWarmerBundle();
@@ -109,6 +128,8 @@ function normalizeOptions(opts: BuildOptions, root: string) {
     minify: opts.minify ?? Boolean(process.env.OPEN_NEXT_MINIFY) ?? false,
     debug: opts.debug ?? Boolean(process.env.OPEN_NEXT_DEBUG) ?? false,
     buildCommand: opts.buildCommand,
+    dangerous: opts.dangerous,
+    streaming: opts.streaming ?? false,
   };
 }
 
@@ -369,7 +390,7 @@ function createStaticAssets() {
   }
 }
 
-function createCacheAssets(monorepoRoot: string) {
+function createCacheAssets(monorepoRoot: string, disableDynamoDBCache = false) {
   console.info(`Bundling cache assets...`);
 
   const { appBuildOutputPath, outputDir } = options;
@@ -398,23 +419,14 @@ function createCacheAssets(monorepoRoot: string) {
       (file.endsWith(".html") && htmlPages.has(file)),
   );
 
-  // Generate dynamodb data
-  // We need to traverse the cache to find every .meta file
-  const metaFiles: {
-    tag: { S: string };
-    path: { S: string };
-    revalidatedAt: { N: string };
-  }[] = [];
-
-  // Copy fetch-cache to cache folder
-  const fetchCachePath = path.join(
-    appBuildOutputPath,
-    ".next/cache/fetch-cache",
-  );
-  if (fs.existsSync(fetchCachePath)) {
-    const fetchOutputPath = path.join(outputDir, "cache", "__fetch", buildId);
-    fs.mkdirSync(fetchOutputPath, { recursive: true });
-    fs.cpSync(fetchCachePath, fetchOutputPath, { recursive: true });
+  if (!disableDynamoDBCache) {
+    // Generate dynamodb data
+    // We need to traverse the cache to find every .meta file
+    const metaFiles: {
+      tag: { S: string };
+      path: { S: string };
+      revalidatedAt: { N: string };
+    }[] = [];
 
     // Compute dynamodb cache data
     // Traverse files inside cache to find all meta files and cache tags associated with them
@@ -444,41 +456,54 @@ function createCacheAssets(monorepoRoot: string) {
       },
     );
 
-    traverseFiles(
-      fetchCachePath,
-      () => true,
-      (filepath) => {
-        const fileContent = fs.readFileSync(filepath, "utf8");
-        const fileData = JSON.parse(fileContent);
-        fileData?.tags?.forEach((tag: string) => {
-          metaFiles.push({
-            tag: { S: path.posix.join(buildId, tag) },
-            path: {
-              S: path.posix.join(
-                buildId,
-                path.relative(fetchCachePath, filepath),
-              ),
-            },
-            revalidatedAt: { N: `${Date.now()}` },
-          });
-        });
-      },
+    // Copy fetch-cache to cache folder
+    const fetchCachePath = path.join(
+      appBuildOutputPath,
+      ".next/cache/fetch-cache",
     );
+    if (fs.existsSync(fetchCachePath)) {
+      const fetchOutputPath = path.join(outputDir, "cache", "__fetch", buildId);
+      fs.mkdirSync(fetchOutputPath, { recursive: true });
+      fs.cpSync(fetchCachePath, fetchOutputPath, { recursive: true });
 
-    const providerPath = path.join(outputDir, "dynamodb-provider");
+      traverseFiles(
+        fetchCachePath,
+        () => true,
+        (filepath) => {
+          const fileContent = fs.readFileSync(filepath, "utf8");
+          const fileData = JSON.parse(fileContent);
+          fileData?.tags?.forEach((tag: string) => {
+            metaFiles.push({
+              tag: { S: path.posix.join(buildId, tag) },
+              path: {
+                S: path.posix.join(
+                  buildId,
+                  path.relative(fetchCachePath, filepath),
+                ),
+              },
+              revalidatedAt: { N: `${Date.now()}` },
+            });
+          });
+        },
+      );
+    }
 
-    esbuildSync({
-      external: ["@aws-sdk/client-dynamodb"],
-      entryPoints: [path.join(__dirname, "adapters", "dynamo-provider.js")],
-      outfile: path.join(providerPath, "index.mjs"),
-      target: ["node18"],
-    });
+    if (metaFiles.length > 0) {
+      const providerPath = path.join(outputDir, "dynamodb-provider");
 
-    // TODO: check if metafiles doesn't contain duplicates
-    fs.writeFileSync(
-      path.join(providerPath, "dynamodb-cache.json"),
-      JSON.stringify(metaFiles),
-    );
+      esbuildSync({
+        external: ["@aws-sdk/client-dynamodb"],
+        entryPoints: [path.join(__dirname, "adapters", "dynamo-provider.js")],
+        outfile: path.join(providerPath, "index.mjs"),
+        target: ["node18"],
+      });
+
+      // TODO: check if metafiles doesn't contain duplicates
+      fs.writeFileSync(
+        path.join(providerPath, "dynamodb-cache.json"),
+        JSON.stringify(metaFiles),
+      );
+    }
   }
 }
 
@@ -602,7 +627,7 @@ async function createServerBundle(monorepoRoot: string, streaming = false) {
   addPublicFilesList(outputPath, packagePath);
   injectMiddlewareGeolocation(outputPath, packagePath);
   removeCachedPages(outputPath, packagePath);
-  addCacheHandler(outputPath);
+  addCacheHandler(outputPath, options.dangerous);
 }
 
 function addMonorepoEntrypoint(outputPath: string, packagePath: string) {
@@ -710,13 +735,20 @@ function removeCachedPages(outputPath: string, packagePath: string) {
     );
 }
 
-function addCacheHandler(outputPath: string) {
+function addCacheHandler(outputPath: string, options?: DangerousOptions) {
   esbuildSync({
     external: ["next", "styled-jsx", "react"],
     entryPoints: [path.join(__dirname, "adapters", "cache.js")],
     outfile: path.join(outputPath, "cache.cjs"),
     target: ["node18"],
     format: "cjs",
+    banner: {
+      js: `globalThis.disableIncrementalCache = ${
+        options?.disableIncrementalCache ?? false
+      }; globalThis.disableDynamoDBCache = ${
+        options?.disableDynamoDBCache ?? false
+      };`,
+    },
   });
 }
 

packages/open-next/src/index.ts

@@ -14,6 +14,14 @@ build({
   appPath: args["--app-path"],
   minify: Object.keys(args).includes("--minify"),
   streaming: Object.keys(args).includes("--streaming"),
+  dangerous: {
+    disableDynamoDBCache: Object.keys(args).includes(
+      "--dangerously-disable-dynamodb-cache",
+    ),
+    disableIncrementalCache: Object.keys(args).includes(
+      "--dangerously-disable-incremental-cache",
+    ),
+  },
 });
 
 function parseArgs() {