address byk's comments

Author: sergical

scripts/generate-md-exports.mjs

@@ -58,13 +58,13 @@ async function uploadToCFR2(s3Client, relativePath, data) {
   return;
 }
 
-function taskFinishHandler(
-  {id, success, failedTasks, usedCacheFiles},
-  allUsedCacheFiles
-) {
-  // Collect cache files used by this worker
-  if (usedCacheFiles) {
-    usedCacheFiles.forEach(file => allUsedCacheFiles.add(file));
+// Global set to track which cache files are used across all workers
+let globalUsedCacheFiles = null;
+
+function taskFinishHandler({id, success, failedTasks, usedCacheFiles}) {
+  // Collect cache files used by this worker into the global set
+  if (usedCacheFiles && globalUsedCacheFiles) {
+    usedCacheFiles.forEach(file => globalUsedCacheFiles.add(file));
   }
 
   if (failedTasks.length === 0) {
@@ -104,7 +104,7 @@ async function createWork() {
   }
 
   // Track which cache files are used during this build
-  const usedCacheFiles = new Set();
+  globalUsedCacheFiles = new Set();
 
   // On a 16-core machine, 8 workers were optimal (and slightly faster than 16)
   const numWorkers = Math.max(Math.floor(cpus().length / 2), 2);
@@ -174,7 +174,7 @@ async function createWork() {
         },
       });
       let hasErrors = false;
-      worker.on('message', data => (hasErrors = taskFinishHandler(data, usedCacheFiles)));
+      worker.on('message', data => (hasErrors = taskFinishHandler(data)));
       worker.on('error', reject);
       worker.on('exit', code => {
         if (code !== 0) {
@@ -195,7 +195,7 @@ async function createWork() {
       noCache,
       usedCacheFiles: mainThreadUsedFiles,
     }).then(data => {
-      if (taskFinishHandler(data, usedCacheFiles)) {
+      if (taskFinishHandler(data)) {
         throw new Error(`Worker[${data.id}] had some errors.`);
       }
     })
@@ -207,17 +207,13 @@ async function createWork() {
   if (!noCache) {
     try {
       const allFiles = await readdir(CACHE_DIR);
-      let cleanedCount = 0;
-
-      for (const file of allFiles) {
-        if (!usedCacheFiles.has(file)) {
-          await rm(path.join(CACHE_DIR, file), {force: true});
-          cleanedCount++;
-        }
-      }
+      const filesToDelete = allFiles.filter(file => !globalUsedCacheFiles.has(file));
 
-      if (cleanedCount > 0) {
-        console.log(`🧹 Cleaned up ${cleanedCount} unused cache files`);
+      if (filesToDelete.length > 0) {
+        await Promise.all(
+          filesToDelete.map(file => rm(path.join(CACHE_DIR, file), {force: true}))
+        );
+        console.log(`🧹 Cleaned up ${filesToDelete.length} unused cache files`);
       }
     } catch (err) {
       console.warn('Failed to clean unused cache files:', err);

src/mdx.ts

@@ -4,7 +4,7 @@ import yaml from 'js-yaml';
 import {bundleMDX} from 'mdx-bundler';
 import {BinaryLike, createHash} from 'node:crypto';
 import {createReadStream, createWriteStream, mkdirSync} from 'node:fs';
-import {access, cp, mkdir, opendir, readFile, rm, stat} from 'node:fs/promises';
+import {access, cp, mkdir, opendir, readFile} from 'node:fs/promises';
 import path from 'node:path';
 // @ts-expect-error ts(2305) -- For some reason "compose" is not recognized in the types
 import {compose, Readable} from 'node:stream';
@@ -64,45 +64,6 @@ const CACHE_COMPRESS_LEVEL = 4;
 const CACHE_DIR = path.join(root, '.next', 'cache', 'mdx-bundler');
 if (process.env.CI) {
   mkdirSync(CACHE_DIR, {recursive: true});
-
-  // Clean up old cache files in background to prevent unbounded growth
-  // Delete any file not accessed in the last 24 hours (meaning it wasn't used in recent builds)
-  // This runs once per worker process and doesn't block the build
-  (async () => {
-    try {
-      const MAX_CACHE_AGE_MS = 24 * 60 * 60 * 1000; // 24 hours
-      const now = Date.now();
-      let cleanedCount = 0;
-
-      const dir = await opendir(CACHE_DIR);
-
-      for await (const dirent of dir) {
-        if (!dirent.isFile() && !dirent.isDirectory()) {
-          continue;
-        }
-
-        const itemPath = path.join(CACHE_DIR, dirent.name);
-        try {
-          const stats = await stat(itemPath);
-          const age = now - stats.atimeMs; // Time since last access
-
-          if (age > MAX_CACHE_AGE_MS) {
-            await rm(itemPath, {recursive: true, force: true});
-            cleanedCount++;
-          }
-        } catch (err) {
-          // Skip items we can't stat/delete
-        }
-      }
-
-      if (cleanedCount > 0) {
-        // eslint-disable-next-line no-console
-        console.log(`🧹 MDX cache: Cleaned up ${cleanedCount} unused items (>24h)`);
-      }
-    } catch (err) {
-      // Silently fail - cache cleanup is not critical
-    }
-  })();
 }
 
 const md5 = (data: BinaryLike) => createHash('md5').update(data).digest('hex');