test release registry cache approach

Author: sergical

src/mdx.ts

@@ -26,7 +26,6 @@ import remarkMdxImages from 'remark-mdx-images';
 import getAppRegistry from './build/appRegistry';
 import getPackageRegistry from './build/packageRegistry';
 import {apiCategories} from './build/resolveOpenAPI';
-import {BuildTimer, logBuildInfo, OperationAggregator} from './buildTimer';
 import getAllFilesRecursively from './files';
 import remarkDefList from './mdx-deflist';
 import rehypeOnboardingLines from './rehype-onboarding-lines';
@@ -66,12 +65,6 @@ if (process.env.CI) {
   mkdirSync(CACHE_DIR, {recursive: true});
 }
 
-// Track MDX compilation performance
-const mdxCompilationAggregator = new OperationAggregator('MDX Compilation', {
-  progressInterval: 500, // log every 500 compilations (less verbose)
-  slowThreshold: 3000, // 3 seconds (only really slow pages)
-});
-
 const md5 = (data: BinaryLike) => createHash('md5').update(data).digest('hex');
 
 async function readCacheFile<T>(file: string): Promise<T> {
@@ -153,108 +146,7 @@ export const getVersionsFromDoc = (frontMatter: FrontMatter[], docPath: string)
   return [...new Set(versions)];
 };
 
-// Lock file management for cross-worker cache coordination
-const lockFile = (cacheFile: string) =>
-  path.join(root, '.next', 'cache', `${cacheFile}.lock`);
-
-/**
- * Try to load frontmatter from disk cache created by first worker
- * If cache is being built, wait for it to complete
- */
-async function loadFrontmatterFromCache(
-  cacheFile: string
-): Promise<FrontMatter[] | null> {
-  const cachePath = path.join(root, '.next', 'cache', cacheFile);
-  const lockPath = lockFile(cacheFile);
-
-  // Check if another worker is building the cache
-  let waitAttempts = 0;
-  while (waitAttempts < 30) {
-    // Wait up to 15 seconds
-    try {
-      // If lock file exists, another worker is building cache
-      await access(lockPath);
-      await new Promise(resolve => setTimeout(resolve, 500));
-      waitAttempts++;
-      continue;
-    } catch {
-      // No lock file, proceed
-      break;
-    }
-  }
-
-  try {
-    const cacheData = JSON.parse(await readFile(cachePath, 'utf-8'));
-    const age = Date.now() - cacheData.timestamp;
-
-    // Cache is valid for 1 hour (in case of incremental builds)
-    if (age < 60 * 60 * 1000) {
-      logBuildInfo(
-        `✓ Loaded ${cacheData.frontmatter.length} files from worker cache (${(age / 1000).toFixed(0)}s old)`
-      );
-      return cacheData.frontmatter;
-    }
-    return null;
-  } catch {
-    // Cache doesn't exist or is invalid
-    return null;
-  }
-}
-
-/**
- * Save frontmatter to disk cache for other workers
- */
-async function saveFrontmatterToCache(
-  cacheFile: string,
-  frontmatter: FrontMatter[]
-): Promise<void> {
-  try {
-    const cachePath = path.join(root, '.next', 'cache', cacheFile);
-    const cacheData = {
-      timestamp: Date.now(),
-      frontmatter,
-    };
-    await mkdir(path.dirname(cachePath), {recursive: true});
-    await import('fs/promises').then(fs =>
-      fs.writeFile(cachePath, JSON.stringify(cacheData))
-    );
-    logBuildInfo(`✓ Saved ${frontmatter.length} files to worker cache for other workers`);
-  } catch (error) {
-    // Non-fatal - other workers will just build their own
-    // eslint-disable-next-line no-console
-    console.error('Failed to save frontmatter cache:', error);
-  }
-}
-
 async function getDocsFrontMatterUncached(): Promise<FrontMatter[]> {
-  // Try to load from worker cache first
-  const cached = await loadFrontmatterFromCache('frontmatter.json');
-  if (cached) {
-    return cached;
-  }
-
-  // Create lock file so other workers wait
-  const lockPath = lockFile('frontmatter.json');
-  let lockFd: number | null = null;
-  try {
-    // Try to create lock file atomically
-    const {open} = await import('fs/promises');
-    lockFd = await open(lockPath, 'wx')
-      .then(fh => fh.fd)
-      .catch(() => null);
-
-    // If we couldn't create lock, another worker is building - wait for them
-    if (!lockFd) {
-      const cached2 = await loadFrontmatterFromCache('frontmatter.json');
-      if (cached2) {
-        return cached2;
-      }
-      // Cache not ready, fall through to build ourselves
-    }
-  } catch {
-    // Lock file issues are non-fatal
-  }
-
   const frontMatter = await getAllFilesFrontMatter();
 
   const categories = await apiCategories();
@@ -287,50 +179,10 @@ async function getDocsFrontMatterUncached(): Promise<FrontMatter[]> {
     }
   });
 
-  // Save to cache for other workers (if we got the lock)
-  if (lockFd) {
-    await saveFrontmatterToCache('frontmatter.json', frontMatter);
-    // Remove lock file
-    try {
-      const {unlink} = await import('fs/promises');
-      await unlink(lockPath);
-    } catch {
-      // Non-fatal
-    }
-  }
-
   return frontMatter;
 }
 
 export async function getDevDocsFrontMatterUncached(): Promise<FrontMatter[]> {
-  // Try to load from worker cache first
-  const cached = await loadFrontmatterFromCache('frontmatter-dev.json');
-  if (cached) {
-    return cached;
-  }
-
-  // Create lock file so other workers wait
-  const lockPath = lockFile('frontmatter-dev.json');
-  let lockFd: number | null = null;
-  try {
-    // Try to create lock file atomically
-    const {open} = await import('fs/promises');
-    lockFd = await open(lockPath, 'wx')
-      .then(fh => fh.fd)
-      .catch(() => null);
-
-    // If we couldn't create lock, another worker is building - wait for them
-    if (!lockFd) {
-      const cached2 = await loadFrontmatterFromCache('frontmatter-dev.json');
-      if (cached2) {
-        return cached2;
-      }
-      // Cache not ready, fall through to build ourselves
-    }
-  } catch {
-    // Lock file issues are non-fatal
-  }
-
   const folder = 'develop-docs';
   const docsPath = path.join(root, folder);
   const files = await getAllFilesRecursively(docsPath);
@@ -358,18 +210,6 @@ export async function getDevDocsFrontMatterUncached(): Promise<FrontMatter[]> {
     )
   ).filter(isNotNil);
 
-  // Save to cache for other workers (if we got the lock)
-  if (lockFd) {
-    await saveFrontmatterToCache('frontmatter-dev.json', frontMatters);
-    // Remove lock file
-    try {
-      const {unlink} = await import('fs/promises');
-      await unlink(lockPath);
-    } catch {
-      // Non-fatal
-    }
-  }
-
   return frontMatters;
 }
 
@@ -383,12 +223,10 @@ export function getDevDocsFrontMatter(): Promise<FrontMatter[]> {
 }
 
 async function getAllFilesFrontMatter(): Promise<FrontMatter[]> {
-  const timer = new BuildTimer('getAllFilesFrontMatter');
   const docsPath = path.join(root, 'docs');
   const files = await getAllFilesRecursively(docsPath);
   const allFrontMatter: FrontMatter[] = [];
 
-  const readFilesTimer = new BuildTimer('Reading MDX frontmatter');
   await Promise.all(
     files.map(
       limitFunction(
@@ -414,10 +252,8 @@ async function getAllFilesFrontMatter(): Promise<FrontMatter[]> {
       )
     )
   );
-  readFilesTimer.end(true); // Silent - we'll show in summary
 
   // Add all `common` files in the right place.
-  const commonFilesTimer = new BuildTimer('Processing common platform files');
   const platformsPath = path.join(docsPath, 'platforms');
   for await (const platform of await opendir(platformsPath)) {
     if (platform.isFile()) {
@@ -561,8 +397,7 @@ async function getAllFilesFrontMatter(): Promise<FrontMatter[]> {
       );
     }
   }
-  commonFilesTimer.end(true); // Silent - we'll show in summary
-  timer.end();
+
   return allFrontMatter;
 }
 
@@ -600,8 +435,6 @@ export const addVersionToFilePath = (filePath: string, version: string) => {
 };
 
 export async function getFileBySlug(slug: string): Promise<SlugFile> {
-  const compileStart = Date.now();
-
   // no versioning on a config file
   const configPath = path.join(root, slug.split(VERSION_INDICATOR)[0], 'config.yml');
 
@@ -707,71 +540,54 @@ export async function getFileBySlug(slug: string): Promise<SlugFile> {
     source.includes('<LambdaLayerDetail');
 
   if (process.env.CI) {
-    // Build cache key from: source content + registry data (if needed) + build ID
+    // Build cache key from source content
     const sourceHash = md5(source);
 
     // For files that depend on registry, include registry version in cache key
-    let registryHash = '';
+    // This prevents serving stale content when registry is updated
     if (dependsOnRegistry) {
       try {
-        // Get registry data (already cached in memory by each worker)
+        // Get registry data (cached in memory per worker)
         const [apps, packages] = await Promise.all([
           getAppRegistry(),
           getPackageRegistry(),
         ]);
         // Hash the registry data to create a stable version identifier
-        registryHash = md5(JSON.stringify({apps, packages}));
+        const registryHash = md5(JSON.stringify({apps, packages}));
+        cacheKey = `${sourceHash}-${registryHash}`;
       } catch (err) {
-        // If registry fetch fails, skip caching
+        // If registry fetch fails, skip caching for safety
         // eslint-disable-next-line no-console
-        console.warn(`Failed to fetch registry for cache key: ${err.message}`);
-        registryHash = 'no-registry';
+        console.warn(
+          `Failed to fetch registry for cache key, skipping cache: ${err.message}`
+        );
+        cacheKey = null;
       }
+    } else {
+      // Regular files without registry dependencies
+      cacheKey = sourceHash;
     }
 
-    // Use VERCEL_GIT_COMMIT_REF (branch name) for cache key
-    // This allows cache to persist across commits on the same branch!
-    // Falls back to VERCEL_DEPLOYMENT_ID for non-git deploys, then BUILD_ID, then 'local'
-    const cacheVersion =
-      process.env.VERCEL_GIT_COMMIT_REF ||
-      process.env.VERCEL_DEPLOYMENT_ID ||
-      process.env.BUILD_ID ||
-      'local';
-
-    // Cache key: source + registry version + branch/build identifier
-    // This ensures:
-    // 1. All workers in the same build share cache (same cacheVersion)
-    // 2. Cache persists across commits on same branch (same VERCEL_GIT_COMMIT_REF)
-    // 3. Cache is invalidated when registry changes (registryHash changes)
-    // 4. Cache is invalidated when source changes (sourceHash changes)
-    cacheKey = dependsOnRegistry
-      ? `${sourceHash}-${registryHash}-${cacheVersion}`
-      : `${sourceHash}-${cacheVersion}`;
-
-    cacheFile = path.join(CACHE_DIR, `${cacheKey}.br`);
-    assetsCacheDir = path.join(CACHE_DIR, cacheKey);
+    if (cacheKey) {
+      cacheFile = path.join(CACHE_DIR, `${cacheKey}.br`);
+      assetsCacheDir = path.join(CACHE_DIR, cacheKey);
 
-    try {
-      const [cached, _] = await Promise.all([
-        readCacheFile<SlugFile>(cacheFile),
-        cp(assetsCacheDir, outdir, {recursive: true}),
-      ]);
-      if (dependsOnRegistry) {
-        // eslint-disable-next-line no-console
-        console.info(
-          `✓ Using cached registry-dependent file: ${sourcePath} (branch: ${cacheVersion}, registry: ${registryHash.slice(0, 8)})`
-        );
-      }
-      return cached;
-    } catch (err) {
-      if (
-        err.code !== 'ENOENT' &&
-        err.code !== 'ABORT_ERR' &&
-        err.code !== 'Z_BUF_ERROR'
-      ) {
-        // If cache is corrupted, ignore and proceed
-        // eslint-disable-next-line no-console
-        console.warn(`Failed to read MDX cache: ${cacheFile}`, err);
+      try {
+        const [cached, _] = await Promise.all([
+          readCacheFile<SlugFile>(cacheFile),
+          cp(assetsCacheDir, outdir, {recursive: true}),
+        ]);
+        return cached;
+      } catch (err) {
+        if (
+          err.code !== 'ENOENT' &&
+          err.code !== 'ABORT_ERR' &&
+          err.code !== 'Z_BUF_ERROR'
+        ) {
+          // If cache is corrupted, ignore and proceed
+          // eslint-disable-next-line no-console
+          console.warn(`Failed to read MDX cache: ${cacheFile}`, err);
+        }
       }
     }
   }
@@ -915,25 +731,9 @@ export async function getFileBySlug(slug: string): Promise<SlugFile> {
     });
   }
 
-  // Track compilation time
-  const compileDuration = Date.now() - compileStart;
-  mdxCompilationAggregator.track(slug, compileDuration);
-
   return resultObj;
 }
 
-// Log final MDX compilation stats when process exits
-if (typeof process !== 'undefined') {
-  process.on('beforeExit', () => {
-    const stats = mdxCompilationAggregator.getStats();
-    if (stats.count > 0) {
-      logBuildInfo('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
-      mdxCompilationAggregator.logFinalSummary();
-      logBuildInfo('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
-    }
-  });
-}
-
 const fileBySlugCache = new Map<string, Promise<SlugFile>>();
 
 /**