ci: Bring back MDX caching with public folder cache

Author: BYK

.gitignore

@@ -4,9 +4,6 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 
-# Ignore generated export markdown files 
-/public/md-exports/
-
 # Runtime data
 pids
 *.pid
@@ -96,6 +93,8 @@ public/page-data
 # tsbuildinfo file generated by CI
 tsconfig.tsbuildinfo
 
+# Ignore generated files
+/public/md-exports/
 public/mdx-images/*
 
 # yalc

src/mdx.ts

@@ -3,8 +3,19 @@ import matter from 'gray-matter';
 import {s} from 'hastscript';
 import yaml from 'js-yaml';
 import {bundleMDX} from 'mdx-bundler';
-import {access, opendir, readFile} from 'node:fs/promises';
+import {BinaryLike, createHash} from 'node:crypto';
+import {createReadStream, createWriteStream, mkdirSync} from 'node:fs';
+import {access, cp, opendir, readFile} from 'node:fs/promises';
 import path from 'node:path';
+// @ts-expect-error ts(2305) -- For some reason "compose" is not recognized in the types
+import {compose, Readable} from 'node:stream';
+import {json} from 'node:stream/consumers';
+import {pipeline} from 'node:stream/promises';
+import {
+  constants as zlibConstants,
+  createBrotliCompress,
+  createBrotliDecompress,
+} from 'node:zlib';
 import {limitFunction} from 'p-limit';
 import rehypeAutolinkHeadings from 'rehype-autolink-headings';
 import rehypePresetMinify from 'rehype-preset-minify';
@@ -48,6 +59,33 @@ const root = process.cwd();
 // Functions which looks like AWS Lambda and we get `EMFILE` errors when trying to open
 // so many files at once.
 const FILE_CONCURRENCY_LIMIT = 200;
+const CACHE_COMPRESS_LEVEL = 4;
+const CACHE_DIR = path.join(root, '.next', 'cache', 'mdx-bundler');
+mkdirSync(CACHE_DIR, {recursive: true});
+
+const md5 = (data: BinaryLike) => createHash('md5').update(data).digest('hex');
+
+async function readCacheFile<T>(file: string): Promise<T> {
+  const reader = createReadStream(file);
+  const decompressor = createBrotliDecompress();
+
+  return (await json(compose(reader, decompressor))) as T;
+}
+
+async function writeCacheFile(file: string, data: string) {
+  await pipeline(
+    Readable.from(data),
+    createBrotliCompress({
+      chunkSize: 32 * 1024,
+      params: {
+        [zlibConstants.BROTLI_PARAM_MODE]: zlibConstants.BROTLI_MODE_TEXT,
+        [zlibConstants.BROTLI_PARAM_QUALITY]: CACHE_COMPRESS_LEVEL,
+        [zlibConstants.BROTLI_PARAM_SIZE_HINT]: data.length,
+      },
+    }),
+    createWriteStream(file)
+  );
+}
 
 function formatSlug(slug: string) {
   return slug.replace(/\.(mdx|md)/, '');
@@ -484,6 +522,20 @@ export async function getFileBySlug(slug: string): Promise<SlugFile> {
     );
   }
 
+  const cacheKey = md5(source);
+  const cacheFile = path.join(CACHE_DIR, `${cacheKey}.br`);
+
+  try {
+    const cached = await readCacheFile<SlugFile>(cacheFile);
+    return cached;
+  } catch (err) {
+    if (err.code !== 'ENOENT' && err.code !== 'ABORT_ERR' && err.code !== 'Z_BUF_ERROR') {
+      // If cache is corrupted, ignore and proceed
+      // eslint-disable-next-line no-console
+      console.warn(`Failed to read MDX cache: ${cacheFile}`, err);
+    }
+  }
+
   process.env.ESBUILD_BINARY_PATH = path.join(
     root,
     'node_modules',
@@ -496,6 +548,8 @@ export async function getFileBySlug(slug: string): Promise<SlugFile> {
 
   // cwd is how mdx-bundler knows how to resolve relative paths
   const cwd = path.dirname(sourcePath);
+  const assetsCacheDir = path.join(CACHE_DIR, cacheKey);
+  const outdir = path.join(root, 'public', 'mdx-images');
 
   const result = await bundleMDX<Platform>({
     source,
@@ -577,9 +631,10 @@ export async function getFileBySlug(slug: string): Promise<SlugFile> {
         // inline svgs
         '.svg': 'dataurl',
       };
+      options.metafile = true;
       // Set the `outdir` to a public location for this bundle.
       // this where this images will be copied
-      options.outdir = path.join(root, 'public', 'mdx-images');
+      options.outdir = assetsCacheDir;
 
       // Set write to true so that esbuild will output the files.
       options.write = true;
@@ -609,6 +664,15 @@ export async function getFileBySlug(slug: string): Promise<SlugFile> {
     },
   };
 
+  await cp(assetsCacheDir, outdir, {
+    recursive: true,
+    force: true,
+  });
+  writeCacheFile(cacheFile, JSON.stringify(resultObj)).catch(e => {
+    // eslint-disable-next-line no-console
+    console.warn(`Failed to write MDX cache: ${cacheFile}`, e);
+  });
+
   return resultObj;
 }