diff --git a/.gitignore b/.gitignore index 7c590f048bd13..1080e5838761c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,9 +4,6 @@ npm-debug.log* yarn-debug.log* yarn-error.log* -# Ignore generated export markdown files -/public/md-exports/ - # Runtime data pids *.pid @@ -96,6 +93,8 @@ public/page-data # tsbuildinfo file generated by CI tsconfig.tsbuildinfo +# Ignore generated files +/public/md-exports/ public/mdx-images/* # yalc diff --git a/scripts/generate-md-exports.mjs b/scripts/generate-md-exports.mjs index 2deff4d557a5e..2a199b73e841a 100644 --- a/scripts/generate-md-exports.mjs +++ b/scripts/generate-md-exports.mjs @@ -110,7 +110,7 @@ async function createWork() { }) ); continuationToken = response.NextContinuationToken; - for (const {Key, ETag} of response.Contents) { + for (const {Key, ETag} of response.Contents || []) { existingFilesOnR2.set(Key, ETag.slice(1, -1)); // Remove quotes from ETag } } while (continuationToken); diff --git a/src/mdx.ts b/src/mdx.ts index f9c363d713a77..846ebbbbcec45 100644 --- a/src/mdx.ts +++ b/src/mdx.ts @@ -1,10 +1,20 @@ -import {cache} from 'react'; import matter from 'gray-matter'; import {s} from 'hastscript'; import yaml from 'js-yaml'; import {bundleMDX} from 'mdx-bundler'; -import {access, opendir, readFile} from 'node:fs/promises'; +import {BinaryLike, createHash} from 'node:crypto'; +import {createReadStream, createWriteStream, mkdirSync} from 'node:fs'; +import {access, cp, mkdir, opendir, readFile} from 'node:fs/promises'; import path from 'node:path'; +// @ts-expect-error ts(2305) -- For some reason "compose" is not recognized in the types +import {compose, Readable} from 'node:stream'; +import {json} from 'node:stream/consumers'; +import {pipeline} from 'node:stream/promises'; +import { + constants as zlibConstants, + createBrotliCompress, + createBrotliDecompress, +} from 'node:zlib'; import {limitFunction} from 'p-limit'; import rehypeAutolinkHeadings from 'rehype-autolink-headings'; import rehypePresetMinify from 'rehype-preset-minify'; @@ -48,6 +58,34 @@ const root = process.cwd(); // Functions which looks like AWS Lambda and we get `EMFILE` errors when trying to open // so many files at once. const FILE_CONCURRENCY_LIMIT = 200; +const CACHE_COMPRESS_LEVEL = 4; +const CACHE_DIR = path.join(root, '.next', 'cache', 'mdx-bundler'); +mkdirSync(CACHE_DIR, {recursive: true}); + +const md5 = (data: BinaryLike) => createHash('md5').update(data).digest('hex'); + +async function readCacheFile(file: string): Promise { + const reader = createReadStream(file); + const decompressor = createBrotliDecompress(); + + return (await json(compose(reader, decompressor))) as T; +} + +async function writeCacheFile(file: string, data: string) { + const bufferData = Buffer.from(data); + await pipeline( + Readable.from(bufferData), + createBrotliCompress({ + chunkSize: 32 * 1024, + params: { + [zlibConstants.BROTLI_PARAM_MODE]: zlibConstants.BROTLI_MODE_TEXT, + [zlibConstants.BROTLI_PARAM_QUALITY]: CACHE_COMPRESS_LEVEL, + [zlibConstants.BROTLI_PARAM_SIZE_HINT]: bufferData.length, + }, + }), + createWriteStream(file) + ); +} function formatSlug(slug: string) { return slug.replace(/\.(mdx|md)/, ''); @@ -484,6 +522,36 @@ export async function getFileBySlug(slug: string): Promise { ); } + let cacheKey: string | null = null; + let cacheFile: string | null = null; + let assetsCacheDir: string | null = null; + const outdir = path.join(root, 'public', 'mdx-images'); + await mkdir(outdir, {recursive: true}); + + if (process.env.CI) { + cacheKey = md5(source); + cacheFile = path.join(CACHE_DIR, `${cacheKey}.br`); + assetsCacheDir = path.join(CACHE_DIR, cacheKey); + + try { + const [cached, _] = await Promise.all([ + readCacheFile(cacheFile), + cp(assetsCacheDir, outdir, {recursive: true}), + ]); + return cached; + } catch (err) { + if ( + err.code !== 'ENOENT' && + err.code !== 'ABORT_ERR' && + err.code !== 'Z_BUF_ERROR' + ) { + // If cache is corrupted, ignore and proceed + // eslint-disable-next-line no-console + console.warn(`Failed to read MDX cache: ${cacheFile}`, err); + } + } + } + process.env.ESBUILD_BINARY_PATH = path.join( root, 'node_modules', @@ -578,8 +646,12 @@ export async function getFileBySlug(slug: string): Promise { '.svg': 'dataurl', }; // Set the `outdir` to a public location for this bundle. - // this where this images will be copied - options.outdir = path.join(root, 'public', 'mdx-images'); + // this is where these images will be copied + // the reason we use the cache folder when it's + // enabled is because mdx-images is a dumping ground + // for all images, so we cannot filter it out only + // for this specific slug easily + options.outdir = assetsCacheDir || outdir; // Set write to true so that esbuild will output the files. options.write = true; @@ -609,12 +681,30 @@ export async function getFileBySlug(slug: string): Promise { }, }; + if (assetsCacheDir && cacheFile) { + await cp(assetsCacheDir, outdir, {recursive: true}); + writeCacheFile(cacheFile, JSON.stringify(resultObj)).catch(e => { + // eslint-disable-next-line no-console + console.warn(`Failed to write MDX cache: ${cacheFile}`, e); + }); + } + return resultObj; } +const fileBySlugCache = new Map>(); + /** * Cache the result of {@link getFileBySlug}. * * This is useful for performance when rendering the same file multiple times. */ -export const getFileBySlugWithCache = cache(getFileBySlug); +export function getFileBySlugWithCache(slug: string): Promise { + let cached = fileBySlugCache.get(slug); + if (!cached) { + cached = getFileBySlug(slug); + fileBySlugCache.set(slug, cached); + } + + return cached; +}