diff --git a/next.config.ts b/next.config.ts index 56fb66502edf1b..3ab0fb515b6808 100644 --- a/next.config.ts +++ b/next.config.ts @@ -5,22 +5,38 @@ import {redirects} from './redirects.js'; const outputFileTracingExcludes = process.env.NEXT_PUBLIC_DEVELOPER_DOCS ? { - '/**/*': ['./.git/**/*', './apps/**/*', 'docs/**/*'], + '/**/*': [ + '**/*.map', + './.git/**/*', + './apps/**/*', + './.next/cache/mdx-bundler/**/*', + './.next/cache/md-exports/**/*', + 'docs/**/*', + ], } : { '/**/*': [ + '**/*.map', './.git/**/*', + './.next/cache/mdx-bundler/**/*', + './.next/cache/md-exports/**/*', './apps/**/*', 'develop-docs/**/*', - 'node_modules/@esbuild/darwin-arm64', + 'node_modules/@esbuild/*', ], - '/platform-redirect': ['**/*.gif', 'public/mdx-images/**/*', '*.pdf'], + '/platform-redirect': ['**/*.gif', 'public/mdx-images/**/*', '**/*.pdf'], '\\[\\[\\.\\.\\.path\\]\\]': [ 'docs/**/*', 'node_modules/prettier/plugins', 'node_modules/rollup/dist', ], - 'sitemap.xml': ['docs/**/*', 'public/mdx-images/**/*', '*.gif', '*.pdf', '*.png'], + 'sitemap.xml': [ + 'docs/**/*', + 'public/mdx-images/**/*', + '**/*.gif', + '**/*.pdf', + '**/*.png', + ], }; if ( diff --git a/scripts/generate-md-exports.mjs b/scripts/generate-md-exports.mjs index 9299d9ef5611ca..2d3ddece84e74f 100644 --- a/scripts/generate-md-exports.mjs +++ b/scripts/generate-md-exports.mjs @@ -283,9 +283,7 @@ async function processTaskList({id, tasks, cacheDir, noCache}) { const fileHash = md5(data); if (r2Hash !== fileHash) { r2CacheMisses.push(relativePath); - console.log( - `📤 Worker[${id}]: Uploading ${relativePath} to R2, hash mismatch: ${r2Hash} !== ${fileHash}` - ); + await uploadToCFR2(s3Client, relativePath, data); } } @@ -296,7 +294,7 @@ async function processTaskList({id, tasks, cacheDir, noCache}) { const success = tasks.length - failedTasks.length; if (r2CacheMisses.length / tasks.length > 0.1) { console.warn( - `⚠️ Worker[${id}]: More than 10% of files had a different hash on R2, this might indicate a problem with the cache or the generation process.` + `⚠️ Worker[${id}]: More than 10% of files had a different hash on R2 with the generation process.` ); } else if (r2CacheMisses.length > 0) { console.log( diff --git a/src/mdx.ts b/src/mdx.ts index 9336d7f341372e..f9c363d713a774 100644 --- a/src/mdx.ts +++ b/src/mdx.ts @@ -1,22 +1,10 @@ -import {BinaryLike, createHash} from 'crypto'; - import {cache} from 'react'; import matter from 'gray-matter'; import {s} from 'hastscript'; import yaml from 'js-yaml'; import {bundleMDX} from 'mdx-bundler'; -import {createReadStream, createWriteStream, mkdirSync} from 'node:fs'; import {access, opendir, readFile} from 'node:fs/promises'; import path from 'node:path'; -// @ts-expect-error ts(2305) -- For some reason "compose" is not recognized in the types -import {compose, Readable} from 'node:stream'; -import {json} from 'node:stream/consumers'; -import {pipeline} from 'node:stream/promises'; -import { - constants as zlibConstants, - createBrotliCompress, - createBrotliDecompress, -} from 'node:zlib'; import {limitFunction} from 'p-limit'; import rehypeAutolinkHeadings from 'rehype-autolink-headings'; import rehypePresetMinify from 'rehype-preset-minify'; @@ -60,33 +48,6 @@ const root = process.cwd(); // Functions which looks like AWS Lambda and we get `EMFILE` errors when trying to open // so many files at once. const FILE_CONCURRENCY_LIMIT = 200; -const CACHE_COMPRESS_LEVEL = 4; -const CACHE_DIR = path.join(root, '.next', 'cache', 'mdx-bundler'); -mkdirSync(CACHE_DIR, {recursive: true}); - -const md5 = (data: BinaryLike) => createHash('md5').update(data).digest('hex'); - -async function readCacheFile(file: string): Promise { - const reader = createReadStream(file); - const decompressor = createBrotliDecompress(); - - return (await json(compose(reader, decompressor))) as T; -} - -async function writeCacheFile(file: string, data: string) { - await pipeline( - Readable.from(data), - createBrotliCompress({ - chunkSize: 32 * 1024, - params: { - [zlibConstants.BROTLI_PARAM_MODE]: zlibConstants.BROTLI_MODE_TEXT, - [zlibConstants.BROTLI_PARAM_QUALITY]: CACHE_COMPRESS_LEVEL, - [zlibConstants.BROTLI_PARAM_SIZE_HINT]: data.length, - }, - }), - createWriteStream(file) - ); -} function formatSlug(slug: string) { return slug.replace(/\.(mdx|md)/, ''); @@ -523,20 +484,6 @@ export async function getFileBySlug(slug: string): Promise { ); } - const cacheKey = md5(source); - const cacheFile = path.join(CACHE_DIR, cacheKey); - - try { - const cached = await readCacheFile(cacheFile); - return cached; - } catch (err) { - if (err.code !== 'ENOENT' && err.code !== 'ABORT_ERR') { - // If cache is corrupted, ignore and proceed - // eslint-disable-next-line no-console - console.warn(`Failed to read MDX cache: ${cacheFile}`, err); - } - } - process.env.ESBUILD_BINARY_PATH = path.join( root, 'node_modules', @@ -662,11 +609,6 @@ export async function getFileBySlug(slug: string): Promise { }, }; - writeCacheFile(cacheFile, JSON.stringify(resultObj)).catch(e => { - // eslint-disable-next-line no-console - console.warn(`Failed to write MDX cache: ${cacheFile}`, e); - }); - return resultObj; }