diff --git a/.babelrc.js.bak b/.babelrc.js.bak deleted file mode 100644 index 68d19cf2a5d8a..0000000000000 --- a/.babelrc.js.bak +++ /dev/null @@ -1,16 +0,0 @@ -/* eslint-env node */ -/* eslint import/no-nodejs-modules:0 */ - -let ignore = [`**/dist`]; - -// Jest needs to compile this code, but generally we don't want this copied -// to output folders -if (process.env.NODE_ENV !== `test`) { - ignore.push(`**/__tests__`); -} - -module.exports = { - sourceMaps: true, - presets: [], - ignore, -}; diff --git a/app/sitemap.ts b/app/sitemap.ts index 89498d034b61c..173989e0a6fae 100644 --- a/app/sitemap.ts +++ b/app/sitemap.ts @@ -5,7 +5,7 @@ import {getDevDocsFrontMatter, getDocsFrontMatter} from 'sentry-docs/mdx'; export default async function sitemap(): Promise { if (isDeveloperDocs) { - const docs = getDevDocsFrontMatter(); + const docs = await getDevDocsFrontMatter(); const baseUrl = 'https://develop.sentry.dev'; return docsToSitemap(docs, baseUrl); } diff --git a/docs/product/explore/session-replay/web/index.mdx b/docs/product/explore/session-replay/web/index.mdx index 4d8d3495964b7..0b3230193360f 100644 --- a/docs/product/explore/session-replay/web/index.mdx +++ b/docs/product/explore/session-replay/web/index.mdx @@ -4,8 +4,6 @@ sidebar_order: 10 description: "Learn about Session Replay and its video-like reproductions of user interactions, which can help you see when users are frustrated and build a better web experience." --- - - Session Replay allows you to see video-like reproductions of user sessions which can help you understand what happened before, during, and after an error or performance issue occurred. You'll be able to gain deeper debugging context into issues so that you can reproduce and resolve problems faster without the guesswork. As you play back each session, you'll be able to see every user interaction in relation to network requests, DOM events, and console messages. It’s effectively like having [DevTools](https://developer.chrome.com/docs/devtools/overview/) active in your production user sessions. Replays are integrated with other parts of the Sentry product so you can see how the user experience is impacted by errors and slow transactions. You'll see session replays associated with error events on the [Issue Details](/product/issues/issue-details/) page, and those associated with slow transactions on the [Transaction Summary](/product/insights/overview/transaction-summary/) page. For [backend error replays](/product/explore/session-replay/web/getting-started/#replays-for-backend-errors), any contributing backend errors will be included in the replay's timeline, [breadcrumbs](https://docs.sentry.io/product/issues/issue-details/breadcrumbs/), and errors. diff --git a/docs/product/sentry-basics/performance-monitoring.mdx b/docs/product/sentry-basics/performance-monitoring.mdx index 1131df0e8cf4c..5da714f8f777b 100644 --- a/docs/product/sentry-basics/performance-monitoring.mdx +++ b/docs/product/sentry-basics/performance-monitoring.mdx @@ -4,8 +4,6 @@ sidebar_order: 1 description: "Understand and monitor how your application performs in production. Track key metrics, analyze bottlenecks, and resolve performance issues with distributed tracing, detailed transaction data, and automated issue detection." --- - - In many tools, Performance Monitoring is just about tracking a few key metrics on your web pages. Sentry takes a different approach. By setting up [Tracing](/concepts/key-terms/tracing/), Sentry captures detailed performance data for every transaction in your entire application stack and automatically presents it in a variety of easy-to-use but powerful features so you can rapidly identify and resolve performance issues as they happen - all in one place. diff --git a/package.json b/package.json index 203edf3cd2e4a..1c80d5af6f92e 100644 --- a/package.json +++ b/package.json @@ -78,6 +78,7 @@ "next-plausible": "^3.12.4", "next-themes": "^0.3.0", "nextjs-toploader": "^1.6.6", + "p-limit": "^6.2.0", "platformicons": "^8.0.4", "prism-sentry": "^1.0.2", "query-string": "^6.13.1", @@ -116,7 +117,7 @@ "@tailwindcss/forms": "^0.5.7", "@tailwindcss/typography": "^0.5.10", "@types/dompurify": "3.0.5", - "@types/node": "^20", + "@types/node": "^22", "@types/react": "18.3.12", "@types/react-dom": "18.3.1", "@types/ws": "^8.5.10", @@ -140,10 +141,11 @@ }, "resolutions": { "dompurify": "3.2.4", - "@types/dompurify": "3.0.5" + "@types/dompurify": "3.0.5", + "@types/node": "^22" }, "volta": { - "node": "20.11.0", + "node": "22.16.0", "yarn": "1.22.22" } } diff --git a/platform-includes/sourcemaps/overview/javascript.cloudflare.mdx b/platform-includes/sourcemaps/overview/javascript.cloudflare.mdx index e4ffaf08ecadf..4d7502d2cacb4 100644 --- a/platform-includes/sourcemaps/overview/javascript.cloudflare.mdx +++ b/platform-includes/sourcemaps/overview/javascript.cloudflare.mdx @@ -19,9 +19,7 @@ If you want to configure source maps to upload manually, follow the guide for yo ### Guides for Source Maps -- - TypeScript (tsc) - +- TypeScript (tsc) If you're using a bundler like Webpack, Vite, Rollup, or Esbuild, use the diff --git a/platform-includes/sourcemaps/upload/primer/javascript.cloudflare.mdx b/platform-includes/sourcemaps/upload/primer/javascript.cloudflare.mdx index c8226f2478d54..cea61cd2f6581 100644 --- a/platform-includes/sourcemaps/upload/primer/javascript.cloudflare.mdx +++ b/platform-includes/sourcemaps/upload/primer/javascript.cloudflare.mdx @@ -7,6 +7,6 @@ If you can't find the tool of your choice in the list below, we recommend you ch - + - \ No newline at end of file + diff --git a/scripts/algolia.ts b/scripts/algolia.ts index 4dc0dcb4c4248..71aa8e041e962 100644 --- a/scripts/algolia.ts +++ b/scripts/algolia.ts @@ -64,9 +64,9 @@ indexAndUpload(); async function indexAndUpload() { // the page front matters are the source of truth for the static doc routes // as they are used directly by generateStaticParams() on [[..path]] page - const pageFrontMatters = isDeveloperDocs + const pageFrontMatters = await (isDeveloperDocs ? getDevDocsFrontMatter() - : await getDocsFrontMatter(); + : getDocsFrontMatter()); const records = await generateAlogliaRecords(pageFrontMatters); console.log('πŸ”₯ Generated %d new Algolia records.', records.length); const existingRecordIds = await fetchExistingRecordIds(index); diff --git a/scripts/generate-md-exports.mjs b/scripts/generate-md-exports.mjs index e50e6ee87e23c..6b99a70e72073 100644 --- a/scripts/generate-md-exports.mjs +++ b/scripts/generate-md-exports.mjs @@ -1,13 +1,20 @@ #!/usr/bin/env node - +/* eslint-disable no-console */ import {selectAll} from 'hast-util-select'; import {createHash} from 'node:crypto'; -import {constants as fsConstants, existsSync} from 'node:fs'; -import {copyFile, mkdir, opendir, readFile, rm, writeFile} from 'node:fs/promises'; +import {createReadStream, createWriteStream, existsSync} from 'node:fs'; +import {mkdir, opendir, readFile, rm} from 'node:fs/promises'; import {cpus} from 'node:os'; import * as path from 'node:path'; +import {Readable} from 'node:stream'; +import {pipeline} from 'node:stream/promises'; import {fileURLToPath} from 'node:url'; import {isMainThread, parentPort, Worker, workerData} from 'node:worker_threads'; +import { + constants as zlibConstants, + createBrotliCompress, + createBrotliDecompress, +} from 'node:zlib'; import rehypeParse from 'rehype-parse'; import rehypeRemark from 'rehype-remark'; import remarkGfm from 'remark-gfm'; @@ -15,14 +22,19 @@ import remarkStringify from 'remark-stringify'; import {unified} from 'unified'; import {remove} from 'unist-util-remove'; +const CACHE_COMPRESS_LEVEL = 4; + function taskFinishHandler(data) { if (data.failedTasks.length === 0) { - console.log(`βœ… Worker[${data.id}]: ${data.success} files successfully.`); - } else { - hasErrors = true; - console.error(`❌ Worker[${data.id}]: ${data.failedTasks.length} files failed:`); - console.error(data.failedTasks); + console.log( + `πŸ’° Worker[${data.id}]: Cache hits: ${data.cacheHits} (${Math.round((data.cacheHits / data.success) * 100)}%)` + ); + console.log(`βœ… Worker[${data.id}]: converted ${data.success} files successfully.`); + return false; } + console.error(`❌ Worker[${data.id}]: ${data.failedTasks.length} files failed:`); + console.error(data.failedTasks); + return true; } async function createWork() { @@ -37,13 +49,6 @@ async function createWork() { const INPUT_DIR = path.join(root, '.next', 'server', 'app'); const OUTPUT_DIR = path.join(root, 'public', 'md-exports'); - const CACHE_VERSION = 1; - const CACHE_DIR = path.join(root, '.next', 'cache', 'md-exports', `v${CACHE_VERSION}`); - const noCache = !existsSync(CACHE_DIR); - if (noCache) { - await mkdir(CACHE_DIR, {recursive: true}); - } - console.log(`πŸš€ Starting markdown generation from: ${INPUT_DIR}`); console.log(`πŸ“ Output directory: ${OUTPUT_DIR}`); @@ -51,6 +56,14 @@ async function createWork() { await rm(OUTPUT_DIR, {recursive: true, force: true}); await mkdir(OUTPUT_DIR, {recursive: true}); + const CACHE_DIR = path.join(root, '.next', 'cache', 'md-exports'); + console.log(`πŸ’° Cache directory: ${CACHE_DIR}`); + const noCache = !existsSync(CACHE_DIR); + if (noCache) { + console.log(`ℹ️ No cache directory found, this will take a while...`); + await mkdir(CACHE_DIR, {recursive: true}); + } + // On a 16-core machine, 8 workers were optimal (and slightly faster than 16) const numWorkers = Math.max(Math.floor(cpus().length / 2), 2); const workerTasks = new Array(numWorkers).fill(null).map(() => []); @@ -86,7 +99,7 @@ async function createWork() { workerData: {id, noCache, cacheDir: CACHE_DIR, tasks: workerTasks[id]}, }); let hasErrors = false; - worker.on('message', taskFinishHandler); + worker.on('message', data => (hasErrors = taskFinishHandler(data))); worker.on('error', reject); worker.on('exit', code => { if (code !== 0) { @@ -104,7 +117,11 @@ async function createWork() { cacheDir: CACHE_DIR, tasks: workerTasks[workerTasks.length - 1], id: workerTasks.length - 1, - }).then(taskFinishHandler) + }).then(data => { + if (taskFinishHandler(data)) { + throw new Error(`Worker[${data.id}] had some errors.`); + } + }) ); await Promise.all(workerPromises); @@ -116,54 +133,85 @@ async function createWork() { const md5 = data => createHash('md5').update(data).digest('hex'); async function genMDFromHTML(source, target, {cacheDir, noCache}) { - const text = await readFile(source, {encoding: 'utf8'}); + const text = (await readFile(source, {encoding: 'utf8'})) + // Remove all script tags, as they are not needed in markdown + // and they are not stable across builds, causing cache misses + .replace(/]*>[\s\S]*?<\/script>/gi, ''); const hash = md5(text); const cacheFile = path.join(cacheDir, hash); if (!noCache) { try { - await copyFile(cacheFile, target, fsConstants.COPYFILE_FICLONE); - return; + await pipeline( + createReadStream(cacheFile), + createBrotliDecompress(), + createWriteStream(target, { + encoding: 'utf8', + }) + ); + + return true; } catch { // pass } } - await writeFile( - target, - String( - await unified() - .use(rehypeParse) - // Need the `main div > hgroup` selector for the headers - .use(() => tree => selectAll('main div > hgroup, div#main', tree)) - // If we don't do this wrapping, rehypeRemark just returns an empty string -- yeah WTF? - .use(() => tree => ({ - type: 'element', - tagName: 'div', - properties: {}, - children: tree, - })) - .use(rehypeRemark, { - document: false, - handlers: { - // Remove buttons as they usually get confusing in markdown, especially since we use them as tab headers - button() {}, - }, - }) - // We end up with empty inline code blocks, probably from some tab logic in the HTML, remove them - .use(() => tree => remove(tree, {type: 'inlineCode', value: ''})) - .use(remarkGfm) - .use(remarkStringify) - .process(text) - ) + const data = String( + await unified() + .use(rehypeParse) + // Need the `main div > hgroup` selector for the headers + .use(() => tree => selectAll('main div > hgroup, div#main', tree)) + // If we don't do this wrapping, rehypeRemark just returns an empty string -- yeah WTF? + .use(() => tree => ({ + type: 'element', + tagName: 'div', + properties: {}, + children: tree, + })) + .use(rehypeRemark, { + document: false, + handlers: { + // Remove buttons as they usually get confusing in markdown, especially since we use them as tab headers + button() {}, + }, + }) + // We end up with empty inline code blocks, probably from some tab logic in the HTML, remove them + .use(() => tree => remove(tree, {type: 'inlineCode', value: ''})) + .use(remarkGfm) + .use(remarkStringify) + .process(text) ); - await copyFile(target, cacheFile, fsConstants.COPYFILE_FICLONE); + const reader = Readable.from(data); + + await Promise.all([ + pipeline( + reader, + createWriteStream(target, { + encoding: 'utf8', + }) + ), + pipeline( + reader, + createBrotliCompress({ + chunkSize: 32 * 1024, + params: { + [zlibConstants.BROTLI_PARAM_MODE]: zlibConstants.BROTLI_MODE_TEXT, + [zlibConstants.BROTLI_PARAM_QUALITY]: CACHE_COMPRESS_LEVEL, + [zlibConstants.BROTLI_PARAM_SIZE_HINT]: data.length, + }, + }), + createWriteStream(cacheFile) + ).catch(err => console.warn('Error writing cache file:', err)), + ]); + + return false; } async function processTaskList({id, tasks, cacheDir, noCache}) { const failedTasks = []; + let cacheHits = 0; for (const {sourcePath, targetPath} of tasks) { try { - await genMDFromHTML(sourcePath, targetPath, { + cacheHits += await genMDFromHTML(sourcePath, targetPath, { cacheDir, noCache, }); @@ -171,7 +219,7 @@ async function processTaskList({id, tasks, cacheDir, noCache}) { failedTasks.push({sourcePath, targetPath, error}); } } - return {id, success: tasks.length - failedTasks.length, failedTasks}; + return {id, success: tasks.length - failedTasks.length, failedTasks, cacheHits}; } async function doWork(work) { diff --git a/src/docTree.ts b/src/docTree.ts index 4240071e4ec45..1857c553eb8e6 100644 --- a/src/docTree.ts +++ b/src/docTree.ts @@ -40,7 +40,7 @@ export function getDocsRootNode(): Promise { async function getDocsRootNodeUncached(): Promise { return frontmatterToTree( - isDeveloperDocs ? getDevDocsFrontMatter() : await getDocsFrontMatter() + await (isDeveloperDocs ? getDevDocsFrontMatter() : getDocsFrontMatter()) ); } diff --git a/src/files.ts b/src/files.ts index db46761aa5bd8..ee2a99838cdae 100644 --- a/src/files.ts +++ b/src/files.ts @@ -1,37 +1,13 @@ -import fs from 'fs'; +import {readdir} from 'fs/promises'; import path from 'path'; -// pipe two functions together -function pipe(f: (x: T) => U, g: (y: U) => V): (x: T) => V; -// pipe three functions -function pipe(f: (x: T) => U, g: (y: U) => V, h: (z: V) => W): (x: T) => W; -function pipe(...fns: Function[]) { - return x => fns.reduce((v, f) => f(v), x); -} - -const map = - (fn: (a: T) => U) => - (input: T[]) => - input.map(fn); - -const walkDir = (fullPath: string) => { - return fs.statSync(fullPath).isFile() ? fullPath : getAllFilesRecursively(fullPath); -}; - -const pathJoinPrefix = (prefix: string) => (extraPath: string) => - path.join(prefix, extraPath); - /** * @returns Array of file paths */ -const getAllFilesRecursively = (folder: string): [string] => { - return pipe( - // yes, this arrow function is necessary to narrow down the readdirSync overload - (x: string) => fs.readdirSync(x), - map(pipe(pathJoinPrefix(folder), walkDir)), - // flattenArray - x => x.flat(Infinity) - )(folder) as [string]; +const getAllFilesRecursively = async (folder: string): Promise => { + return (await readdir(folder, {withFileTypes: true, recursive: true})) + .filter(dirent => dirent.isFile()) + .map(dirent => path.join(dirent.parentPath || dirent.path, dirent.name)); }; export default getAllFilesRecursively; diff --git a/src/mdx.ts b/src/mdx.ts index d7dcc5ecda78d..9336d7f341372 100644 --- a/src/mdx.ts +++ b/src/mdx.ts @@ -1,11 +1,23 @@ -import fs from 'fs'; -import path from 'path'; +import {BinaryLike, createHash} from 'crypto'; import {cache} from 'react'; import matter from 'gray-matter'; import {s} from 'hastscript'; import yaml from 'js-yaml'; import {bundleMDX} from 'mdx-bundler'; +import {createReadStream, createWriteStream, mkdirSync} from 'node:fs'; +import {access, opendir, readFile} from 'node:fs/promises'; +import path from 'node:path'; +// @ts-expect-error ts(2305) -- For some reason "compose" is not recognized in the types +import {compose, Readable} from 'node:stream'; +import {json} from 'node:stream/consumers'; +import {pipeline} from 'node:stream/promises'; +import { + constants as zlibConstants, + createBrotliCompress, + createBrotliDecompress, +} from 'node:zlib'; +import {limitFunction} from 'p-limit'; import rehypeAutolinkHeadings from 'rehype-autolink-headings'; import rehypePresetMinify from 'rehype-preset-minify'; import rehypePrismDiff from 'rehype-prism-diff'; @@ -32,7 +44,49 @@ import {FrontMatter, Platform, PlatformConfig} from './types'; import {isNotNil} from './utils'; import {isVersioned, VERSION_INDICATOR} from './versioning'; +type SlugFile = { + frontMatter: Platform & {slug: string}; + matter: Omit, 'data'> & { + data: Platform; + }; + mdxSource: string; + toc: TocNode[]; +}; + const root = process.cwd(); +// We need to limit this as we have code doing things like Promise.all(allFiles.map(...)) +// where `allFiles` is in the order of thousands. This not only slows down the build but +// it also crashes the dynamic pages such as `/platform-redirect` as these run on Vercel +// Functions which looks like AWS Lambda and we get `EMFILE` errors when trying to open +// so many files at once. +const FILE_CONCURRENCY_LIMIT = 200; +const CACHE_COMPRESS_LEVEL = 4; +const CACHE_DIR = path.join(root, '.next', 'cache', 'mdx-bundler'); +mkdirSync(CACHE_DIR, {recursive: true}); + +const md5 = (data: BinaryLike) => createHash('md5').update(data).digest('hex'); + +async function readCacheFile(file: string): Promise { + const reader = createReadStream(file); + const decompressor = createBrotliDecompress(); + + return (await json(compose(reader, decompressor))) as T; +} + +async function writeCacheFile(file: string, data: string) { + await pipeline( + Readable.from(data), + createBrotliCompress({ + chunkSize: 32 * 1024, + params: { + [zlibConstants.BROTLI_PARAM_MODE]: zlibConstants.BROTLI_MODE_TEXT, + [zlibConstants.BROTLI_PARAM_QUALITY]: CACHE_COMPRESS_LEVEL, + [zlibConstants.BROTLI_PARAM_SIZE_HINT]: data.length, + }, + }), + createWriteStream(file) + ); +} function formatSlug(slug: string) { return slug.replace(/\.(mdx|md)/, ''); @@ -64,10 +118,9 @@ const isSupported = ( let getDocsFrontMatterCache: Promise | undefined; export function getDocsFrontMatter(): Promise { - if (getDocsFrontMatterCache) { - return getDocsFrontMatterCache; + if (!getDocsFrontMatterCache) { + getDocsFrontMatterCache = getDocsFrontMatterUncached(); } - getDocsFrontMatterCache = getDocsFrontMatterUncached(); return getDocsFrontMatterCache; } @@ -92,7 +145,7 @@ export const getVersionsFromDoc = (frontMatter: FrontMatter[], docPath: string) }; async function getDocsFrontMatterUncached(): Promise { - const frontMatter = getAllFilesFrontMatter(); + const frontMatter = await getAllFilesFrontMatter(); const categories = await apiCategories(); categories.forEach(category => { @@ -127,142 +180,220 @@ async function getDocsFrontMatterUncached(): Promise { return frontMatter; } -export function getDevDocsFrontMatter(): FrontMatter[] { +export async function getDevDocsFrontMatterUncached(): Promise { const folder = 'develop-docs'; const docsPath = path.join(root, folder); - const files = getAllFilesRecursively(docsPath); - const fmts = files - .map(file => { - const fileName = file.slice(docsPath.length + 1); - if (path.extname(fileName) !== '.md' && path.extname(fileName) !== '.mdx') { - return undefined; - } + const files = await getAllFilesRecursively(docsPath); + const frontMatters = ( + await Promise.all( + files.map( + limitFunction( + async file => { + const fileName = file.slice(docsPath.length + 1); + if (path.extname(fileName) !== '.md' && path.extname(fileName) !== '.mdx') { + return undefined; + } + + const source = await readFile(file, 'utf8'); + const {data: frontmatter} = matter(source); + return { + ...(frontmatter as FrontMatter), + slug: fileName.replace(/\/index.mdx?$/, '').replace(/\.mdx?$/, ''), + sourcePath: path.join(folder, fileName), + }; + }, + {concurrency: FILE_CONCURRENCY_LIMIT} + ) + ) + ) + ).filter(isNotNil); + return frontMatters; +} - const source = fs.readFileSync(file, 'utf8'); - const {data: frontmatter} = matter(source); - return { - ...(frontmatter as FrontMatter), - slug: fileName.replace(/\/index.mdx?$/, '').replace(/\.mdx?$/, ''), - sourcePath: path.join(folder, fileName), - }; - }) - .filter(isNotNil); - return fmts; +let getDevDocsFrontMatterCache: Promise | undefined; + +export function getDevDocsFrontMatter(): Promise { + if (!getDevDocsFrontMatterCache) { + getDevDocsFrontMatterCache = getDevDocsFrontMatterUncached(); + } + return getDevDocsFrontMatterCache; } -function getAllFilesFrontMatter() { +async function getAllFilesFrontMatter(): Promise { const docsPath = path.join(root, 'docs'); - const files = getAllFilesRecursively(docsPath); + const files = await getAllFilesRecursively(docsPath); const allFrontMatter: FrontMatter[] = []; - files.forEach(file => { - const fileName = file.slice(docsPath.length + 1); - if (path.extname(fileName) !== '.md' && path.extname(fileName) !== '.mdx') { - return; - } - if (fileName.indexOf('/common/') !== -1) { - return; - } + await Promise.all( + files.map( + limitFunction( + async file => { + const fileName = file.slice(docsPath.length + 1); + if (path.extname(fileName) !== '.md' && path.extname(fileName) !== '.mdx') { + return; + } - const source = fs.readFileSync(file, 'utf8'); - const {data: frontmatter} = matter(source); - allFrontMatter.push({ - ...(frontmatter as FrontMatter), - slug: formatSlug(fileName), - sourcePath: path.join('docs', fileName), - }); - }); + if (fileName.indexOf('/common/') !== -1) { + return; + } + + const source = await readFile(file, 'utf8'); + const {data: frontmatter} = matter(source); + allFrontMatter.push({ + ...(frontmatter as FrontMatter), + slug: formatSlug(fileName), + sourcePath: path.join('docs', fileName), + }); + }, + {concurrency: FILE_CONCURRENCY_LIMIT} + ) + ) + ); // Add all `common` files in the right place. const platformsPath = path.join(docsPath, 'platforms'); - const platformNames = fs - .readdirSync(platformsPath) - .filter(p => !fs.statSync(path.join(platformsPath, p)).isFile()); - platformNames.forEach(platformName => { + for await (const platform of await opendir(platformsPath)) { + if (platform.isFile()) { + continue; + } + const platformName = platform.name; + let platformFrontmatter: PlatformConfig = {}; const configPath = path.join(platformsPath, platformName, 'config.yml'); - if (fs.existsSync(configPath)) { + try { platformFrontmatter = yaml.load( - fs.readFileSync(configPath, 'utf8') + await readFile(configPath, 'utf8') ) as PlatformConfig; + } catch (err) { + // the file may not exist and that's fine, for anything else we throw + if (err.code !== 'ENOENT') { + throw err; + } } const commonPath = path.join(platformsPath, platformName, 'common'); - if (!fs.existsSync(commonPath)) { - return; + try { + await access(commonPath); + } catch (err) { + continue; } - const commonFileNames: string[] = getAllFilesRecursively(commonPath).filter( + const commonFileNames: string[] = (await getAllFilesRecursively(commonPath)).filter( p => path.extname(p) === '.mdx' ); - const commonFiles = commonFileNames.map(commonFileName => { - const source = fs.readFileSync(commonFileName, 'utf8'); - const {data: frontmatter} = matter(source); - return {commonFileName, frontmatter: frontmatter as FrontMatter}; - }); - commonFiles.forEach(f => { - if (!isSupported(f.frontmatter, platformName)) { - return; - } + const commonFiles = await Promise.all( + commonFileNames.map( + limitFunction( + async commonFileName => { + const source = await readFile(commonFileName, 'utf8'); + const {data: frontmatter} = matter(source); + return {commonFileName, frontmatter: frontmatter as FrontMatter}; + }, + {concurrency: FILE_CONCURRENCY_LIMIT} + ) + ) + ); - const subpath = f.commonFileName.slice(commonPath.length + 1); - const slug = f.commonFileName.slice(docsPath.length + 1).replace(/\/common\//, '/'); - if ( - !fs.existsSync(path.join(docsPath, slug)) && - !fs.existsSync(path.join(docsPath, slug.replace('/index.mdx', '.mdx'))) - ) { - let frontmatter = f.frontmatter; - if (subpath === 'index.mdx') { - frontmatter = {...frontmatter, ...platformFrontmatter}; - } - allFrontMatter.push({ - ...frontmatter, - slug: formatSlug(slug), - sourcePath: 'docs/' + f.commonFileName.slice(docsPath.length + 1), - }); - } - }); + await Promise.all( + commonFiles.map( + limitFunction( + async f => { + if (!isSupported(f.frontmatter, platformName)) { + return; + } + + const subpath = f.commonFileName.slice(commonPath.length + 1); + const slug = f.commonFileName + .slice(docsPath.length + 1) + .replace(/\/common\//, '/'); + const noFrontMatter = ( + await Promise.allSettled([ + access(path.join(docsPath, slug)), + access(path.join(docsPath, slug.replace('/index.mdx', '.mdx'))), + ]) + ).every(r => r.status === 'rejected'); + if (noFrontMatter) { + let frontmatter = f.frontmatter; + if (subpath === 'index.mdx') { + frontmatter = {...frontmatter, ...platformFrontmatter}; + } + allFrontMatter.push({ + ...frontmatter, + slug: formatSlug(slug), + sourcePath: 'docs/' + f.commonFileName.slice(docsPath.length + 1), + }); + } + }, + {concurrency: FILE_CONCURRENCY_LIMIT} + ) + ) + ); const guidesPath = path.join(docsPath, 'platforms', platformName, 'guides'); - let guideNames: string[] = []; - if (!fs.existsSync(guidesPath)) { - return; + try { + await access(guidesPath); + } catch (err) { + continue; } - guideNames = fs - .readdirSync(guidesPath) - .filter(g => !fs.statSync(path.join(guidesPath, g)).isFile()); - guideNames.forEach(guideName => { + + for await (const guide of await opendir(guidesPath)) { + if (guide.isFile()) { + continue; + } + const guideName = guide.name; + let guideFrontmatter: FrontMatter | null = null; const guideConfigPath = path.join(guidesPath, guideName, 'config.yml'); - if (fs.existsSync(guideConfigPath)) { + try { guideFrontmatter = yaml.load( - fs.readFileSync(guideConfigPath, 'utf8') + await readFile(guideConfigPath, 'utf8') ) as FrontMatter; - } - - commonFiles.forEach(f => { - if (!isSupported(f.frontmatter, platformName, guideName)) { - return; - } - - const subpath = f.commonFileName.slice(commonPath.length + 1); - const slug = path.join('platforms', platformName, 'guides', guideName, subpath); - if (!fs.existsSync(path.join(docsPath, slug))) { - let frontmatter = f.frontmatter; - if (subpath === 'index.mdx') { - frontmatter = {...frontmatter, ...guideFrontmatter}; - } - allFrontMatter.push({ - ...frontmatter, - slug: formatSlug(slug), - sourcePath: 'docs/' + f.commonFileName.slice(docsPath.length + 1), - }); + } catch (err) { + if (err.code !== 'ENOENT') { + throw err; } - }); - }); - }); + } + await Promise.all( + commonFiles.map( + limitFunction( + async f => { + if (!isSupported(f.frontmatter, platformName, guideName)) { + return; + } + + const subpath = f.commonFileName.slice(commonPath.length + 1); + const slug = path.join( + 'platforms', + platformName, + 'guides', + guideName, + subpath + ); + try { + await access(path.join(docsPath, slug)); + return; + } catch { + // pass + } + + let frontmatter = f.frontmatter; + if (subpath === 'index.mdx') { + frontmatter = {...frontmatter, ...guideFrontmatter}; + } + allFrontMatter.push({ + ...frontmatter, + slug: formatSlug(slug), + sourcePath: 'docs/' + f.commonFileName.slice(docsPath.length + 1), + }); + }, + {concurrency: FILE_CONCURRENCY_LIMIT} + ) + ) + ); + } + } return allFrontMatter; } @@ -299,13 +430,18 @@ export const addVersionToFilePath = (filePath: string, version: string) => { return `${filePath}__v${version}`; }; -export async function getFileBySlug(slug: string) { +export async function getFileBySlug(slug: string): Promise { // no versioning on a config file const configPath = path.join(root, slug.split(VERSION_INDICATOR)[0], 'config.yml'); let configFrontmatter: PlatformConfig | undefined; - if (fs.existsSync(configPath)) { - configFrontmatter = yaml.load(fs.readFileSync(configPath, 'utf8')) as PlatformConfig; + try { + configFrontmatter = yaml.load(await readFile(configPath, 'utf8')) as PlatformConfig; + } catch (err) { + // If the config file does not exist, we can ignore it. + if (err.code !== 'ENOENT') { + throw err; + } } let mdxPath = path.join(root, `${slug}.mdx`); @@ -315,10 +451,14 @@ export async function getFileBySlug(slug: string) { let mdIndexPath = path.join(root, slug, 'index.md'); if ( - slug.indexOf('docs/platforms/') === 0 && - [mdxPath, mdxIndexPath, mdPath, mdIndexPath, versionedMdxIndexPath].filter(p => - fs.existsSync(p) - ).length === 0 + slug.startsWith('docs/platforms/') && + ( + await Promise.allSettled( + [mdxPath, mdxIndexPath, mdPath, mdIndexPath, versionedMdxIndexPath].map(p => + access(p) + ) + ) + ).every(r => r.status === 'rejected') ) { // Try the common folder. const slugParts = slug.split('/'); @@ -334,25 +474,68 @@ export async function getFileBySlug(slug: string) { commonFilePath = path.join(commonPath, slugParts.slice(3).join('/')); versionedMdxIndexPath = getVersionedIndexPath(root, commonFilePath, '.mdx'); } - if (commonFilePath && fs.existsSync(commonPath)) { - mdxPath = path.join(root, `${commonFilePath}.mdx`); - mdxIndexPath = path.join(root, commonFilePath, 'index.mdx'); - mdPath = path.join(root, `${commonFilePath}.md`); - mdIndexPath = path.join(root, commonFilePath, 'index.md'); - versionedMdxIndexPath = getVersionedIndexPath(root, commonFilePath, '.mdx'); + if (commonFilePath) { + try { + await access(commonPath); + mdxPath = path.join(root, `${commonFilePath}.mdx`); + mdxIndexPath = path.join(root, commonFilePath, 'index.mdx'); + mdPath = path.join(root, `${commonFilePath}.md`); + mdIndexPath = path.join(root, commonFilePath, 'index.md'); + versionedMdxIndexPath = getVersionedIndexPath(root, commonFilePath, '.mdx'); + } catch (err) { + // If the common folder does not exist, we can ignore it. + if (err.code !== 'ENOENT') { + throw err; + } + } } } // check if a versioned index file exists - if (isVersioned(slug) && fs.existsSync(mdxIndexPath)) { - mdxIndexPath = addVersionToFilePath(mdxIndexPath, slug.split(VERSION_INDICATOR)[1]); + if (isVersioned(slug)) { + try { + await access(mdxIndexPath); + mdxIndexPath = addVersionToFilePath(mdxIndexPath, slug.split(VERSION_INDICATOR)[1]); + } catch (err) { + // pass, the file does not exist + if (err.code !== 'ENOENT') { + throw err; + } + } } - const sourcePath = - [mdxPath, mdxIndexPath, mdPath, versionedMdxIndexPath].find(fs.existsSync) ?? - mdIndexPath; + let source: string | undefined = undefined; + let sourcePath: string | undefined = undefined; + const sourcePaths = [mdxPath, mdxIndexPath, mdPath, versionedMdxIndexPath, mdIndexPath]; + const errors: Error[] = []; + for (const p of sourcePaths) { + try { + source = await readFile(p, 'utf8'); + sourcePath = p; + break; + } catch (e) { + errors.push(e); + } + } + if (source === undefined || sourcePath === undefined) { + throw new Error( + `Failed to find a valid source file for slug "${slug}". Tried:\n${sourcePaths.join('\n')}\nErrors:\n${errors.map(e => e.message).join('\n')}` + ); + } - const source = fs.readFileSync(sourcePath, 'utf8'); + const cacheKey = md5(source); + const cacheFile = path.join(CACHE_DIR, cacheKey); + + try { + const cached = await readCacheFile(cacheFile); + return cached; + } catch (err) { + if (err.code !== 'ENOENT' && err.code !== 'ABORT_ERR') { + // If cache is corrupted, ignore and proceed + // eslint-disable-next-line no-console + console.warn(`Failed to read MDX cache: ${cacheFile}`, err); + } + } process.env.ESBUILD_BINARY_PATH = path.join( root, @@ -469,7 +652,7 @@ export async function getFileBySlug(slug: string) { mergedFrontmatter = {...frontmatter, ...configFrontmatter}; } - return { + const resultObj: SlugFile = { matter: result.matter, mdxSource: code, toc, @@ -478,6 +661,13 @@ export async function getFileBySlug(slug: string) { slug, }, }; + + writeCacheFile(cacheFile, JSON.stringify(resultObj)).catch(e => { + // eslint-disable-next-line no-console + console.warn(`Failed to write MDX cache: ${cacheFile}`, e); + }); + + return resultObj; } /** diff --git a/yarn.lock b/yarn.lock index cc68e32c1b4f8..1cbea29b29578 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3660,12 +3660,12 @@ dependencies: "@types/node" "*" -"@types/node@*", "@types/node@^20": - version "20.17.6" - resolved "https://registry.npmjs.org/@types/node/-/node-20.17.6.tgz" - integrity sha512-VEI7OdvK2wP7XHnsuXbAJnEpEkF6NjSN45QJlL4VGqZSXsnicpesdTWsg9RISeSdYd3yeRj/y3k5KGjUXYnFwQ== +"@types/node@*", "@types/node@^22": + version "22.15.32" + resolved "https://registry.yarnpkg.com/@types/node/-/node-22.15.32.tgz#c301cc2275b535a5e54bb81d516b1d2e9afe06e5" + integrity sha512-3jigKqgSjsH6gYZv2nEsqdXfZqIFGAV36XYYjf9KGZ3PSG+IhLecqPnI310RvjutyMwifE2hhhNEklOUrvx/wA== dependencies: - undici-types "~6.19.2" + undici-types "~6.21.0" "@types/parse-json@^4.0.0": version "4.0.2" @@ -10044,6 +10044,13 @@ p-limit@^3.0.1, p-limit@^3.0.2, p-limit@^3.1.0: dependencies: yocto-queue "^0.1.0" +p-limit@^6.2.0: + version "6.2.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-6.2.0.tgz#c254d22ba6aeef441a3564c5e6c2f2da59268a0f" + integrity sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA== + dependencies: + yocto-queue "^1.1.1" + p-locate@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz" @@ -12284,10 +12291,10 @@ unbox-primitive@^1.0.2: has-symbols "^1.0.3" which-boxed-primitive "^1.0.2" -undici-types@~6.19.2: - version "6.19.8" - resolved "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz" - integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw== +undici-types@~6.21.0: + version "6.21.0" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.21.0.tgz#691d00af3909be93a7faa13be61b3a5b50ef12cb" + integrity sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ== undici@^5.25.4: version "5.28.5" @@ -13054,6 +13061,11 @@ yocto-queue@^0.1.0: resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== +yocto-queue@^1.1.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.2.1.tgz#36d7c4739f775b3cbc28e6136e21aa057adec418" + integrity sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg== + zod@^3.22.4: version "3.23.8" resolved "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz"