Skip to content

Commit 9ce08af

Browse files
BYKBYK
committed
limit concurrency and try again
1 parent 49c83e1 commit 9ce08af

File tree

4 files changed

+153
-138
lines changed

4 files changed

+153
-138
lines changed

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@
7878
"next-plausible": "^3.12.4",
7979
"next-themes": "^0.3.0",
8080
"nextjs-toploader": "^1.6.6",
81+
"p-limit": "^6.2.0",
8182
"platformicons": "^8.0.4",
8283
"prism-sentry": "^1.0.2",
8384
"query-string": "^6.13.1",

‎src/files.ts‎

Lines changed: 5 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,13 @@
1-
import fs from 'fs';
1+
import {readdir} from 'fs/promises';
22
import path from 'path';
33

4-
// pipe two functions together
5-
function pipe<T, U, V>(f: (x: T) => U, g: (y: U) => V): (x: T) => V;
6-
// pipe three functions
7-
function pipe<T, U, V, W>(f: (x: T) => U, g: (y: U) => V, h: (z: V) => W): (x: T) => W;
8-
function pipe(...fns: Function[]) {
9-
return x => fns.reduce((v, f) => f(v), x);
10-
}
11-
12-
const map =
13-
<T, U>(fn: (a: T) => U) =>
14-
(input: T[]) =>
15-
input.map(fn);
16-
17-
const walkDir = (fullPath: string) => {
18-
return fs.statSync(fullPath).isFile() ? fullPath : getAllFilesRecursively(fullPath);
19-
};
20-
21-
const pathJoinPrefix = (prefix: string) => (extraPath: string) =>
22-
path.join(prefix, extraPath);
23-
244
/**
255
* @returns Array of file paths
266
*/
27-
const getAllFilesRecursively = (folder: string): [string] => {
28-
return pipe(
29-
// yes, this arrow function is necessary to narrow down the readdirSync overload
30-
(x: string) => fs.readdirSync(x),
31-
map(pipe(pathJoinPrefix(folder), walkDir)),
32-
// flattenArray
33-
x => x.flat(Infinity)
34-
)(folder) as [string];
7+
const getAllFilesRecursively = async (folder: string): Promise<string[]> => {
8+
return (await readdir(folder, {withFileTypes: true, recursive: true}))
9+
.filter(dirent => dirent.isFile())
10+
.map(dirent => path.join(dirent.parentPath || dirent.path, dirent.name));
3511
};
3612

3713
export default getAllFilesRecursively;

‎src/mdx.ts‎

Lines changed: 135 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import {
1717
createBrotliCompress,
1818
createBrotliDecompress,
1919
} from 'node:zlib';
20+
import {limitFunction} from 'p-limit';
2021
import rehypeAutolinkHeadings from 'rehype-autolink-headings';
2122
import rehypePresetMinify from 'rehype-preset-minify';
2223
import rehypePrismDiff from 'rehype-prism-diff';
@@ -53,6 +54,7 @@ type SlugFile = {
5354
};
5455

5556
const root = process.cwd();
57+
const FILE_CONCURRENCY_LIMIT = 200;
5658
const CACHE_COMPRESS_LEVEL = 4;
5759
const CACHE_DIR = path.join(root, '.next', 'cache', 'mdx-bundler');
5860
mkdirSync(CACHE_DIR, {recursive: true});
@@ -176,51 +178,60 @@ async function getDocsFrontMatterUncached(): Promise<FrontMatter[]> {
176178
export async function getDevDocsFrontMatter(): Promise<FrontMatter[]> {
177179
const folder = 'develop-docs';
178180
const docsPath = path.join(root, folder);
179-
const files = getAllFilesRecursively(docsPath);
181+
const files = await getAllFilesRecursively(docsPath);
180182
const fmts = (
181183
await Promise.all(
182-
files.map(async file => {
183-
const fileName = file.slice(docsPath.length + 1);
184-
if (path.extname(fileName) !== '.md' && path.extname(fileName) !== '.mdx') {
185-
return undefined;
186-
}
187-
188-
const source = await readFile(file, 'utf8');
189-
const {data: frontmatter} = matter(source);
190-
return {
191-
...(frontmatter as FrontMatter),
192-
slug: fileName.replace(/\/index.mdx?$/, '').replace(/\.mdx?$/, ''),
193-
sourcePath: path.join(folder, fileName),
194-
};
195-
})
184+
files.map(
185+
limitFunction(
186+
async file => {
187+
const fileName = file.slice(docsPath.length + 1);
188+
if (path.extname(fileName) !== '.md' && path.extname(fileName) !== '.mdx') {
189+
return undefined;
190+
}
191+
192+
const source = await readFile(file, 'utf8');
193+
const {data: frontmatter} = matter(source);
194+
return {
195+
...(frontmatter as FrontMatter),
196+
slug: fileName.replace(/\/index.mdx?$/, '').replace(/\.mdx?$/, ''),
197+
sourcePath: path.join(folder, fileName),
198+
};
199+
},
200+
{concurrency: FILE_CONCURRENCY_LIMIT}
201+
)
202+
)
196203
)
197204
).filter(isNotNil);
198205
return fmts;
199206
}
200207

201208
async function getAllFilesFrontMatter(): Promise<FrontMatter[]> {
202209
const docsPath = path.join(root, 'docs');
203-
const files = getAllFilesRecursively(docsPath);
210+
const files = await getAllFilesRecursively(docsPath);
204211
const allFrontMatter: FrontMatter[] = [];
212+
205213
await Promise.all(
206-
files.map(async file => {
207-
const fileName = file.slice(docsPath.length + 1);
208-
if (path.extname(fileName) !== '.md' && path.extname(fileName) !== '.mdx') {
209-
return;
210-
}
214+
files.map(
215+
limitFunction(async file => {
216+
const fileName = file.slice(docsPath.length + 1);
217+
if (path.extname(fileName) !== '.md' && path.extname(fileName) !== '.mdx') {
218+
return;
219+
}
211220

212-
if (fileName.indexOf('/common/') !== -1) {
213-
return;
214-
}
221+
if (fileName.indexOf('/common/') !== -1) {
222+
return;
223+
}
215224

216-
const source = await readFile(file, 'utf8');
217-
const {data: frontmatter} = matter(source);
218-
allFrontMatter.push({
219-
...(frontmatter as FrontMatter),
220-
slug: formatSlug(fileName),
221-
sourcePath: path.join('docs', fileName),
222-
});
223-
})
225+
const source = await readFile(file, 'utf8');
226+
const {data: frontmatter} = matter(source);
227+
allFrontMatter.push({
228+
...(frontmatter as FrontMatter),
229+
slug: formatSlug(fileName),
230+
sourcePath: path.join('docs', fileName),
231+
});
232+
},
233+
{concurrency: FILE_CONCURRENCY_LIMIT})
234+
)
224235
);
225236

226237
// Add all `common` files in the right place.
@@ -251,46 +262,56 @@ async function getAllFilesFrontMatter(): Promise<FrontMatter[]> {
251262
continue;
252263
}
253264

254-
const commonFileNames: string[] = getAllFilesRecursively(commonPath).filter(
265+
const commonFileNames: string[] = (await getAllFilesRecursively(commonPath)).filter(
255266
p => path.extname(p) === '.mdx'
256267
);
257268

258269
const commonFiles = await Promise.all(
259-
commonFileNames.map(async commonFileName => {
260-
const source = await readFile(commonFileName, 'utf8');
261-
const {data: frontmatter} = matter(source);
262-
return {commonFileName, frontmatter: frontmatter as FrontMatter};
263-
})
270+
commonFileNames.map(
271+
limitFunction(
272+
async commonFileName => {
273+
const source = await readFile(commonFileName, 'utf8');
274+
const {data: frontmatter} = matter(source);
275+
return {commonFileName, frontmatter: frontmatter as FrontMatter};
276+
},
277+
{concurrency: FILE_CONCURRENCY_LIMIT}
278+
)
279+
)
264280
);
265281

266282
await Promise.all(
267-
commonFiles.map(async f => {
268-
if (!isSupported(f.frontmatter, platformName)) {
269-
return;
270-
}
271-
272-
const subpath = f.commonFileName.slice(commonPath.length + 1);
273-
const slug = f.commonFileName
274-
.slice(docsPath.length + 1)
275-
.replace(/\/common\//, '/');
276-
const noFrontMatter = (
277-
await Promise.allSettled([
278-
access(path.join(docsPath, slug)),
279-
access(path.join(docsPath, slug.replace('/index.mdx', '.mdx'))),
280-
])
281-
).every(r => r.status === 'rejected');
282-
if (noFrontMatter) {
283-
let frontmatter = f.frontmatter;
284-
if (subpath === 'index.mdx') {
285-
frontmatter = {...frontmatter, ...platformFrontmatter};
286-
}
287-
allFrontMatter.push({
288-
...frontmatter,
289-
slug: formatSlug(slug),
290-
sourcePath: 'docs/' + f.commonFileName.slice(docsPath.length + 1),
291-
});
292-
}
293-
})
283+
commonFiles.map(
284+
limitFunction(
285+
async f => {
286+
if (!isSupported(f.frontmatter, platformName)) {
287+
return;
288+
}
289+
290+
const subpath = f.commonFileName.slice(commonPath.length + 1);
291+
const slug = f.commonFileName
292+
.slice(docsPath.length + 1)
293+
.replace(/\/common\//, '/');
294+
const noFrontMatter = (
295+
await Promise.allSettled([
296+
access(path.join(docsPath, slug)),
297+
access(path.join(docsPath, slug.replace('/index.mdx', '.mdx'))),
298+
])
299+
).every(r => r.status === 'rejected');
300+
if (noFrontMatter) {
301+
let frontmatter = f.frontmatter;
302+
if (subpath === 'index.mdx') {
303+
frontmatter = {...frontmatter, ...platformFrontmatter};
304+
}
305+
allFrontMatter.push({
306+
...frontmatter,
307+
slug: formatSlug(slug),
308+
sourcePath: 'docs/' + f.commonFileName.slice(docsPath.length + 1),
309+
});
310+
}
311+
},
312+
{concurrency: FILE_CONCURRENCY_LIMIT}
313+
)
314+
)
294315
);
295316

296317
const guidesPath = path.join(docsPath, 'platforms', platformName, 'guides');
@@ -319,30 +340,41 @@ async function getAllFilesFrontMatter(): Promise<FrontMatter[]> {
319340
}
320341

321342
await Promise.all(
322-
commonFiles.map(async f => {
323-
if (!isSupported(f.frontmatter, platformName, guideName)) {
324-
return;
325-
}
326-
327-
const subpath = f.commonFileName.slice(commonPath.length + 1);
328-
const slug = path.join('platforms', platformName, 'guides', guideName, subpath);
329-
try {
330-
await access(path.join(docsPath, slug));
331-
return;
332-
} catch {
333-
// pass
334-
}
335-
336-
let frontmatter = f.frontmatter;
337-
if (subpath === 'index.mdx') {
338-
frontmatter = {...frontmatter, ...guideFrontmatter};
339-
}
340-
allFrontMatter.push({
341-
...frontmatter,
342-
slug: formatSlug(slug),
343-
sourcePath: 'docs/' + f.commonFileName.slice(docsPath.length + 1),
344-
});
345-
})
343+
commonFiles.map(
344+
limitFunction(
345+
async f => {
346+
if (!isSupported(f.frontmatter, platformName, guideName)) {
347+
return;
348+
}
349+
350+
const subpath = f.commonFileName.slice(commonPath.length + 1);
351+
const slug = path.join(
352+
'platforms',
353+
platformName,
354+
'guides',
355+
guideName,
356+
subpath
357+
);
358+
try {
359+
await access(path.join(docsPath, slug));
360+
return;
361+
} catch {
362+
// pass
363+
}
364+
365+
let frontmatter = f.frontmatter;
366+
if (subpath === 'index.mdx') {
367+
frontmatter = {...frontmatter, ...guideFrontmatter};
368+
}
369+
allFrontMatter.push({
370+
...frontmatter,
371+
slug: formatSlug(slug),
372+
sourcePath: 'docs/' + f.commonFileName.slice(docsPath.length + 1),
373+
});
374+
},
375+
{concurrency: FILE_CONCURRENCY_LIMIT}
376+
)
377+
)
346378
);
347379
}
348380
}
@@ -475,21 +507,17 @@ export async function getFileBySlug(slug: string): Promise<SlugFile> {
475507
);
476508
}
477509

478-
let cacheKey: string | null = null;
479-
let cacheFile: string | null = null;
480-
if (process.env.CI === '1') {
481-
cacheKey = md5(source);
482-
cacheFile = path.join(CACHE_DIR, cacheKey);
510+
const cacheKey = md5(source);
511+
const cacheFile = path.join(CACHE_DIR, cacheKey);
483512

484-
try {
485-
const cached = await readCacheFile<SlugFile>(cacheFile);
486-
return cached;
487-
} catch (err) {
488-
if (err.code !== 'ENOENT' && err.code !== 'ABORT_ERR') {
489-
// If cache is corrupted, ignore and proceed
490-
// eslint-disable-next-line no-console
491-
console.warn(`Failed to read MDX cache: ${cacheFile}`, err);
492-
}
513+
try {
514+
const cached = await readCacheFile<SlugFile>(cacheFile);
515+
return cached;
516+
} catch (err) {
517+
if (err.code !== 'ENOENT' && err.code !== 'ABORT_ERR') {
518+
// If cache is corrupted, ignore and proceed
519+
// eslint-disable-next-line no-console
520+
console.warn(`Failed to read MDX cache: ${cacheFile}`, err);
493521
}
494522
}
495523

@@ -618,12 +646,10 @@ export async function getFileBySlug(slug: string): Promise<SlugFile> {
618646
},
619647
};
620648

621-
if (cacheFile) {
622-
writeCacheFile(cacheFile, JSON.stringify(resultObj)).catch(e => {
623-
// eslint-disable-next-line no-console
624-
console.warn(`Failed to write MDX cache: ${cacheFile}`, e);
625-
});
626-
}
649+
writeCacheFile(cacheFile, JSON.stringify(resultObj)).catch(e => {
650+
// eslint-disable-next-line no-console
651+
console.warn(`Failed to write MDX cache: ${cacheFile}`, e);
652+
});
627653

628654
return resultObj;
629655
}

‎yarn.lock‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10044,6 +10044,13 @@ p-limit@^3.0.1, p-limit@^3.0.2, p-limit@^3.1.0:
1004410044
dependencies:
1004510045
yocto-queue "^0.1.0"
1004610046

10047+
p-limit@^6.2.0:
10048+
version "6.2.0"
10049+
resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-6.2.0.tgz#c254d22ba6aeef441a3564c5e6c2f2da59268a0f"
10050+
integrity sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA==
10051+
dependencies:
10052+
yocto-queue "^1.1.1"
10053+
1004710054
p-locate@^4.1.0:
1004810055
version "4.1.0"
1004910056
resolved "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz"
@@ -13054,6 +13061,11 @@ yocto-queue@^0.1.0:
1305413061
resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz"
1305513062
integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==
1305613063

13064+
yocto-queue@^1.1.1:
13065+
version "1.2.1"
13066+
resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.2.1.tgz#36d7c4739f775b3cbc28e6136e21aa057adec418"
13067+
integrity sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg==
13068+
1305713069
zod@^3.22.4:
1305813070
version "3.23.8"
1305913071
resolved "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz"

0 commit comments

Comments
 (0)