Add concurrency limiting utility function to files module

Co-authored-by: shannon.anahata <[email protected]>

Author: cursoragent

src/files.ts

@@ -1,5 +1,6 @@
 import {readdir} from 'fs/promises';
 import path from 'path';
+import pLimit from 'p-limit';
 
 /**
  * @returns Array of file paths
@@ -10,4 +11,18 @@ const getAllFilesRecursively = async (folder: string): Promise<string[]> => {
     .map(dirent => path.join(dirent.parentPath || dirent.path, dirent.name));
 };
 
+/**
+ * Utility function to limit concurrency of async operations
+ * @param fn - The async function to limit
+ * @param options - Options including concurrency limit
+ * @returns A limited version of the function
+ */
+export function limitFunction<T extends (...args: any[]) => Promise<any>>(
+  fn: T,
+  options: {concurrency: number}
+): T {
+  const limit = pLimit(options.concurrency);
+  return ((...args: Parameters<T>) => limit(() => fn(...args))) as T;
+}
+
 export default getAllFilesRecursively;

src/mdx.ts

@@ -17,7 +17,6 @@ import {
   createBrotliCompress,
   createBrotliDecompress,
 } from 'node:zlib';
-import {limitFunction} from 'p-limit';
 import rehypeAutolinkHeadings from 'rehype-autolink-headings';
 import rehypePresetMinify from 'rehype-preset-minify';
 import rehypePrismDiff from 'rehype-prism-diff';
@@ -28,7 +27,7 @@ import remarkMdxImages from 'remark-mdx-images';
 import getAppRegistry from './build/appRegistry';
 import getPackageRegistry from './build/packageRegistry';
 import {apiCategories} from './build/resolveOpenAPI';
-import getAllFilesRecursively from './files';
+import getAllFilesRecursively, {limitFunction} from './files';
 import remarkDefList from './mdx-deflist';
 import rehypeOnboardingLines from './rehype-onboarding-lines';
 import rehypeSlug from './rehype-slug.js';