diff --git a/bazel/api-golden/BUILD.bazel b/bazel/api-golden/BUILD.bazel index f3a86c15f..f0cd3fc42 100644 --- a/bazel/api-golden/BUILD.bazel +++ b/bazel/api-golden/BUILD.bazel @@ -20,6 +20,7 @@ ts_project( deps = [ "//bazel:node_modules/@microsoft/api-extractor", "//bazel:node_modules/@types/node", + "//bazel:node_modules/piscina", "//bazel:node_modules/typescript", ], ) diff --git a/bazel/api-golden/index_npm_packages.ts b/bazel/api-golden/index_npm_packages.ts index b8a6c1cdf..ccda4508d 100644 --- a/bazel/api-golden/index_npm_packages.ts +++ b/bazel/api-golden/index_npm_packages.ts @@ -12,6 +12,7 @@ import {normalizePathToPosix} from './path-normalize.js'; import {readFileSync} from 'fs'; import {testApiGolden} from './test_api_report.js'; import * as fs from 'fs'; +import {Piscina} from 'piscina'; /** Interface describing contents of a `package.json`. */ export interface PackageJson { @@ -40,11 +41,11 @@ async function main( const packageJsonPath = path.join(npmPackageDir, 'package.json'); const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf8')) as PackageJson; const entryPoints = findEntryPointsWithinNpmPackage(npmPackageDir, packageJson); - const outdatedGoldens: string[] = []; - - let allTestsSucceeding = true; + const worker = new Piscina, string>({ + filename: path.resolve(__dirname, './test_api_report.js'), + }); - for (const {subpath, typesEntryPointPath} of entryPoints) { + const processEntryPoint = async (subpath: string, typesEntryPointPath: string) => { // API extractor generates API reports as markdown files. For each types // entry-point we maintain a separate golden file. These golden files are // based on the name of the defining NodeJS exports subpath in the NPM package, @@ -53,13 +54,15 @@ async function main( const goldenFilePath = path.join(goldenDir, goldenName); const moduleName = normalizePathToPosix(path.join(packageJson.name, subpath)); - const actual = await testApiGolden( + // Run API extractor in child processes. This is because API extractor is very + // synchronous. This allows us to significantly speed up golden testing. + const actual = await worker.run([ typesEntryPointPath, stripExportPattern, typeNames, packageJsonPath, moduleName, - ); + ]); if (actual === null) { console.error(`Could not generate API golden for subpath: "${subpath}". See errors above.`); @@ -67,18 +70,40 @@ async function main( } if (approveGolden) { - fs.mkdirSync(path.dirname(goldenFilePath), {recursive: true}); - fs.writeFileSync(goldenFilePath, actual, 'utf8'); + await fs.promises.mkdir(path.dirname(goldenFilePath), {recursive: true}); + await fs.promises.writeFile(goldenFilePath, actual, 'utf8'); } else { - const expected = fs.readFileSync(goldenFilePath, 'utf8'); + const expected = await fs.promises.readFile(goldenFilePath, 'utf8'); if (actual !== expected) { // Keep track of outdated goldens for error message. outdatedGoldens.push(goldenName); - allTestsSucceeding = false; + return false; } } + + return true; + }; + + const outdatedGoldens: string[] = []; + const tasks: Promise[] = []; + // Process in batches. Otherwise we risk out of memory errors. + const batchSize = 10; + + for (let i = 0; i < entryPoints.length; i += batchSize) { + const batchEntryPoints = entryPoints.slice(i, i + batchSize); + + for (const {subpath, typesEntryPointPath} of batchEntryPoints) { + tasks.push(processEntryPoint(subpath, typesEntryPointPath)); + } + + // Wait for new batch. + await Promise.all(tasks); } + // Wait for final batch/retrieve all results. + const results = await Promise.all(tasks); + const allTestsSucceeding = results.every((r) => r === true); + if (outdatedGoldens.length) { console.error(chalk.red(`The following goldens are outdated:`)); outdatedGoldens.forEach((name) => console.info(`- ${name}`)); diff --git a/bazel/api-golden/test_api_report.ts b/bazel/api-golden/test_api_report.ts index 6fa215a5d..40e8d6cb8 100644 --- a/bazel/api-golden/test_api_report.ts +++ b/bazel/api-golden/test_api_report.ts @@ -55,7 +55,8 @@ export async function testApiGolden( customPackageName: string, ): Promise { const tempDir = - process.env.TEST_TMPDIR ?? fs.mkdtempSync(path.join(os.tmpdir(), 'api-golden-rule')); + process.env.TEST_TMPDIR ?? + (await fs.promises.mkdtemp(path.join(os.tmpdir(), 'api-golden-rule'))); const rjsMode = process.env['RJS_MODE'] === 'true'; let resolvedTypePackages: Awaited> | null = null; @@ -144,8 +145,8 @@ export async function testApiGolden( if (!result.succeeded) { return null; } - const reportOut = fs.readFileSync(reportTmpOutPath, 'utf8'); - fs.rmSync(reportTmpOutPath); + const reportOut = await fs.promises.readFile(reportTmpOutPath, 'utf8'); + await fs.promises.rm(reportTmpOutPath); return reportOut; } @@ -167,7 +168,6 @@ async function processExtractorMessage(message: ExtractorMessage) { } } -/** Resolves the `package.json` of the workspace executing this action. */ -function resolveWorkspacePackageJsonPath(): string { - return path.resolve(`./package.json`); +export default function (args: Parameters) { + return testApiGolden(...args); } diff --git a/bazel/package.json b/bazel/package.json index 79b1fbd26..2e5a14fde 100644 --- a/bazel/package.json +++ b/bazel/package.json @@ -2,8 +2,9 @@ "name": "@devinfra/bazel", "dependencies": { "@microsoft/api-extractor": "7.52.2", - "typescript": "5.8.2", - "@types/node": "22.14.0" + "@types/node": "22.14.0", + "piscina": "^4.9.2", + "typescript": "5.8.2" }, "pnpm": { "onlyBuiltDependencies": [] diff --git a/bazel/pnpm-lock.yaml b/bazel/pnpm-lock.yaml index ae85b31d0..3577571f0 100644 --- a/bazel/pnpm-lock.yaml +++ b/bazel/pnpm-lock.yaml @@ -14,6 +14,9 @@ importers: '@types/node': specifier: 22.14.0 version: 22.14.0 + piscina: + specifier: ^4.9.2 + version: 4.9.2 typescript: specifier: 5.8.2 version: 5.8.2 @@ -33,6 +36,106 @@ packages: '@microsoft/tsdoc@0.15.1': resolution: {integrity: sha512-4aErSrCR/On/e5G2hDP0wjooqDdauzEbIq8hIkIe5pXV0rtWJZvdCEKL0ykZxex+IxIwBp0eGeV48hQN07dXtw==} + '@napi-rs/nice-android-arm-eabi@1.0.1': + resolution: {integrity: sha512-5qpvOu5IGwDo7MEKVqqyAxF90I6aLj4n07OzpARdgDRfz8UbBztTByBp0RC59r3J1Ij8uzYi6jI7r5Lws7nn6w==} + engines: {node: '>= 10'} + cpu: [arm] + os: [android] + + '@napi-rs/nice-android-arm64@1.0.1': + resolution: {integrity: sha512-GqvXL0P8fZ+mQqG1g0o4AO9hJjQaeYG84FRfZaYjyJtZZZcMjXW5TwkL8Y8UApheJgyE13TQ4YNUssQaTgTyvA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@napi-rs/nice-darwin-arm64@1.0.1': + resolution: {integrity: sha512-91k3HEqUl2fsrz/sKkuEkscj6EAj3/eZNCLqzD2AA0TtVbkQi8nqxZCZDMkfklULmxLkMxuUdKe7RvG/T6s2AA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@napi-rs/nice-darwin-x64@1.0.1': + resolution: {integrity: sha512-jXnMleYSIR/+TAN/p5u+NkCA7yidgswx5ftqzXdD5wgy/hNR92oerTXHc0jrlBisbd7DpzoaGY4cFD7Sm5GlgQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@napi-rs/nice-freebsd-x64@1.0.1': + resolution: {integrity: sha512-j+iJ/ezONXRQsVIB/FJfwjeQXX7A2tf3gEXs4WUGFrJjpe/z2KB7sOv6zpkm08PofF36C9S7wTNuzHZ/Iiccfw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [freebsd] + + '@napi-rs/nice-linux-arm-gnueabihf@1.0.1': + resolution: {integrity: sha512-G8RgJ8FYXYkkSGQwywAUh84m946UTn6l03/vmEXBYNJxQJcD+I3B3k5jmjFG/OPiU8DfvxutOP8bi+F89MCV7Q==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@napi-rs/nice-linux-arm64-gnu@1.0.1': + resolution: {integrity: sha512-IMDak59/W5JSab1oZvmNbrms3mHqcreaCeClUjwlwDr0m3BoR09ZiN8cKFBzuSlXgRdZ4PNqCYNeGQv7YMTjuA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/nice-linux-arm64-musl@1.0.1': + resolution: {integrity: sha512-wG8fa2VKuWM4CfjOjjRX9YLIbysSVV1S3Kgm2Fnc67ap/soHBeYZa6AGMeR5BJAylYRjnoVOzV19Cmkco3QEPw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/nice-linux-ppc64-gnu@1.0.1': + resolution: {integrity: sha512-lxQ9WrBf0IlNTCA9oS2jg/iAjQyTI6JHzABV664LLrLA/SIdD+I1i3Mjf7TsnoUbgopBcCuDztVLfJ0q9ubf6Q==} + engines: {node: '>= 10'} + cpu: [ppc64] + os: [linux] + + '@napi-rs/nice-linux-riscv64-gnu@1.0.1': + resolution: {integrity: sha512-3xs69dO8WSWBb13KBVex+yvxmUeEsdWexxibqskzoKaWx9AIqkMbWmE2npkazJoopPKX2ULKd8Fm9veEn0g4Ig==} + engines: {node: '>= 10'} + cpu: [riscv64] + os: [linux] + + '@napi-rs/nice-linux-s390x-gnu@1.0.1': + resolution: {integrity: sha512-lMFI3i9rlW7hgToyAzTaEybQYGbQHDrpRkg+1gJWEpH0PLAQoZ8jiY0IzakLfNWnVda1eTYYlxxFYzW8Rqczkg==} + engines: {node: '>= 10'} + cpu: [s390x] + os: [linux] + + '@napi-rs/nice-linux-x64-gnu@1.0.1': + resolution: {integrity: sha512-XQAJs7DRN2GpLN6Fb+ZdGFeYZDdGl2Fn3TmFlqEL5JorgWKrQGRUrpGKbgZ25UeZPILuTKJ+OowG2avN8mThBA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/nice-linux-x64-musl@1.0.1': + resolution: {integrity: sha512-/rodHpRSgiI9o1faq9SZOp/o2QkKQg7T+DK0R5AkbnI/YxvAIEHf2cngjYzLMQSQgUhxym+LFr+UGZx4vK4QdQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/nice-win32-arm64-msvc@1.0.1': + resolution: {integrity: sha512-rEcz9vZymaCB3OqEXoHnp9YViLct8ugF+6uO5McifTedjq4QMQs3DHz35xBEGhH3gJWEsXMUbzazkz5KNM5YUg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@napi-rs/nice-win32-ia32-msvc@1.0.1': + resolution: {integrity: sha512-t7eBAyPUrWL8su3gDxw9xxxqNwZzAqKo0Szv3IjVQd1GpXXVkb6vBBQUuxfIYaXMzZLwlxRQ7uzM2vdUE9ULGw==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + + '@napi-rs/nice-win32-x64-msvc@1.0.1': + resolution: {integrity: sha512-JlF+uDcatt3St2ntBG8H02F1mM45i5SF9W+bIKiReVE6wiy3o16oBP/yxt+RZ+N6LbCImJXJ6bXNO2kn9AXicg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@napi-rs/nice@1.0.1': + resolution: {integrity: sha512-zM0mVWSXE0a0h9aKACLwKmD6nHcRiKrPpCfvaKqG1CqDEyjEawId0ocXxVzPMCAm6kkWr2P025msfxXEnt8UGQ==} + engines: {node: '>= 10'} + '@rushstack/node-core-library@5.13.0': resolution: {integrity: sha512-IGVhy+JgUacAdCGXKUrRhwHMTzqhWwZUI+qEPcdzsb80heOw0QPbhhoVsoiMF7Klp8eYsp7hzpScMXmOa3Uhfg==} peerDependencies: @@ -146,6 +249,9 @@ packages: path-parse@1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + piscina@4.9.2: + resolution: {integrity: sha512-Fq0FERJWFEUpB4eSY59wSNwXD4RYqR+nR/WiEVcZW8IWfVBxJJafcgTEZDQo8k3w0sUarJ8RyVbbUF4GQ2LGbQ==} + punycode@2.3.1: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} @@ -242,6 +348,74 @@ snapshots: '@microsoft/tsdoc@0.15.1': {} + '@napi-rs/nice-android-arm-eabi@1.0.1': + optional: true + + '@napi-rs/nice-android-arm64@1.0.1': + optional: true + + '@napi-rs/nice-darwin-arm64@1.0.1': + optional: true + + '@napi-rs/nice-darwin-x64@1.0.1': + optional: true + + '@napi-rs/nice-freebsd-x64@1.0.1': + optional: true + + '@napi-rs/nice-linux-arm-gnueabihf@1.0.1': + optional: true + + '@napi-rs/nice-linux-arm64-gnu@1.0.1': + optional: true + + '@napi-rs/nice-linux-arm64-musl@1.0.1': + optional: true + + '@napi-rs/nice-linux-ppc64-gnu@1.0.1': + optional: true + + '@napi-rs/nice-linux-riscv64-gnu@1.0.1': + optional: true + + '@napi-rs/nice-linux-s390x-gnu@1.0.1': + optional: true + + '@napi-rs/nice-linux-x64-gnu@1.0.1': + optional: true + + '@napi-rs/nice-linux-x64-musl@1.0.1': + optional: true + + '@napi-rs/nice-win32-arm64-msvc@1.0.1': + optional: true + + '@napi-rs/nice-win32-ia32-msvc@1.0.1': + optional: true + + '@napi-rs/nice-win32-x64-msvc@1.0.1': + optional: true + + '@napi-rs/nice@1.0.1': + optionalDependencies: + '@napi-rs/nice-android-arm-eabi': 1.0.1 + '@napi-rs/nice-android-arm64': 1.0.1 + '@napi-rs/nice-darwin-arm64': 1.0.1 + '@napi-rs/nice-darwin-x64': 1.0.1 + '@napi-rs/nice-freebsd-x64': 1.0.1 + '@napi-rs/nice-linux-arm-gnueabihf': 1.0.1 + '@napi-rs/nice-linux-arm64-gnu': 1.0.1 + '@napi-rs/nice-linux-arm64-musl': 1.0.1 + '@napi-rs/nice-linux-ppc64-gnu': 1.0.1 + '@napi-rs/nice-linux-riscv64-gnu': 1.0.1 + '@napi-rs/nice-linux-s390x-gnu': 1.0.1 + '@napi-rs/nice-linux-x64-gnu': 1.0.1 + '@napi-rs/nice-linux-x64-musl': 1.0.1 + '@napi-rs/nice-win32-arm64-msvc': 1.0.1 + '@napi-rs/nice-win32-ia32-msvc': 1.0.1 + '@napi-rs/nice-win32-x64-msvc': 1.0.1 + optional: true + '@rushstack/node-core-library@5.13.0(@types/node@22.14.0)': dependencies: ajv: 8.13.0 @@ -363,6 +537,10 @@ snapshots: path-parse@1.0.7: {} + piscina@4.9.2: + optionalDependencies: + '@napi-rs/nice': 1.0.1 + punycode@2.3.1: {} require-from-string@2.0.2: {}