perf: replace p-queue with evan/concurrency (#143)

* perf: replace p-queue with evan/concurrency

* refactor: unify queue error handling

Author: ayuhito

bun.lockb

@@ -1,7 +1,7 @@
 {
 	"name": "google-font-metadata",
 	"description": "A metadata generator for Google Fonts.",
-	"version": "6.0.2",
+	"version": "6.0.3",
 	"author": "Ayuhito <[email protected]>",
 	"main": "./dist/index.js",
 	"module": "./dist/index.mjs",
@@ -25,14 +25,13 @@
 		"unicode range"
 	],
 	"dependencies": {
+		"@evan/concurrency": "^0.0.3",
 		"@octokit/core": "^6.1.2",
-		"@types/stylis": "^4.2.7",
 		"cac": "^6.7.14",
 		"consola": "^3.3.3",
 		"deepmerge": "^4.3.1",
 		"json-stringify-pretty-compact": "^4.0.0",
 		"linkedom": "^0.18.6",
-		"p-queue": "^8.0.1",
 		"pathe": "^1.1.2",
 		"picocolors": "^1.1.1",
 		"playwright": "^1.49.1",
@@ -41,9 +40,9 @@
 	},
 	"devDependencies": {
 		"@biomejs/biome": "1.9.4",
-		"@playwright/test": "^1.49.1",
 		"@types/bun": "latest",
 		"@types/node": "^22.10.2",
+		"@types/stylis": "^4.2.7",
 		"c8": "^10.1.3",
 		"magic-string": "^0.30.17",
 		"msw": "^2.7.0",
@@ -58,8 +57,7 @@
 		"test": "vitest",
 		"test:generate-fixtures": "bun run ./tests/utils/generate-css-fixtures",
 		"coverage": "vitest --coverage",
-		"format": "biome format --fix",
-		"lint": "biome lint --fix",
+		"lint": "biome check --fix",
 		"prepublishOnly": "bunx biome ci && bun run build"
 	},
 	"files": ["dist/*", "data/*"],

package.json

@@ -1,19 +1,23 @@
 import * as fs from 'node:fs/promises';
 import { fileURLToPath } from 'node:url';
 
+import { Limiter } from '@evan/concurrency';
 import { consola } from 'consola';
 import stringify from 'json-stringify-pretty-compact';
-import PQueue from 'p-queue';
 import { dirname, join } from 'pathe';
 import { compile } from 'stylis';
 
 import { apiv1 as userAgents } from '../data/user-agents.json';
 import { APIDirect, APIv1 } from './data';
+import { LOOP_LIMIT, addError, checkErrors } from './errors';
 import type { APIResponse, FontObjectV1 } from './types';
 import { orderObject, weightListGen } from './utils';
 import { validate } from './validate';
 
 const baseurl = 'https://fonts.googleapis.com/css?subset=';
+const queue = Limiter(18);
+
+const results: FontObjectV1[] = [];
 
 export const fetchCSS = async (
 	font: APIResponse,
@@ -49,10 +53,10 @@ export const fetchCSS = async (
 	return cssMap.join('');
 };
 
+// Download CSS stylesheets for each file format
 export const fetchAllCSS = async (
 	font: APIResponse,
 ): Promise<[string, string, string]> =>
-	// Download CSS stylesheets for each file format
 	await Promise.all([
 		fetchCSS(font, userAgents.woff2),
 		fetchCSS(font, userAgents.woff),
@@ -176,69 +180,63 @@ export const processCSS = (
 	return fontObject;
 };
 
-const results: FontObjectV1[] = [];
-
-const processQueue = async (font: APIResponse, force: boolean) => {
-	const id = font.family.replaceAll(/\s/g, '-').toLowerCase();
-
-	// If last-modified matches latest API, skip fetching CSS and processing.
-	if (
-		APIv1[id] !== undefined &&
-		font.lastModified === APIv1[id].lastModified &&
-		!force
-	) {
-		results.push({ [id]: APIv1[id] });
-	} else {
-		const css = await fetchAllCSS(font);
-		const fontObject = processCSS(css, font);
-		results.push(fontObject);
-		consola.info(`Updated ${id}`);
+const processQueue = async (
+	font: APIResponse,
+	force: boolean,
+): Promise<void> => {
+	try {
+		const id = font.family.replaceAll(/\s/g, '-').toLowerCase();
+
+		// If last-modified matches latest API, skip fetching CSS and processing.
+		if (
+			APIv1[id] !== undefined &&
+			font.lastModified === APIv1[id].lastModified &&
+			!force
+		) {
+			results.push({ [id]: APIv1[id] });
+		} else {
+			const css = await fetchAllCSS(font);
+			const fontObject = processCSS(css, font);
+			results.push(fontObject);
+			consola.info(`Updated ${id}`);
+		}
+		consola.success(`Parsed ${id}`);
+	} catch (error) {
+		addError(`${font.family} experienced an error. ${String(error)}`);
 	}
-	consola.success(`Parsed ${id}`);
 };
 
-// Queue control
-const queue = new PQueue({ concurrency: 18 });
-
-// @ts-ignore - rollup-plugin-dts fails to compile this typing
-queue.on('error', (error: Error) => {
-	consola.error(error);
-});
-
 /**
  * Parses the fetched API data and writes it to the APIv1 JSON dataset.
  * @param force - Force update all fonts without using cache.
  * @param noValidate - Skip automatic validation of generated data.
  */
 export const parsev1 = async (force: boolean, noValidate: boolean) => {
 	for (const font of APIDirect) {
-		try {
-			queue.add(async () => {
-				await processQueue(font, force);
-			});
-		} catch (error) {
-			throw new Error(`${font.family} experienced an error. ${String(error)}`);
-		}
+		checkErrors(LOOP_LIMIT);
+		queue.add(() => processQueue(font, force));
 	}
-	await queue.onIdle().then(async () => {
-		// Order the font objects alphabetically for consistency and not create huge diffs
-		const unordered: FontObjectV1 = Object.assign({}, ...results);
-		const ordered = orderObject(unordered);
 
-		if (!noValidate) {
-			validate('v1', ordered);
-		}
+	await queue.flush();
+	checkErrors();
 
-		await fs.writeFile(
-			join(
-				dirname(fileURLToPath(import.meta.url)),
-				'../data/google-fonts-v1.json',
-			),
-			stringify(ordered),
-		);
-
-		consola.success(
-			`All ${results.length} font datapoints using CSS APIv1 have been generated.`,
-		);
-	});
+	// Order the font objects alphabetically for consistency and not create huge diffs
+	const unordered: FontObjectV1 = Object.assign({}, ...results);
+	const ordered = orderObject(unordered);
+
+	if (!noValidate) {
+		validate('v1', ordered);
+	}
+
+	await fs.writeFile(
+		join(
+			dirname(fileURLToPath(import.meta.url)),
+			'../data/google-fonts-v1.json',
+		),
+		stringify(ordered),
+	);
+
+	consola.success(
+		`All ${results.length} font datapoints using CSS APIv1 have been generated.`,
+	);
 };

src/api-parser-v1.ts

@@ -1,19 +1,23 @@
 import * as fs from 'node:fs/promises';
 import { fileURLToPath } from 'node:url';
 
+import { Limiter } from '@evan/concurrency';
 import { consola } from 'consola';
 import stringify from 'json-stringify-pretty-compact';
-import PQueue from 'p-queue';
 import { dirname, join } from 'pathe';
 import { compile } from 'stylis';
 
 import { apiv2 as userAgents } from '../data/user-agents.json';
 import { APIDirect, APIv2 } from './data';
+import { LOOP_LIMIT, addError, checkErrors } from './errors';
 import type { APIResponse, FontObjectV2 } from './types';
 import { orderObject, weightListGen } from './utils';
 import { validate } from './validate';
 
 const baseurl = 'https://fonts.googleapis.com/css2?family=';
+const queue = Limiter(18);
+
+const results: FontObjectV2[] = [];
 
 export const fetchCSS = async (
 	fontFamily: string,
@@ -234,69 +238,63 @@ export const processCSS = (
 	return fontObject;
 };
 
-const results: FontObjectV2[] = [];
-
-const processQueue = async (font: APIResponse, force: boolean) => {
-	const id = font.family.replaceAll(/\s/g, '-').toLowerCase();
-
-	// If last-modified matches latest API, skip fetching CSS and processing.
-	if (
-		APIv2[id] !== undefined &&
-		font.lastModified === APIv2[id].lastModified &&
-		!force
-	) {
-		results.push({ [id]: APIv2[id] });
-	} else {
-		const css = await fetchAllCSS(font);
-		const fontObject = processCSS(css, font);
-		results.push(fontObject);
-		consola.info(`Updated ${id}`);
+const processQueue = async (
+	font: APIResponse,
+	force: boolean,
+): Promise<void> => {
+	try {
+		const id = font.family.replaceAll(/\s/g, '-').toLowerCase();
+
+		// If last-modified matches latest API, skip fetching CSS and processing.
+		if (
+			APIv2[id] !== undefined &&
+			font.lastModified === APIv2[id].lastModified &&
+			!force
+		) {
+			results.push({ [id]: APIv2[id] });
+		} else {
+			const css = await fetchAllCSS(font);
+			const fontObject = processCSS(css, font);
+			results.push(fontObject);
+			consola.info(`Updated ${id}`);
+		}
+		consola.success(`Parsed ${id}`);
+	} catch (error) {
+		addError(`${font.family} experienced an error. ${String(error)}`);
 	}
-	consola.success(`Parsed ${id}`);
 };
 
-// Queue control
-const queue = new PQueue({ concurrency: 18 });
-
-// @ts-ignore - rollup-plugin-dts fails to compile this typing
-queue.on('error', (error: Error) => {
-	consola.error(error);
-});
-
 /**
  * Parses the fetched API and writes it to the APIv2 dataset.
  * @param force - Force update all fonts without using cache.
  * @param noValidate - Skip automatic validation of generated data.
  */
 export const parsev2 = async (force: boolean, noValidate: boolean) => {
 	for (const font of APIDirect) {
-		try {
-			queue.add(async () => {
-				await processQueue(font, force);
-			});
-		} catch (error) {
-			throw new Error(`${font.family} experienced an error. ${String(error)}`);
-		}
+		checkErrors(LOOP_LIMIT);
+		queue.add(() => processQueue(font, force));
 	}
-	await queue.onIdle().then(async () => {
-		// Order the font objects alphabetically for consistency and not create huge diffs
-		const unordered: FontObjectV2 = Object.assign({}, ...results);
-		const ordered = orderObject(unordered);
 
-		if (!noValidate) {
-			validate('v2', ordered);
-		}
+	await queue.flush();
+	checkErrors();
 
-		await fs.writeFile(
-			join(
-				dirname(fileURLToPath(import.meta.url)),
-				'../data/google-fonts-v2.json',
-			),
-			stringify(ordered),
-		);
+	// Order the font objects alphabetically for consistency and not create huge diffs
+	const unordered: FontObjectV2 = Object.assign({}, ...results);
+	const ordered = orderObject(unordered);
 
-		consola.success(
-			`All ${results.length} font datapoints using CSS APIv2 have been generated.`,
-		);
-	});
+	if (!noValidate) {
+		validate('v2', ordered);
+	}
+
+	await fs.writeFile(
+		join(
+			dirname(fileURLToPath(import.meta.url)),
+			'../data/google-fonts-v2.json',
+		),
+		stringify(ordered),
+	);
+
+	consola.success(
+		`All ${results.length} font datapoints using CSS APIv2 have been generated.`,
+	);
 };

src/api-parser-v2.ts

@@ -0,0 +1,23 @@
+import consola from 'consola';
+
+const errs: string[] = [];
+
+export const LOOP_LIMIT = 5;
+
+export const addError = (error: string) => {
+	errs.push(error);
+};
+
+export const checkErrors = (limit = 0) => {
+	if (errs.length > limit) {
+		for (const err of errs) {
+			consola.error(err);
+		}
+
+		if (limit > 0) {
+			throw new Error('Too many errors occurred during parsing. Stopping...');
+		}
+
+		throw new Error('Some fonts experienced errors during parsing.');
+	}
+};