use lru-cache to avoid memory usage, fix embed icon src paths

Author: fennifith

package.json

@@ -63,6 +63,7 @@
 		"@types/hast": "^3.0.4",
 		"@types/html-escaper": "^3.0.0",
 		"@types/json5": "^2.2.0",
+		"@types/lru-cache": "^7.10.10",
 		"@types/mdast": "^4.0.3",
 		"@types/node": "^20.5.0",
 		"@types/uuid": "^10.0.0",
@@ -96,6 +97,7 @@
 		"junk": "^4.0.1",
 		"lint-staged": "^15.2.7",
 		"live-server": "^1.2.2",
+		"lru-cache": "^11.0.2",
 		"msw": "^2.3.5",
 		"npm-run-all": "^4.1.5",
 		"octokit": "^4.0.2",

pnpm-lock.yaml

@@ -1,21 +1,34 @@
 import { Element, Root } from "hast";
 import { fromHtml } from "hast-util-from-html";
 import { find } from "unist-util-find";
+import { LRUCache } from "lru-cache";
+
+export async function fetchAsBrowser(input: string | URL, init?: RequestInit) {
+	const response = await fetch(input, {
+		...init,
+		headers: {
+			"User-Agent":
+				"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+			"Accept-Language": "en",
+			...init?.headers,
+		},
+	});
+	const isSuccess = `${response.status}`.startsWith("2");
+	if (!isSuccess)
+		throw new Error(`Request ${input} returned an error: ${response.status}`);
+	return response;
+}
 
-const pageHtmlMap = new Map<string, Promise<Root | null>>();
+const pageHtmlCache = new LRUCache<string, Promise<Root | null>>({
+	max: 50,
+});
 
 export function fetchPageHtml(src: string): Promise<Root | null> {
-	if (pageHtmlMap.has(src)) return pageHtmlMap.get(src)!;
+	if (pageHtmlCache.has(src)) return pageHtmlCache.get(src)!;
 
 	const promise = (async () => {
-		const srcHTML = await fetch(src, {
-			headers: {
-				"User-Agent":
-					"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
-				"Accept-Language": "en",
-			},
-		})
-			.then((r) => (r.status === 200 ? r.text() : undefined))
+		const srcHTML = await fetchAsBrowser(src)
+			.then(async (r) => await r.text())
 			.catch(() => null);
 
 		// if fetch fails...
@@ -26,7 +39,7 @@ export function fetchPageHtml(src: string): Promise<Root | null> {
 		return srcHast;
 	})();
 
-	pageHtmlMap.set(src, promise);
+	pageHtmlCache.set(src, promise);
 	return promise;
 }
 

src/utils/fetch-page-html.ts

@@ -5,7 +5,6 @@ import { Plugin } from "unified";
 import { visit } from "unist-util-visit";
 
 import { EMBED_MIN_HEIGHT, EMBED_SIZE } from "../constants";
-import { fromHtml } from "hast-util-from-html";
 import { find } from "unist-util-find";
 import { getLargestManifestIcon } from "../../get-largest-manifest-icon";
 import { IFramePlaceholder } from "./iframe-placeholder";
@@ -15,6 +14,7 @@ import * as stream from "stream";
 import sharp from "sharp";
 import * as svgo from "svgo";
 import { fetchPageHtml, getPageTitle } from "utils/fetch-page-html";
+import { LRUCache } from "lru-cache";
 
 interface RehypeUnicornIFrameClickToRunProps {
 	srcReplacements?: Array<(val: string, root: VFile) => string>;
@@ -29,22 +29,28 @@ function getIconPath(src: URL) {
 
 // Cache the fetch *promises* - so that only one request per manifest/icon is processed,
 //   and multiple fetchPageInfo() calls can await the same icon
-const pageIconMap = new Map<string, Promise<string>>();
+const pageIconCache = new LRUCache<string, Promise<string>>({
+	max: 50,
+});
+
 function fetchPageIcon(src: URL, srcHast: Root): Promise<string> {
-	if (pageIconMap.has(src.hostname)) return pageIconMap.get(src.hostname)!;
+	if (pageIconCache.has(src.hostname)) return pageIconCache.get(src.hostname)!;
 
 	const promise = (async () => {
 		const iconPath = getIconPath(src);
-		const iconDir = await fs.promises
-			.readdir(path.dirname(iconPath))
+		const iconDir = path.dirname("public/" + iconPath);
+		await fs.promises.mkdir(iconDir, { recursive: true });
+
+		const existingIconFiles = await fs.promises
+			.readdir(iconDir)
 			.catch(() => []);
 
 		// If an icon has already been downloaded for the origin (in a previous build)
-		const existingIconFile = iconDir.find((file) =>
+		const existingIconFile = existingIconFiles.find((file) =>
 			file.startsWith(path.basename(iconPath)),
 		);
 		if (existingIconFile) {
-			return path.join(path.dirname(iconPath), existingIconFile);
+			return iconDir.replace(/^public/, "") + "/" + existingIconFile;
 		}
 
 		// <link rel="manifest" href="/manifest.json">
@@ -132,10 +138,13 @@ function fetchPageIcon(src: URL, srcHast: Root): Promise<string> {
 		return "/" + iconPath + iconExt;
 	})()
 		// if an error is thrown, or response is null, use the default page icon
-		.catch(() => null)
+		.catch((e) => {
+			console.error("[rehypeIFrameClickToRun]", e);
+			return null;
+		})
 		.then((p) => p || defaultPageIcon);
 
-	pageIconMap.set(src.hostname, promise);
+	pageIconCache.set(src.hostname, promise);
 	return promise;
 }