From b2d000350c4b110015e66709292131c0433b67d0 Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Wed, 9 Oct 2024 21:26:25 -0400 Subject: [PATCH 1/4] use module contents to determine cache validity --- .../site-kit/src/lib/markdown/renderer.ts | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/packages/site-kit/src/lib/markdown/renderer.ts b/packages/site-kit/src/lib/markdown/renderer.ts index 29ea1982ad..ecb7097909 100644 --- a/packages/site-kit/src/lib/markdown/renderer.ts +++ b/packages/site-kit/src/lib/markdown/renderer.ts @@ -1,5 +1,5 @@ import MagicString from 'magic-string'; -import { createHash } from 'node:crypto'; +import { createHash, Hash } from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import ts from 'typescript'; @@ -453,14 +453,13 @@ function find_nearest_node_modules(file: string): string | null { } /** - * Get the `mtime` of the most recently modified file in a dependency graph, + * Get the hash of a dependency graph, * excluding imports from `node_modules` */ -function get_mtime(file: string, seen = new Set()) { - if (seen.has(file)) return -1; +function hash_graph(hash: Hash, file: string, seen = new Set()) { + if (seen.has(file)) return; seen.add(file); - let mtime = fs.statSync(file).mtimeMs; const content = fs.readFileSync(file, 'utf-8'); for (const [_, source] of content.matchAll(/^import(?:.+?\s+from\s+)?['"](.+)['"];?$/gm)) { @@ -471,17 +470,16 @@ function get_mtime(file: string, seen = new Set()) { if (!fs.existsSync(resolved)) throw new Error(`Could not resolve ${source} relative to ${file}`); - mtime = Math.max(mtime, get_mtime(resolved, seen)); + hash_graph(hash, resolved, seen); } - return mtime; + hash.update(content); } -const mtime = Math.max( - get_mtime(fileURLToPath(import.meta.url)), - fs.statSync('node_modules').mtimeMs, - fs.statSync('../../pnpm-lock.yaml').mtimeMs -); +const hash = createHash('sha256'); +hash.update(fs.readFileSync('../../pnpm-lock.yaml', 'utf-8')); +hash_graph(hash, fileURLToPath(import.meta.url)); +const digest = hash.digest().toString('base64').replace(/\//g, '-'); /** * Utility function to work with code snippet caching. @@ -500,10 +498,11 @@ const mtime = Math.max( async function create_snippet_cache(should: boolean) { const cache = new Map(); const directory = find_nearest_node_modules(import.meta.url) + '/.snippets'; + const current = `${directory}/${digest}`; if (fs.existsSync(directory)) { for (const dir of fs.readdirSync(directory)) { - if (!fs.statSync(`${directory}/${dir}`).isDirectory() || +dir < mtime) { + if (dir !== digest) { fs.rmSync(`${directory}/${dir}`, { force: true, recursive: true }); } } @@ -512,7 +511,7 @@ async function create_snippet_cache(should: boolean) { } try { - fs.mkdirSync(`${directory}/${mtime}`); + fs.mkdirSync(`${directory}/${digest}`); } catch {} function get_file(source: string) { @@ -520,7 +519,7 @@ async function create_snippet_cache(should: boolean) { hash.update(source); const digest = hash.digest().toString('base64').replace(/\//g, '-'); - return `${directory}/${mtime}/${digest}.html`; + return `${current}/${digest}.html`; } return { From 74d6a803048a7b89d2e5c4a6861dc50586fc31c1 Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Wed, 9 Oct 2024 21:27:34 -0400 Subject: [PATCH 2/4] share cache --- .../site-kit/src/lib/markdown/renderer.ts | 152 +++++++++--------- 1 file changed, 75 insertions(+), 77 deletions(-) diff --git a/packages/site-kit/src/lib/markdown/renderer.ts b/packages/site-kit/src/lib/markdown/renderer.ts index ecb7097909..af617cd50e 100644 --- a/packages/site-kit/src/lib/markdown/renderer.ts +++ b/packages/site-kit/src/lib/markdown/renderer.ts @@ -34,6 +34,81 @@ const theme = createCssVariablesTheme({ fontStyle: true }); +const hash = createHash('sha256'); +hash.update(fs.readFileSync('../../pnpm-lock.yaml', 'utf-8')); +hash_graph(hash, fileURLToPath(import.meta.url)); +const digest = hash.digest().toString('base64').replace(/\//g, '-'); + +/** + * Utility function to work with code snippet caching. + * + * @example + * + * ```js + * const SNIPPETS_CACHE = create_snippet_cache(true); + * + * const { uid, code } = SNIPPETS_CACHE.get(source); + * + * // Later to save the code to the cache + * SNIPPETS_CACHE.save(uid, processed_code); + * ``` + */ +async function create_snippet_cache() { + const cache = new Map(); + const directory = find_nearest_node_modules(import.meta.url) + '/.snippets'; + const current = `${directory}/${digest}`; + + if (fs.existsSync(directory)) { + for (const dir of fs.readdirSync(directory)) { + if (dir !== digest) { + fs.rmSync(`${directory}/${dir}`, { force: true, recursive: true }); + } + } + } else { + fs.mkdirSync(directory); + } + + try { + fs.mkdirSync(`${directory}/${digest}`); + } catch {} + + function get_file(source: string) { + const hash = createHash('sha256'); + hash.update(source); + const digest = hash.digest().toString('base64').replace(/\//g, '-'); + + return `${current}/${digest}.html`; + } + + return { + get(source: string) { + let snippet = cache.get(source); + + if (snippet === undefined) { + const file = get_file(source); + + if (fs.existsSync(file)) { + snippet = fs.readFileSync(file, 'utf-8'); + cache.set(source, snippet); + } + } + + return snippet; + }, + save(source: string, html: string) { + cache.set(source, html); + + try { + fs.mkdirSync(directory); + } catch {} + + fs.writeFileSync(get_file(source), html); + } + }; +} + +const snippets = await create_snippet_cache(); + /** * A super markdown renderer function. Renders svelte and kit docs specific specific markdown code to html. * @@ -119,8 +194,6 @@ export async function render_content_markdown( body: string, { twoslashBanner }: { twoslashBanner?: TwoslashBanner } = {} ) { - const snippets = await create_snippet_cache(true); - const headings: string[] = []; return await transform(body, { @@ -476,81 +549,6 @@ function hash_graph(hash: Hash, file: string, seen = new Set()) { hash.update(content); } -const hash = createHash('sha256'); -hash.update(fs.readFileSync('../../pnpm-lock.yaml', 'utf-8')); -hash_graph(hash, fileURLToPath(import.meta.url)); -const digest = hash.digest().toString('base64').replace(/\//g, '-'); - -/** - * Utility function to work with code snippet caching. - * - * @example - * - * ```js - * const SNIPPETS_CACHE = create_snippet_cache(true); - * - * const { uid, code } = SNIPPETS_CACHE.get(source); - * - * // Later to save the code to the cache - * SNIPPETS_CACHE.save(uid, processed_code); - * ``` - */ -async function create_snippet_cache(should: boolean) { - const cache = new Map(); - const directory = find_nearest_node_modules(import.meta.url) + '/.snippets'; - const current = `${directory}/${digest}`; - - if (fs.existsSync(directory)) { - for (const dir of fs.readdirSync(directory)) { - if (dir !== digest) { - fs.rmSync(`${directory}/${dir}`, { force: true, recursive: true }); - } - } - } else { - fs.mkdirSync(directory); - } - - try { - fs.mkdirSync(`${directory}/${digest}`); - } catch {} - - function get_file(source: string) { - const hash = createHash('sha256'); - hash.update(source); - const digest = hash.digest().toString('base64').replace(/\//g, '-'); - - return `${current}/${digest}.html`; - } - - return { - get(source: string) { - if (!should) return; - - let snippet = cache.get(source); - - if (snippet === undefined) { - const file = get_file(source); - - if (fs.existsSync(file)) { - snippet = fs.readFileSync(file, 'utf-8'); - cache.set(source, snippet); - } - } - - return snippet; - }, - save(source: string, html: string) { - cache.set(source, html); - - try { - fs.mkdirSync(directory); - } catch {} - - fs.writeFileSync(get_file(source), html); - } - }; -} - function create_type_links( modules: Modules | undefined, resolve_link?: (module_name: string, type_name: string) => { slug: string; page: string } From 6c7a0845d7943cd714144aafbcb103808262715f Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Wed, 9 Oct 2024 21:27:56 -0400 Subject: [PATCH 3/4] remove some unused code --- .../site-kit/src/lib/markdown/renderer.ts | 33 ------------------- 1 file changed, 33 deletions(-) diff --git a/packages/site-kit/src/lib/markdown/renderer.ts b/packages/site-kit/src/lib/markdown/renderer.ts index af617cd50e..8a4656d716 100644 --- a/packages/site-kit/src/lib/markdown/renderer.ts +++ b/packages/site-kit/src/lib/markdown/renderer.ts @@ -549,39 +549,6 @@ function hash_graph(hash: Hash, file: string, seen = new Set()) { hash.update(content); } -function create_type_links( - modules: Modules | undefined, - resolve_link?: (module_name: string, type_name: string) => { slug: string; page: string } -): { - type_regex: RegExp | null; - type_links: Map | null; -} { - if (!modules || modules.length === 0 || !resolve_link) - return { type_regex: null, type_links: null }; - - const type_regex = new RegExp( - `(import\\('(?:svelte|@sveltejs\\/kit)'\\)\\.)?\\b(${modules - .flatMap((module) => module.types) - .map((type) => type?.name) - .join('|')})\\b`, - 'g' - ); - - /** @type {Map} */ - const type_links = new Map(); - - for (const module of modules) { - if (!module || !module.name) continue; - - for (const type of module.types ?? []) { - const link = resolve_link(module.name, type.name); - type_links.set(type.name, { ...link, relativeURL: link.page + '#' + link.slug }); - } - } - - return { type_regex, type_links }; -} - function parse_options(source: string, language: string) { METADATA_REGEX.lastIndex = 0; From ce48b1223f4b168fbd61f66e1679ca15ef95ab57 Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Wed, 9 Oct 2024 21:35:36 -0400 Subject: [PATCH 4/4] empty