From b15b07f33db56aa32d285ccf92422be802720b57 Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Sat, 12 Oct 2024 06:58:14 -0400 Subject: [PATCH 1/4] unify markdown --- apps/svelte.dev/package.json | 2 - .../tutorial/[...slug]/content.server.ts | 94 ++----------------- pnpm-lock.yaml | 17 ---- 3 files changed, 10 insertions(+), 103 deletions(-) diff --git a/apps/svelte.dev/package.json b/apps/svelte.dev/package.json index 608e32e33a..2f88d25e0e 100644 --- a/apps/svelte.dev/package.json +++ b/apps/svelte.dev/package.json @@ -54,8 +54,6 @@ "flexsearch": "^0.7.43", "flru": "^1.0.2", "port-authority": "^2.0.1", - "prism-svelte": "^0.5.0", - "prismjs": "^1.29.0", "shiki": "^1.22.0", "topojson-client": "^3.1.0", "vitest": "^2.1.2", diff --git a/apps/svelte.dev/src/routes/tutorial/[...slug]/content.server.ts b/apps/svelte.dev/src/routes/tutorial/[...slug]/content.server.ts index 950b6db98c..7833330069 100644 --- a/apps/svelte.dev/src/routes/tutorial/[...slug]/content.server.ts +++ b/apps/svelte.dev/src/routes/tutorial/[...slug]/content.server.ts @@ -1,18 +1,10 @@ -// @ts-expect-error has no types -import PrismJS from 'prismjs'; -import 'prismjs/components/prism-bash.js'; -import 'prismjs/components/prism-diff.js'; -import 'prismjs/components/prism-typescript.js'; -import 'prism-svelte'; import { read } from '$app/server'; import { index } from '$lib/server/content'; -import { transform } from '@sveltejs/site-kit/markdown'; import type { Exercise, ExerciseStub, PartStub, Scope } from '$lib/tutorial'; import { error } from '@sveltejs/kit'; import { text_files } from './shared'; import type { Document } from '@sveltejs/site-kit'; -import { escape_html } from '$lib/utils/escape'; -import type { Renderer } from 'marked'; +import { render_content } from '$lib/server/renderer'; const lookup: Record< string, @@ -100,74 +92,6 @@ async function get(assets: Record, key: string) { : Buffer.from(await response.arrayBuffer()).toString('base64'); } -const languages = { - bash: 'bash', - env: 'bash', - html: 'markup', - svelte: 'svelte', - js: 'javascript', - css: 'css', - ts: 'typescript', - '': '' -}; - -const delimiter_substitutes = { - '+++': ' ', - '---': ' ', - ':::': ' ' -}; - -function highlight_spans(content: string, classname: string) { - return `${content}`; - // return content.replace(/ { - // return ` = { - code: ({ text, lang = '' }) => { - /** @type {Record} */ - const options: Record = {}; - - let source = text - .replace(/\/\/\/ (.+?)(?:: (.+))?\n/gm, (_, key, value) => { - options[key] = value; - return ''; - }) - .replace(/(\+\+\+|---|:::)/g, (_, delimiter: keyof typeof delimiter_substitutes) => { - return delimiter_substitutes[delimiter]; - }) - .replace(/\*\\\//g, '*/'); - - let html = '
'; - - if (options.file) { - html += `${options.file}`; - } - - html += '
'; - - const plang = languages[lang as keyof typeof languages]; - const highlighted = plang - ? // TODO use shiki here rather than Prism? - PrismJS.highlight(source, PrismJS.languages[plang], lang) - : escape_html(source); - - html += `
${highlighted}
`; - - return html - .replace(/ {13}([^ ][^]+?) {13}/g, (_, content) => { - return highlight_spans(content, 'highlight add'); - }) - .replace(/ {11}([^ ][^]+?) {11}/g, (_, content) => { - return highlight_spans(content, 'highlight remove'); - }) - .replace(/ {9}([^ ][^]+?) {9}/g, (_, content) => { - return highlight_spans(content, 'highlight'); - }); - } -}; - export async function load_exercise(slug: string): Promise { if (!(slug in lookup)) { error(404, 'No such tutorial found'); @@ -228,13 +152,15 @@ export async function load_exercise(slug: string): Promise { prev: prev && { slug: prev.slug }, next, markdown: exercise.body, - html: await transform(exercise.body, { - ...default_renderer, - codespan: ({ text }) => - filenames.size > 1 && filenames.has(text) - ? `${text}` - : `${text}` - }), + html: (await render_content(exercise.file, exercise.body)).replace( + /(.+?)<\/code>/g, + (match, filename) => { + // TODO wire this up + return filenames.size > 1 && filenames.has(filename) + ? `${filename}` + : match; + } + ), dir: exercise.file.split('/').slice(0, -1).join('/'), editing_constraints: { create: new Set(exercise.metadata.editing_constraints?.create ?? []), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7e84fa10ba..5e1d0e548f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -128,12 +128,6 @@ importers: port-authority: specifier: ^2.0.1 version: 2.0.1 - prism-svelte: - specifier: ^0.5.0 - version: 0.5.0 - prismjs: - specifier: ^1.29.0 - version: 1.29.0 shiki: specifier: ^1.22.0 version: 1.22.0 @@ -2592,13 +2586,6 @@ packages: resolution: {integrity: sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==} engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} - prism-svelte@0.5.0: - resolution: {integrity: sha512-db91Bf3pRGKDPz1lAqLFSJXeW13mulUJxhycysFpfXV5MIK7RgWWK2E5aPAa71s8TCzQUXxF5JOV42/iOs6QkA==} - - prismjs@1.29.0: - resolution: {integrity: sha512-Kx/1w86q/epKcmte75LNrEoT+lX8pBpavuAbvJWRXar7Hz8jrtF+e3vY751p0R8H9HdArwaCTNDDzHg/ScJK1Q==} - engines: {node: '>=6'} - property-information@6.5.0: resolution: {integrity: sha512-PgTgs/BlvHxOu8QuEN7wi5A0OmXaBcHpmCSTehcs6Uuu9IkDIEo13Hy7n898RHfrQ49vKCoGeWZSaAK01nwVig==} @@ -5389,10 +5376,6 @@ snapshots: ansi-styles: 5.2.0 react-is: 17.0.2 - prism-svelte@0.5.0: {} - - prismjs@1.29.0: {} - property-information@6.5.0: {} pseudomap@1.0.2: {} From 2225d8b2885d0c5e7156e26fcca05865aa4c9fa4 Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Sat, 12 Oct 2024 07:05:53 -0400 Subject: [PATCH 2/4] add option to skip typechecking --- apps/svelte.dev/src/lib/server/renderer.ts | 137 +++++++++--------- .../tutorial/[...slug]/content.server.ts | 2 +- .../site-kit/src/lib/markdown/renderer.ts | 34 +++-- 3 files changed, 92 insertions(+), 81 deletions(-) diff --git a/apps/svelte.dev/src/lib/server/renderer.ts b/apps/svelte.dev/src/lib/server/renderer.ts index 89113fe4f2..c22d239ab7 100644 --- a/apps/svelte.dev/src/lib/server/renderer.ts +++ b/apps/svelte.dev/src/lib/server/renderer.ts @@ -1,82 +1,85 @@ import { render_content_markdown } from '@sveltejs/site-kit/markdown'; -export const render_content = (filename: string, body: string) => - render_content_markdown(filename, body, { - twoslashBanner: (filename, source) => { - // TODO these are copied from Svelte and SvelteKit - adjust for new filenames - const injected = []; +export const render_content = ( + filename: string, + body: string, + options: { check?: boolean } = {} +) => { + return render_content_markdown(filename, body, options, (filename, source) => { + // TODO these are copied from Svelte and SvelteKit - adjust for new filenames + const injected = []; - if (/(svelte)/.test(source) || filename.includes('typescript')) { - injected.push(`// @filename: ambient.d.ts`, `/// `); - } - - if (filename.includes('svelte-compiler')) { - injected.push('// @esModuleInterop'); - } + if (/(svelte)/.test(source) || filename.includes('typescript')) { + injected.push(`// @filename: ambient.d.ts`, `/// `); + } - if (filename.includes('svelte.md')) { - injected.push('// @errors: 2304'); - } + if (filename.includes('svelte-compiler')) { + injected.push('// @esModuleInterop'); + } - // Actions JSDoc examples are invalid. Too many errors, edge cases - // d.ts files are not properly supported right now, fix this later - if (filename.includes('svelte-action') || source.includes(' declare const ')) { - injected.push('// @noErrors'); - } + if (filename.includes('svelte.md')) { + injected.push('// @errors: 2304'); + } - if (filename.includes('typescript')) { - injected.push('// @errors: 2304'); - } + // Actions JSDoc examples are invalid. Too many errors, edge cases + // d.ts files are not properly supported right now, fix this later + if (filename.includes('svelte-action') || source.includes(' declare const ')) { + injected.push('// @noErrors'); + } - if ( - source.includes('$app/') || - source.includes('$service-worker') || - source.includes('@sveltejs/kit/') - ) { - injected.push(`// @filename: ambient-kit.d.ts`, `/// `); - } + if (filename.includes('typescript')) { + injected.push('// @errors: 2304'); + } - if (source.includes('$env/')) { - // TODO we're hardcoding static env vars that are used in code examples - // in the types, which isn't... totally ideal, but will do for now - injected.push( - `declare module '$env/dynamic/private' { export const env: Record }`, - `declare module '$env/dynamic/public' { export const env: Record }`, - `declare module '$env/static/private' { export const API_KEY: string }`, - `declare module '$env/static/public' { export const PUBLIC_BASE_URL: string }` - ); - } + if ( + source.includes('$app/') || + source.includes('$service-worker') || + source.includes('@sveltejs/kit/') + ) { + injected.push(`// @filename: ambient-kit.d.ts`, `/// `); + } - if (source.includes('./$types') && !source.includes('@filename: $types.d.ts')) { - injected.push( - `// @filename: $types.d.ts`, - `import type * as Kit from '@sveltejs/kit';`, - `export type PageLoad = Kit.Load>;`, - `export type PageServerLoad = Kit.ServerLoad>;`, - `export type LayoutLoad = Kit.Load>;`, - `export type LayoutServerLoad = Kit.ServerLoad>;`, - `export type RequestHandler = Kit.RequestHandler>;`, - `export type Action = Kit.Action>;`, - `export type Actions = Kit.Actions>;`, - `export type EntryGenerator = () => Promise>> | Array>;` - ); - } + if (source.includes('$env/')) { + // TODO we're hardcoding static env vars that are used in code examples + // in the types, which isn't... totally ideal, but will do for now + injected.push( + `declare module '$env/dynamic/private' { export const env: Record }`, + `declare module '$env/dynamic/public' { export const env: Record }`, + `declare module '$env/static/private' { export const API_KEY: string }`, + `declare module '$env/static/public' { export const PUBLIC_BASE_URL: string }` + ); + } - // special case — we need to make allowances for code snippets coming - // from e.g. ambient.d.ts - if (filename.endsWith('$env-all.md') || filename.endsWith('$app-forms.md')) { - injected.push('// @errors: 7006 7031'); - } + if (source.includes('./$types') && !source.includes('@filename: $types.d.ts')) { + injected.push( + `// @filename: $types.d.ts`, + `import type * as Kit from '@sveltejs/kit';`, + `export type PageLoad = Kit.Load>;`, + `export type PageServerLoad = Kit.ServerLoad>;`, + `export type LayoutLoad = Kit.Load>;`, + `export type LayoutServerLoad = Kit.ServerLoad>;`, + `export type RequestHandler = Kit.RequestHandler>;`, + `export type Action = Kit.Action>;`, + `export type Actions = Kit.Actions>;`, + `export type EntryGenerator = () => Promise>> | Array>;` + ); + } - if (filename.endsWith('10-configuration.md')) { - injected.push('// @errors: 2307'); - } + // special case — we need to make allowances for code snippets coming + // from e.g. ambient.d.ts + if (filename.endsWith('$env-all.md') || filename.endsWith('$app-forms.md')) { + injected.push('// @errors: 7006 7031'); + } - // another special case - if (source.includes('$lib/types')) { - injected.push(`declare module '$lib/types' { export interface User {} }`); - } + if (filename.endsWith('10-configuration.md')) { + injected.push('// @errors: 2307'); + } - return injected.join('\n'); + // another special case + if (source.includes('$lib/types')) { + injected.push(`declare module '$lib/types' { export interface User {} }`); } + + return injected.join('\n'); }); +}; diff --git a/apps/svelte.dev/src/routes/tutorial/[...slug]/content.server.ts b/apps/svelte.dev/src/routes/tutorial/[...slug]/content.server.ts index 7833330069..ac72b4cf43 100644 --- a/apps/svelte.dev/src/routes/tutorial/[...slug]/content.server.ts +++ b/apps/svelte.dev/src/routes/tutorial/[...slug]/content.server.ts @@ -152,7 +152,7 @@ export async function load_exercise(slug: string): Promise { prev: prev && { slug: prev.slug }, next, markdown: exercise.body, - html: (await render_content(exercise.file, exercise.body)).replace( + html: (await render_content(exercise.file, exercise.body, { check: false })).replace( /(.+?)<\/code>/g, (match, filename) => { // TODO wire this up diff --git a/packages/site-kit/src/lib/markdown/renderer.ts b/packages/site-kit/src/lib/markdown/renderer.ts index 2ec91cc8d5..580e37ecf9 100644 --- a/packages/site-kit/src/lib/markdown/renderer.ts +++ b/packages/site-kit/src/lib/markdown/renderer.ts @@ -191,9 +191,11 @@ const snippets = await create_snippet_cache(); export async function render_content_markdown( filename: string, body: string, - { twoslashBanner }: { twoslashBanner?: TwoslashBanner } = {} + options: { check?: boolean }, + twoslashBanner?: TwoslashBanner ) { const headings: string[] = []; + const { check = true } = options; return await transform(body, { async walkTokens(token) { @@ -248,7 +250,8 @@ export async function render_content_markdown( language: token.lang, source, twoslashBanner, - options + options, + check }); if (converted) { @@ -258,7 +261,8 @@ export async function render_content_markdown( language: token.lang === 'js' ? 'ts' : token.lang, source: converted, twoslashBanner, - options + options, + check }); } @@ -654,7 +658,8 @@ async function syntax_highlight({ filename, language, twoslashBanner, - options + options, + check }: { prelude: string; source: string; @@ -662,6 +667,7 @@ async function syntax_highlight({ language: string; twoslashBanner?: TwoslashBanner; options: SnippetOptions; + check: boolean; }) { let html = ''; @@ -696,15 +702,17 @@ async function syntax_highlight({ html = await codeToHtml(prelude + redacted, { lang: 'ts', theme, - transformers: [ - transformerTwoslash({ - twoslashOptions: { - compilerOptions: { - types: ['svelte', '@sveltejs/kit'] - } - } - }) - ] + transformers: check + ? [ + transformerTwoslash({ + twoslashOptions: { + compilerOptions: { + types: ['svelte', '@sveltejs/kit'] + } + } + }) + ] + : [] }); html = html.replace(/ {27,}/g, () => redactions.shift()!); From 678424d9c7c398d88268e0b15313fb43a440218d Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Sat, 12 Oct 2024 07:43:25 -0400 Subject: [PATCH 3/4] fix --- .../site-kit/src/lib/markdown/renderer.ts | 62 ++++++------------- 1 file changed, 19 insertions(+), 43 deletions(-) diff --git a/packages/site-kit/src/lib/markdown/renderer.ts b/packages/site-kit/src/lib/markdown/renderer.ts index 580e37ecf9..2398d133ef 100644 --- a/packages/site-kit/src/lib/markdown/renderer.ts +++ b/packages/site-kit/src/lib/markdown/renderer.ts @@ -15,12 +15,7 @@ interface SnippetOptions { copy: boolean; } -type TwoslashBanner = ( - filename: string, - content: string, - language: string, - options: SnippetOptions -) => string; +type TwoslashBanner = (filename: string, content: string) => string; // Supports js, svelte, yaml files const METADATA_REGEX = @@ -209,11 +204,17 @@ export async function render_content_markdown( let { source, options } = parse_options(token.text, token.lang); source = adjust_tab_indentation(source, token.lang); - const match = /((?:[\s\S]+)\/\/ ---cut---\n)?([\s\S]+)/.exec(source)!; + let prelude = ''; - const prelude = match[1]; + if ((token.lang === 'js' || token.lang === 'ts') && check) { + const match = /((?:[\s\S]+)\/\/ ---cut---\n)?([\s\S]+)/.exec(source)!; + [, prelude = '// ---cut---\n', source] = match; + + const banner = twoslashBanner?.(filename, source); + if (banner) prelude = '// @filename: injected.d.ts\n' + banner + '\n' + prelude; + } - source = match[2].replace( + source = source.replace( /(\+\+\+|---|:::)/g, (_, delimiter: keyof typeof delimiter_substitutes) => { return delimiter_substitutes[delimiter]; @@ -244,26 +245,16 @@ export async function render_content_markdown( html += ''; - html += await syntax_highlight({ - prelude, - filename, - language: token.lang, - source, - twoslashBanner, - options, - check - }); + html += await syntax_highlight({ filename, language: token.lang, prelude, source, check }); if (converted) { - html += await syntax_highlight({ - prelude, - filename, - language: token.lang === 'js' ? 'ts' : token.lang, - source: converted, - twoslashBanner, - options, - check - }); + const language = token.lang === 'js' ? 'ts' : token.lang; + + if (language === 'ts') { + prelude = prelude.replace(/(\/\/ @filename: .+)\.js$/gm, '$1.ts'); + } + + html += await syntax_highlight({ filename, language, prelude, source: converted, check }); } html += ''; @@ -657,16 +648,12 @@ async function syntax_highlight({ source, filename, language, - twoslashBanner, - options, check }: { prelude: string; source: string; filename: string; language: string; - twoslashBanner?: TwoslashBanner; - options: SnippetOptions; check: boolean; }) { let html = ''; @@ -679,18 +666,6 @@ async function syntax_highlight({ }) ); } else if (language === 'js' || language === 'ts') { - let banner = twoslashBanner?.(filename, source, language, options); - - if (banner) { - banner = '// @filename: injected.d.ts\n' + banner; - } - - prelude = (banner ?? '') + '\n' + (prelude ?? '// ---cut---\n'); - - if (language === 'ts') { - prelude = prelude.replace(/(\/\/ @filename: .+)\.js$/gm, '$1.ts'); - } - /** We need to stash code wrapped in `---` highlights, because otherwise TS will error on e.g. bad syntax, duplicate declarations */ const redactions: string[] = []; @@ -698,6 +673,7 @@ async function syntax_highlight({ redactions.push(content); return ' '.repeat(content.length); }); + try { html = await codeToHtml(prelude + redacted, { lang: 'ts', From 7e30ceb71be5dbbc07a2a6852d735663aeac2ee2 Mon Sep 17 00:00:00 2001 From: Rich Harris Date: Sat, 12 Oct 2024 07:55:02 -0400 Subject: [PATCH 4/4] tidy up --- .../site-kit/src/lib/markdown/renderer.ts | 41 +++++-------------- 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/packages/site-kit/src/lib/markdown/renderer.ts b/packages/site-kit/src/lib/markdown/renderer.ts index 2398d133ef..110901d60c 100644 --- a/packages/site-kit/src/lib/markdown/renderer.ts +++ b/packages/site-kit/src/lib/markdown/renderer.ts @@ -313,7 +313,6 @@ export async function render_content_markdown( /** * Pre-render step. Takes in all the code snippets, and replaces them with TS snippets if possible - * May replace the language labels (```js) to custom labels(```generated-ts, ```original-js, ```generated-svelte,```original-svelte) */ async function generate_ts_from_js( code: string, @@ -323,32 +322,23 @@ async function generate_ts_from_js( // No named file -> assume that the code is not meant to be shown in two versions if (!options.file) return; - if (language === 'js') { - // config files have no .ts equivalent - if (options.file === 'svelte.config.js') return; + // config files have no .ts equivalent + if (options.file === 'svelte.config.js') return; - let [before, after] = code.split('// ---cut---\n'); + if (language === 'svelte') { + // Assumption: no module blocks + const script = code.match(/`); } - // Assumption: no module blocks - const script = code.match(/`); + return await convert_to_ts(code); } function get_jsdoc(node: ts.Node) { @@ -367,15 +357,6 @@ async function convert_to_ts(js_code: string, indent = '', offset = '') { // *\/ appears in some JsDoc comments in d.ts files due to the JSDoc-in-JSDoc problem .replace(/\*\\\//g, '*/'); - // TODO temp - if (js_code.includes('// ---cut---')) { - throw new Error('unexpected cut directive'); - } - - if (js_code.includes('/// file:')) { - throw new Error('unexpected file directive'); - } - const ast = ts.createSourceFile( 'filename.ts', js_code,