From 3beff85f592b180a1c777d401e17838919016d63 Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sat, 26 Apr 2025 16:27:58 -0400 Subject: [PATCH 001/101] [WIP] Simplify core --- src/core/highlight-all.ts | 49 ++++++++++++++++++ src/core/hooks.ts | 106 +------------------------------------- src/core/prism.ts | 28 +++++----- src/global.ts | 10 ++-- 4 files changed, 67 insertions(+), 126 deletions(-) create mode 100644 src/core/highlight-all.ts diff --git a/src/core/highlight-all.ts b/src/core/highlight-all.ts new file mode 100644 index 0000000000..83ea5772c6 --- /dev/null +++ b/src/core/highlight-all.ts @@ -0,0 +1,49 @@ +import { HookState } from './hook-state'; +import prism, {Prism} from './prism'; + +/** + * This is the most high-level function in Prism’s API. + * It queries all the elements that have a `.language-xxxx` class and then calls {@link Prism#highlightElement} on + * each one of them. + * + * The following hooks will be run: + * 1. `before-highlightall` + * 2. `before-all-elements-highlight` + * 3. All hooks of {@link Prism#highlightElement} for each element. + */ +export function highlightAll (this: Prism, options: HighlightAllOptions = {}) { + const { root, async, callback } = options; + + const env: Record = + { + callback, + root: root ?? document, + selector: + 'code[class*="language-"], [class*="language-"] code, code[class*="lang-"], [class*="lang-"] code', + state: new HookState(), + }; + + this.hooks.run('before-highlightall', env); + + env.elements = [...env.root.querySelectorAll(env.selector)]; + + this.hooks.run('before-all-elements-highlight', env); + + for (const element of env.elements) { + this.highlightElement(element, { async, callback: env.callback }); + } +} + +export interface HighlightAllOptions { + /** + * The root element, whose descendants that have a `.language-xxxx` class will be highlighted. + */ + root?: ParentNode; + async?: AsyncHighlighter; + /** + * An optional callback to be invoked on each element after its highlighting is done. + * + * @see HighlightElementOptions#callback + */ + callback?: (element: Element) => void; +} diff --git a/src/core/hooks.ts b/src/core/hooks.ts index 8856ab48e1..0cfe80db41 100644 --- a/src/core/hooks.ts +++ b/src/core/hooks.ts @@ -1,7 +1,3 @@ -import type { Grammar, TokenName } from '../types'; -import type { HookState } from './hook-state'; -import type { TokenStream } from './token'; - export class Hooks { // eslint-disable-next-line func-call-spacing private _all = new Map void)[]>(); @@ -47,7 +43,7 @@ export class Hooks { * @param name The name of the hook. * @param env The environment variables of the hook passed to all callbacks registered. */ - run (name: Name, env: HookEnv): void { + run (name: Name, env: Record): void { const callbacks = this._all.get(name); if (!callbacks || !callbacks.length) { @@ -60,102 +56,4 @@ export class Hooks { } } -/** - * An interface containing all hooks Prism runs. - */ -export interface HookEnvMap { - // Prism.highlightAll - 'before-highlightall': BeforeHighlightAllEnv; - 'before-all-elements-highlight': BeforeAllElementsHighlightEnv; - - // Prism.highlightElement - 'before-sanity-check': BeforeSanityCheckEnv; - 'before-highlight': BeforeHighlightEnv; - - 'before-insert': BeforeInsertEnv; - 'after-highlight': AfterHighlightEnv; - 'complete': CompleteEnv; - - // Prism.highlight - 'before-tokenize': BeforeTokenizeEnv; - 'after-tokenize': AfterTokenizeEnv; - - // stringify - 'wrap': WrapEnv; -} - -export type HookEnv = HookName extends keyof HookEnvMap - ? HookEnvMap[HookName] - : unknown; - -export type HookCallback = (env: HookEnv) => void; - -interface StatefulEnv { - readonly state: HookState; -} - -export interface BeforeHighlightAllEnv extends StatefulEnv { - root: ParentNode; - selector: string; - callback?: (element: Element) => void; -} -export interface BeforeAllElementsHighlightEnv extends StatefulEnv { - root: ParentNode; - selector: string; - callback?: (element: Element) => void; - elements: Element[]; -} - -export interface BeforeSanityCheckEnv extends StatefulEnv { - element: Element; - language: string; - grammar: Grammar | undefined; - code: string; -} -export interface BeforeHighlightEnv extends StatefulEnv { - element: Element; - language: string; - grammar: Grammar | undefined; - code: string; -} -export interface CompleteEnv extends StatefulEnv { - element: Element; - language: string; - grammar: Grammar | undefined; - code: string; -} -export interface BeforeInsertEnv extends StatefulEnv { - element: Element; - language: string; - grammar: Grammar | undefined; - code: string; - highlightedCode: string; -} -export interface AfterHighlightEnv extends StatefulEnv { - element: Element; - language: string; - grammar: Grammar | undefined; - code: string; - highlightedCode: string; -} - -export interface BeforeTokenizeEnv { - code: string; - language: string; - grammar: Grammar | undefined; -} -export interface AfterTokenizeEnv { - code: string; - language: string; - grammar: Grammar; - tokens: TokenStream; -} - -export interface WrapEnv { - type: TokenName; - content: string; - tag: string; - classes: string[]; - attributes: Record; - language: string; -} +export type HookCallback = (env?: Record) => void; diff --git a/src/core/prism.ts b/src/core/prism.ts index ed1324f8a9..6355f70117 100644 --- a/src/core/prism.ts +++ b/src/core/prism.ts @@ -8,15 +8,15 @@ import { Registry } from './registry'; import { Token } from './token'; import type { KnownPlugins } from '../known-plugins'; import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types'; -import type { HookEnvMap } from './hooks'; import type { LinkedListHeadNode, LinkedListMiddleNode, LinkedListTailNode } from './linked-list'; import type { TokenStream } from './token'; +import {highlightAll} from './highlight-all'; /** * Prism: Lightweight, robust, elegant syntax highlighting * * @license MIT - * @author Lea Verou + * @author Lea Verou and contributors */ export class Prism { hooks = new Hooks(); @@ -408,20 +408,6 @@ export interface AsyncHighlightingData { } export type AsyncHighlighter = (data: AsyncHighlightingData) => Promise; -export interface HighlightAllOptions { - /** - * The root element, whose descendants that have a `.language-xxxx` class will be highlighted. - */ - root?: ParentNode; - async?: AsyncHighlighter; - /** - * An optional callback to be invoked on each element after its highlighting is done. - * - * @see HighlightElementOptions#callback - */ - callback?: (element: Element) => void; -} - export interface HighlightElementOptions { async?: AsyncHighlighter; /** @@ -538,3 +524,13 @@ function resolve ( } return undefined; } + +/** + * Prism singleton. + * This will always be available, and will automatically read config options. + * This instance of Prism is unique. Even if this module is imported from + * different sources, the same Prism instance will be returned. + * In global builds, it will also be the Prism global variable. + * Any imported plugins and languages will automatically be added to this instance. + */ +export default new Prism(); diff --git a/src/global.ts b/src/global.ts index 8c66f9ee5a..44e5a8d040 100644 --- a/src/global.ts +++ b/src/global.ts @@ -1,10 +1,8 @@ -import { Prism } from './core/prism'; +import globalPrism, { Prism } from './core/prism'; -const globalSymbol = Symbol.for('Prism global'); - -// eslint-disable-next-line no-undef -const namespace = globalThis as Partial>; -const globalPrism = (namespace[globalSymbol] ??= new Prism()); +declare global { + var Prism: Prism | undefined +} /** * The global {@link Prism} instance. From 46806b6055e3106229e4c60c8963e014102c2f69 Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sat, 26 Apr 2025 16:45:22 -0400 Subject: [PATCH 002/101] highlightAll --- src/core/highlight-all.ts | 9 +++++++++ src/core/prism.ts | 35 +++-------------------------------- 2 files changed, 12 insertions(+), 32 deletions(-) diff --git a/src/core/highlight-all.ts b/src/core/highlight-all.ts index 83ea5772c6..12d0669fbf 100644 --- a/src/core/highlight-all.ts +++ b/src/core/highlight-all.ts @@ -1,5 +1,6 @@ import { HookState } from './hook-state'; import prism, {Prism} from './prism'; +import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types'; /** * This is the most high-level function in Prism’s API. @@ -47,3 +48,11 @@ export interface HighlightAllOptions { */ callback?: (element: Element) => void; } + +export interface AsyncHighlightingData { + language: string; + code: string; + grammar: Grammar; +} +export type AsyncHighlighter = (data: AsyncHighlightingData) => Promise; + diff --git a/src/core/prism.ts b/src/core/prism.ts index 6355f70117..19fd929d3a 100644 --- a/src/core/prism.ts +++ b/src/core/prism.ts @@ -10,7 +10,7 @@ import type { KnownPlugins } from '../known-plugins'; import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types'; import type { LinkedListHeadNode, LinkedListMiddleNode, LinkedListTailNode } from './linked-list'; import type { TokenStream } from './token'; -import {highlightAll} from './highlight-all'; +import {highlightAll, HighlightAllOptions} from './highlight-all'; /** * Prism: Lightweight, robust, elegant syntax highlighting @@ -34,27 +34,7 @@ export class Prism { * 3. All hooks of {@link Prism#highlightElement} for each element. */ highlightAll (options: HighlightAllOptions = {}) { - const { root, async, callback } = options; - - const env: HookEnvMap['before-highlightall'] | HookEnvMap['before-all-elements-highlight'] = - { - callback, - root: root ?? document, - selector: - 'code[class*="language-"], [class*="language-"] code, code[class*="lang-"], [class*="lang-"] code', - state: new HookState(), - }; - - this.hooks.run('before-highlightall', env); - - assertEnv<'before-all-elements-highlight'>(env); - env.elements = [...env.root.querySelectorAll(env.selector)]; - - this.hooks.run('before-all-elements-highlight', env); - - for (const element of env.elements) { - this.highlightElement(element, { async, callback: env.callback }); - } + return highlightAll.call(this, options); } /** @@ -408,16 +388,7 @@ export interface AsyncHighlightingData { } export type AsyncHighlighter = (data: AsyncHighlightingData) => Promise; -export interface HighlightElementOptions { - async?: AsyncHighlighter; - /** - * An optional callback to be invoked after the highlighting is done. - * Mostly useful when `async` is `true`, since in that case, the highlighting is done asynchronously. - * - * @param element The element successfully highlighted. - */ - callback?: (element: Element) => void; -} + export interface HighlightOptions { grammar?: Grammar; From e4f6634ddcb03faec1c622233968c662228d983d Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sat, 26 Apr 2025 16:45:33 -0400 Subject: [PATCH 003/101] Just use properties on env, not a separate `state` property --- src/plugins/keep-markup/prism-keep-markup.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/keep-markup/prism-keep-markup.ts b/src/plugins/keep-markup/prism-keep-markup.ts index da99f76eea..6c104bac14 100644 --- a/src/plugins/keep-markup/prism-keep-markup.ts +++ b/src/plugins/keep-markup/prism-keep-markup.ts @@ -77,11 +77,11 @@ export default { if (data.length) { // data is an array of all existing tags - env.state.set(markupData, data); + env.markupData = data; } }, 'after-highlight': (env) => { - const data = env.state.get(markupData, []); + const data = env.markupdata ?? []; if (data.length) { type End = [node: Text, pos: number] From 183817c8b4fcaa26df855e5acff29f975cb02ec6 Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sat, 26 Apr 2025 16:49:18 -0400 Subject: [PATCH 004/101] Move singleton, eliminate state further --- src/core.ts | 2 +- src/core/highlight-all.ts | 12 +- src/core/prism-class.ts | 497 +++++++++++++++++++++++++++++++++++++ src/core/prism.ts | 498 +------------------------------------- src/core/registry.ts | 2 +- src/global.ts | 2 +- src/types.d.ts | 2 +- 7 files changed, 508 insertions(+), 507 deletions(-) create mode 100644 src/core/prism-class.ts diff --git a/src/core.ts b/src/core.ts index d03c7e928e..7997036216 100644 --- a/src/core.ts +++ b/src/core.ts @@ -1,2 +1,2 @@ -export { Prism } from './core/prism'; +export { Prism } from './core/prism-class'; export { Token } from './core/token'; diff --git a/src/core/highlight-all.ts b/src/core/highlight-all.ts index 12d0669fbf..7fb9506f1b 100644 --- a/src/core/highlight-all.ts +++ b/src/core/highlight-all.ts @@ -1,5 +1,5 @@ -import { HookState } from './hook-state'; -import prism, {Prism} from './prism'; +import Prism from './prism-class'; +import prism from "./prism"; import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types'; /** @@ -13,6 +13,7 @@ import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types' * 3. All hooks of {@link Prism#highlightElement} for each element. */ export function highlightAll (this: Prism, options: HighlightAllOptions = {}) { + const context = this ?? prism; const { root, async, callback } = options; const env: Record = @@ -21,17 +22,16 @@ export function highlightAll (this: Prism, options: HighlightAllOptions = {}) { root: root ?? document, selector: 'code[class*="language-"], [class*="language-"] code, code[class*="lang-"], [class*="lang-"] code', - state: new HookState(), }; - this.hooks.run('before-highlightall', env); + context.hooks.run('before-highlightall', env); env.elements = [...env.root.querySelectorAll(env.selector)]; - this.hooks.run('before-all-elements-highlight', env); + context.hooks.run('before-all-elements-highlight', env); for (const element of env.elements) { - this.highlightElement(element, { async, callback: env.callback }); + context.highlightElement(element, { async, callback: env.callback }); } } diff --git a/src/core/prism-class.ts b/src/core/prism-class.ts new file mode 100644 index 0000000000..ddff195980 --- /dev/null +++ b/src/core/prism-class.ts @@ -0,0 +1,497 @@ +import { getLanguage, setLanguage } from '../shared/dom-util'; +import { rest, tokenize } from '../shared/symbols'; +import { htmlEncode } from '../shared/util'; +import { HookState } from './hook-state'; +import { Hooks } from './hooks'; +import { LinkedList } from './linked-list'; +import { Registry } from './registry'; +import { Token } from './token'; +import type { KnownPlugins } from '../known-plugins'; +import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types'; +import type { LinkedListHeadNode, LinkedListMiddleNode, LinkedListTailNode } from './linked-list'; +import type { TokenStream } from './token'; +import {highlightAll, HighlightAllOptions} from './highlight-all'; + +/** + * Prism: Lightweight, robust, elegant syntax highlighting + * + * @license MIT + * @author Lea Verou and contributors + */ +export default class Prism { + hooks = new Hooks(); + components = new Registry(this); + plugins: Partial & KnownPlugins> = {}; + + /** + * This is the most high-level function in Prism’s API. + * It queries all the elements that have a `.language-xxxx` class and then calls {@link Prism#highlightElement} on + * each one of them. + * + * The following hooks will be run: + * 1. `before-highlightall` + * 2. `before-all-elements-highlight` + * 3. All hooks of {@link Prism#highlightElement} for each element. + */ + highlightAll (options: HighlightAllOptions = {}) { + return highlightAll.call(this, options); + } + + /** + * Highlights the code inside a single element. + * + * The following hooks will be run: + * 1. `before-sanity-check` + * 2. `before-highlight` + * 3. All hooks of {@link Prism#highlight}. These hooks will be run by an asynchronous worker if `async` is `true`. + * 4. `before-insert` + * 5. `after-highlight` + * 6. `complete` + * + * Some the above hooks will be skipped if the element doesn't contain any text or there is no grammar loaded for + * the element's language. + * + * @param element The element containing the code. + * It must have a class of `language-xxxx` to be processed, where `xxxx` is a valid language identifier. + */ + highlightElement (element: Element, options: HighlightElementOptions = {}) { + const { async, callback } = options; + + // Find language + const language = getLanguage(element); + const languageId = this.components.resolveAlias(language); + const grammar = this.components.getLanguage(languageId); + + // Set language on the element, if not present + setLanguage(element, language); + + // Set language on the parent, for styling + let parent = element.parentElement; + if (parent && parent.nodeName.toLowerCase() === 'pre') { + setLanguage(parent, language); + } + + const code = element.textContent as string; + + const env: HookEnvMap['before-sanity-check'] = { + element, + language, + grammar, + code, + state: new HookState(), + }; + + const insertHighlightedCode = (highlightedCode: string) => { + assertEnv<'before-insert'>(env); + env.highlightedCode = highlightedCode; + this.hooks.run('before-insert', env); + + env.element.innerHTML = env.highlightedCode; + + this.hooks.run('after-highlight', env); + this.hooks.run('complete', env); + callback?.(env.element); + }; + + this.hooks.run('before-sanity-check', env); + + // plugins may change/add the parent/element + parent = env.element.parentElement; + if (parent && parent.nodeName.toLowerCase() === 'pre' && !parent.hasAttribute('tabindex')) { + parent.setAttribute('tabindex', '0'); + } + + if (!env.code) { + this.hooks.run('complete', env); + callback?.(env.element); + return; + } + + this.hooks.run('before-highlight', env); + + if (!env.grammar) { + insertHighlightedCode(htmlEncode(env.code)); + return; + } + + if (async) { + async({ + language: env.language, + code: env.code, + grammar: env.grammar, + }).then(insertHighlightedCode, error => console.log(error)); + } + else { + insertHighlightedCode(this.highlight(env.code, env.language, { grammar: env.grammar })); + } + } + + /** + * Low-level function, only use if you know what you’re doing. It accepts a string of text as input + * and the language definitions to use, and returns a string with the HTML produced. + * + * The following hooks will be run: + * 1. `before-tokenize` + * 2. `after-tokenize` + * 3. `wrap`: On each {@link Token}. + * + * @param text A string with the code to be highlighted. + * @param language The name of the language definition passed to `grammar`. + * @param options An object containing the tokens to use. + * + * Usually a language definition like `Prism.languages.markup`. + * @returns The highlighted HTML. + * @example + * Prism.highlight('var foo = true;', 'javascript'); + */ + highlight (text: string, language: string, options?: HighlightOptions): string { + const languageId = this.components.resolveAlias(language); + const grammar = options?.grammar ?? this.components.getLanguage(languageId); + + const env: HookEnvMap['before-tokenize'] | HookEnvMap['after-tokenize'] = { + code: text, + grammar, + language, + }; + this.hooks.run('before-tokenize', env); + if (!env.grammar) { + throw new Error('The language "' + env.language + '" has no grammar.'); + } + + assertEnv<'after-tokenize'>(env); + env.tokens = this.tokenize(env.code, env.grammar); + this.hooks.run('after-tokenize', env); + + return stringify(env.tokens, env.language, this.hooks); + } + + /** + * This is the heart of Prism, and the most low-level function you can use. It accepts a string of text as input + * and the language definitions to use, and returns an array with the tokenized code. + * + * When the language definition includes nested tokens, the function is called recursively on each of these tokens. + * + * This method could be useful in other contexts as well, as a very crude parser. + * + * @param text A string with the code to be highlighted. + * @param grammar An object containing the tokens to use. + * + * Usually a language definition like `Prism.languages.markup`. + * @returns An array of strings and tokens, a token stream. + * @example + * let code = `var foo = 0;`; + * let tokens = Prism.tokenize(code, Prism.getLanguage('javascript')); + * tokens.forEach(token => { + * if (token instanceof Token && token.type === 'number') { + * console.log(`Found numeric literal: ${token.content}`); + * } + * }); + */ + tokenize (text: string, grammar: Grammar): TokenStream { + const customTokenize = grammar[tokenize]; + if (customTokenize) { + return customTokenize(text, grammar, this); + } + + let restGrammar = resolve(this.components, grammar[rest]); + while (restGrammar) { + grammar = { ...grammar, ...restGrammar }; + restGrammar = resolve(this.components, restGrammar[rest]); + } + + const tokenList = new LinkedList(); + tokenList.addAfter(tokenList.head, text); + + this._matchGrammar(text, tokenList, grammar, tokenList.head, 0); + + return tokenList.toArray(); + } + + private _matchGrammar ( + text: string, + tokenList: LinkedList, + grammar: GrammarTokens, + startNode: LinkedListHeadNode | LinkedListMiddleNode, + startPos: number, + rematch?: RematchOptions + ): void { + for (const token in grammar) { + const tokenValue = grammar[token]; + if (!grammar.hasOwnProperty(token) || !tokenValue) { + continue; + } + + const patterns = Array.isArray(tokenValue) ? tokenValue : [tokenValue]; + + for (let j = 0; j < patterns.length; ++j) { + if (rematch && rematch.cause === `${token},${j}`) { + return; + } + + const patternObj = toGrammarToken(patterns[j]); + let { pattern, lookbehind = false, greedy = false, alias, inside } = patternObj; + const insideGrammar = resolve(this.components, inside); + + if (greedy && !pattern.global) { + // Without the global flag, lastIndex won't work + patternObj.pattern = pattern = RegExp(pattern.source, pattern.flags + 'g'); + } + + for ( + // iterate the token list and keep track of the current token/string position + let currentNode = startNode.next, pos = startPos; + currentNode.next !== null; + pos += currentNode.value.length, currentNode = currentNode.next + ) { + if (rematch && pos >= rematch.reach) { + break; + } + + let str = currentNode.value; + + if (tokenList.length > text.length) { + // Something went terribly wrong, ABORT, ABORT! + return; + } + + if (str instanceof Token) { + continue; + } + + let removeCount = 1; // this is the to parameter of removeBetween + let match; + + if (greedy) { + match = matchPattern(pattern, pos, text, lookbehind); + if (!match || match.index >= text.length) { + break; + } + + const from = match.index; + const to = match.index + match[0].length; + let p = pos; + + // find the node that contains the match + p += currentNode.value.length; + while (from >= p) { + currentNode = currentNode.next; + if (currentNode.next === null) { + throw new Error( + 'The linked list and the actual text have become de-synced' + ); + } + p += currentNode.value.length; + } + // adjust pos (and p) + p -= currentNode.value.length; + pos = p; + + // the current node is a Token, then the match starts inside another Token, which is invalid + if (currentNode.value instanceof Token) { + continue; + } + + // find the last node which is affected by this match + let k: + | LinkedListMiddleNode + | LinkedListTailNode = currentNode; + for ( + ; + k.next !== null && (p < to || typeof k.value === 'string'); + k = k.next + ) { + removeCount++; + p += k.value.length; + } + removeCount--; + + // replace with the new match + str = text.slice(pos, p); + match.index -= pos; + } + else { + match = matchPattern(pattern, 0, str, lookbehind); + if (!match) { + continue; + } + } + + // eslint-disable-next-line no-redeclare + const from = match.index; + const matchStr = match[0]; + const before = str.slice(0, from); + const after = str.slice(from + matchStr.length); + + const reach = pos + str.length; + if (rematch && reach > rematch.reach) { + rematch.reach = reach; + } + + let removeFrom = currentNode.prev; + + if (before) { + removeFrom = tokenList.addAfter(removeFrom, before); + pos += before.length; + } + + tokenList.removeRange(removeFrom, removeCount); + + const wrapped = new Token( + token, + insideGrammar ? this.tokenize(matchStr, insideGrammar) : matchStr, + alias, + matchStr + ); + currentNode = tokenList.addAfter(removeFrom, wrapped); + + if (after) { + tokenList.addAfter(currentNode, after); + } + + if (removeCount > 1) { + // at least one Token object was removed, so we have to do some rematching + // this can only happen if the current pattern is greedy + + const nestedRematch: RematchOptions = { + cause: `${token},${j}`, + reach, + }; + this._matchGrammar( + text, + tokenList, + grammar, + currentNode.prev, + pos, + nestedRematch + ); + + // the reach might have been extended because of the rematching + if (rematch && nestedRematch.reach > rematch.reach) { + rematch.reach = nestedRematch.reach; + } + } + } + } + } + } +} + +interface RematchOptions { + cause: string; + reach: number; +} + +export interface AsyncHighlightingData { + language: string; + code: string; + grammar: Grammar; +} +export type AsyncHighlighter = (data: AsyncHighlightingData) => Promise; + + + +export interface HighlightOptions { + grammar?: Grammar; +} + +function assertEnv (env: unknown): asserts env is HookEnvMap[T] { + /* noop */ +} + +function matchPattern (pattern: RegExp, pos: number, text: string, lookbehind: boolean) { + pattern.lastIndex = pos; + const match = pattern.exec(text); + if (match && lookbehind && match[1]) { + // change the match to remove the text matched by the Prism lookbehind group + const lookbehindLength = match[1].length; + match.index += lookbehindLength; + match[0] = match[0].slice(lookbehindLength); + } + return match; +} + +/** + * Converts the given token or token stream to an HTML representation. + * + * The following hooks will be run: + * 1. `wrap`: On each {@link Token}. + * + * @param o The token or token stream to be converted. + * @param language The name of current language. + * @returns The HTML representation of the token or token stream. + */ +function stringify (o: string | Token | TokenStream, language: string, hooks: Hooks): string { + if (typeof o === 'string') { + return htmlEncode(o); + } + if (Array.isArray(o)) { + let s = ''; + o.forEach(e => { + s += stringify(e, language, hooks); + }); + return s; + } + + const env: HookEnvMap['wrap'] = { + type: o.type, + content: stringify(o.content, language, hooks), + tag: 'span', + classes: ['token', o.type], + attributes: {}, + language, + }; + + const aliases = o.alias; + if (aliases) { + if (Array.isArray(aliases)) { + env.classes.push(...aliases); + } + else { + env.classes.push(aliases); + } + } + + hooks.run('wrap', env); + + let attributes = ''; + for (const name in env.attributes) { + attributes += + ' ' + name + '="' + (env.attributes[name] || '').replace(/"/g, '"') + '"'; + } + + return ( + '<' + + env.tag + + ' class="' + + env.classes.join(' ') + + '"' + + attributes + + '>' + + env.content + + '' + ); +} + +function toGrammarToken (pattern: GrammarToken | RegExpLike): GrammarToken { + if (!pattern.pattern) { + return { pattern }; + } + else { + return pattern; + } +} + +function resolve ( + components: Registry, + reference: Grammar | string | null | undefined +): Grammar | undefined { + if (reference) { + if (typeof reference === 'string') { + return components.getLanguage(reference); + } + return reference; + } + return undefined; +} diff --git a/src/core/prism.ts b/src/core/prism.ts index 19fd929d3a..c2f1c519ca 100644 --- a/src/core/prism.ts +++ b/src/core/prism.ts @@ -1,500 +1,4 @@ -import { getLanguage, setLanguage } from '../shared/dom-util'; -import { rest, tokenize } from '../shared/symbols'; -import { htmlEncode } from '../shared/util'; -import { HookState } from './hook-state'; -import { Hooks } from './hooks'; -import { LinkedList } from './linked-list'; -import { Registry } from './registry'; -import { Token } from './token'; -import type { KnownPlugins } from '../known-plugins'; -import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types'; -import type { LinkedListHeadNode, LinkedListMiddleNode, LinkedListTailNode } from './linked-list'; -import type { TokenStream } from './token'; -import {highlightAll, HighlightAllOptions} from './highlight-all'; - -/** - * Prism: Lightweight, robust, elegant syntax highlighting - * - * @license MIT - * @author Lea Verou and contributors - */ -export class Prism { - hooks = new Hooks(); - components = new Registry(this); - plugins: Partial & KnownPlugins> = {}; - - /** - * This is the most high-level function in Prism’s API. - * It queries all the elements that have a `.language-xxxx` class and then calls {@link Prism#highlightElement} on - * each one of them. - * - * The following hooks will be run: - * 1. `before-highlightall` - * 2. `before-all-elements-highlight` - * 3. All hooks of {@link Prism#highlightElement} for each element. - */ - highlightAll (options: HighlightAllOptions = {}) { - return highlightAll.call(this, options); - } - - /** - * Highlights the code inside a single element. - * - * The following hooks will be run: - * 1. `before-sanity-check` - * 2. `before-highlight` - * 3. All hooks of {@link Prism#highlight}. These hooks will be run by an asynchronous worker if `async` is `true`. - * 4. `before-insert` - * 5. `after-highlight` - * 6. `complete` - * - * Some the above hooks will be skipped if the element doesn't contain any text or there is no grammar loaded for - * the element's language. - * - * @param element The element containing the code. - * It must have a class of `language-xxxx` to be processed, where `xxxx` is a valid language identifier. - */ - highlightElement (element: Element, options: HighlightElementOptions = {}) { - const { async, callback } = options; - - // Find language - const language = getLanguage(element); - const languageId = this.components.resolveAlias(language); - const grammar = this.components.getLanguage(languageId); - - // Set language on the element, if not present - setLanguage(element, language); - - // Set language on the parent, for styling - let parent = element.parentElement; - if (parent && parent.nodeName.toLowerCase() === 'pre') { - setLanguage(parent, language); - } - - const code = element.textContent as string; - - const env: HookEnvMap['before-sanity-check'] = { - element, - language, - grammar, - code, - state: new HookState(), - }; - - const insertHighlightedCode = (highlightedCode: string) => { - assertEnv<'before-insert'>(env); - env.highlightedCode = highlightedCode; - this.hooks.run('before-insert', env); - - env.element.innerHTML = env.highlightedCode; - - this.hooks.run('after-highlight', env); - this.hooks.run('complete', env); - callback?.(env.element); - }; - - this.hooks.run('before-sanity-check', env); - - // plugins may change/add the parent/element - parent = env.element.parentElement; - if (parent && parent.nodeName.toLowerCase() === 'pre' && !parent.hasAttribute('tabindex')) { - parent.setAttribute('tabindex', '0'); - } - - if (!env.code) { - this.hooks.run('complete', env); - callback?.(env.element); - return; - } - - this.hooks.run('before-highlight', env); - - if (!env.grammar) { - insertHighlightedCode(htmlEncode(env.code)); - return; - } - - if (async) { - async({ - language: env.language, - code: env.code, - grammar: env.grammar, - }).then(insertHighlightedCode, error => console.log(error)); - } - else { - insertHighlightedCode(this.highlight(env.code, env.language, { grammar: env.grammar })); - } - } - - /** - * Low-level function, only use if you know what you’re doing. It accepts a string of text as input - * and the language definitions to use, and returns a string with the HTML produced. - * - * The following hooks will be run: - * 1. `before-tokenize` - * 2. `after-tokenize` - * 3. `wrap`: On each {@link Token}. - * - * @param text A string with the code to be highlighted. - * @param language The name of the language definition passed to `grammar`. - * @param options An object containing the tokens to use. - * - * Usually a language definition like `Prism.languages.markup`. - * @returns The highlighted HTML. - * @example - * Prism.highlight('var foo = true;', 'javascript'); - */ - highlight (text: string, language: string, options?: HighlightOptions): string { - const languageId = this.components.resolveAlias(language); - const grammar = options?.grammar ?? this.components.getLanguage(languageId); - - const env: HookEnvMap['before-tokenize'] | HookEnvMap['after-tokenize'] = { - code: text, - grammar, - language, - }; - this.hooks.run('before-tokenize', env); - if (!env.grammar) { - throw new Error('The language "' + env.language + '" has no grammar.'); - } - - assertEnv<'after-tokenize'>(env); - env.tokens = this.tokenize(env.code, env.grammar); - this.hooks.run('after-tokenize', env); - - return stringify(env.tokens, env.language, this.hooks); - } - - /** - * This is the heart of Prism, and the most low-level function you can use. It accepts a string of text as input - * and the language definitions to use, and returns an array with the tokenized code. - * - * When the language definition includes nested tokens, the function is called recursively on each of these tokens. - * - * This method could be useful in other contexts as well, as a very crude parser. - * - * @param text A string with the code to be highlighted. - * @param grammar An object containing the tokens to use. - * - * Usually a language definition like `Prism.languages.markup`. - * @returns An array of strings and tokens, a token stream. - * @example - * let code = `var foo = 0;`; - * let tokens = Prism.tokenize(code, Prism.getLanguage('javascript')); - * tokens.forEach(token => { - * if (token instanceof Token && token.type === 'number') { - * console.log(`Found numeric literal: ${token.content}`); - * } - * }); - */ - tokenize (text: string, grammar: Grammar): TokenStream { - const customTokenize = grammar[tokenize]; - if (customTokenize) { - return customTokenize(text, grammar, this); - } - - let restGrammar = resolve(this.components, grammar[rest]); - while (restGrammar) { - grammar = { ...grammar, ...restGrammar }; - restGrammar = resolve(this.components, restGrammar[rest]); - } - - const tokenList = new LinkedList(); - tokenList.addAfter(tokenList.head, text); - - this._matchGrammar(text, tokenList, grammar, tokenList.head, 0); - - return tokenList.toArray(); - } - - private _matchGrammar ( - text: string, - tokenList: LinkedList, - grammar: GrammarTokens, - startNode: LinkedListHeadNode | LinkedListMiddleNode, - startPos: number, - rematch?: RematchOptions - ): void { - for (const token in grammar) { - const tokenValue = grammar[token]; - if (!grammar.hasOwnProperty(token) || !tokenValue) { - continue; - } - - const patterns = Array.isArray(tokenValue) ? tokenValue : [tokenValue]; - - for (let j = 0; j < patterns.length; ++j) { - if (rematch && rematch.cause === `${token},${j}`) { - return; - } - - const patternObj = toGrammarToken(patterns[j]); - let { pattern, lookbehind = false, greedy = false, alias, inside } = patternObj; - const insideGrammar = resolve(this.components, inside); - - if (greedy && !pattern.global) { - // Without the global flag, lastIndex won't work - patternObj.pattern = pattern = RegExp(pattern.source, pattern.flags + 'g'); - } - - for ( - // iterate the token list and keep track of the current token/string position - let currentNode = startNode.next, pos = startPos; - currentNode.next !== null; - pos += currentNode.value.length, currentNode = currentNode.next - ) { - if (rematch && pos >= rematch.reach) { - break; - } - - let str = currentNode.value; - - if (tokenList.length > text.length) { - // Something went terribly wrong, ABORT, ABORT! - return; - } - - if (str instanceof Token) { - continue; - } - - let removeCount = 1; // this is the to parameter of removeBetween - let match; - - if (greedy) { - match = matchPattern(pattern, pos, text, lookbehind); - if (!match || match.index >= text.length) { - break; - } - - const from = match.index; - const to = match.index + match[0].length; - let p = pos; - - // find the node that contains the match - p += currentNode.value.length; - while (from >= p) { - currentNode = currentNode.next; - if (currentNode.next === null) { - throw new Error( - 'The linked list and the actual text have become de-synced' - ); - } - p += currentNode.value.length; - } - // adjust pos (and p) - p -= currentNode.value.length; - pos = p; - - // the current node is a Token, then the match starts inside another Token, which is invalid - if (currentNode.value instanceof Token) { - continue; - } - - // find the last node which is affected by this match - let k: - | LinkedListMiddleNode - | LinkedListTailNode = currentNode; - for ( - ; - k.next !== null && (p < to || typeof k.value === 'string'); - k = k.next - ) { - removeCount++; - p += k.value.length; - } - removeCount--; - - // replace with the new match - str = text.slice(pos, p); - match.index -= pos; - } - else { - match = matchPattern(pattern, 0, str, lookbehind); - if (!match) { - continue; - } - } - - // eslint-disable-next-line no-redeclare - const from = match.index; - const matchStr = match[0]; - const before = str.slice(0, from); - const after = str.slice(from + matchStr.length); - - const reach = pos + str.length; - if (rematch && reach > rematch.reach) { - rematch.reach = reach; - } - - let removeFrom = currentNode.prev; - - if (before) { - removeFrom = tokenList.addAfter(removeFrom, before); - pos += before.length; - } - - tokenList.removeRange(removeFrom, removeCount); - - const wrapped = new Token( - token, - insideGrammar ? this.tokenize(matchStr, insideGrammar) : matchStr, - alias, - matchStr - ); - currentNode = tokenList.addAfter(removeFrom, wrapped); - - if (after) { - tokenList.addAfter(currentNode, after); - } - - if (removeCount > 1) { - // at least one Token object was removed, so we have to do some rematching - // this can only happen if the current pattern is greedy - - const nestedRematch: RematchOptions = { - cause: `${token},${j}`, - reach, - }; - this._matchGrammar( - text, - tokenList, - grammar, - currentNode.prev, - pos, - nestedRematch - ); - - // the reach might have been extended because of the rematching - if (rematch && nestedRematch.reach > rematch.reach) { - rematch.reach = nestedRematch.reach; - } - } - } - } - } - } -} - -interface RematchOptions { - cause: string; - reach: number; -} - -export interface AsyncHighlightingData { - language: string; - code: string; - grammar: Grammar; -} -export type AsyncHighlighter = (data: AsyncHighlightingData) => Promise; - - - -export interface HighlightOptions { - grammar?: Grammar; -} - -function assertEnv (env: unknown): asserts env is HookEnvMap[T] { - /* noop */ -} - -function matchPattern (pattern: RegExp, pos: number, text: string, lookbehind: boolean) { - pattern.lastIndex = pos; - const match = pattern.exec(text); - if (match && lookbehind && match[1]) { - // change the match to remove the text matched by the Prism lookbehind group - const lookbehindLength = match[1].length; - match.index += lookbehindLength; - match[0] = match[0].slice(lookbehindLength); - } - return match; -} - -/** - * Converts the given token or token stream to an HTML representation. - * - * The following hooks will be run: - * 1. `wrap`: On each {@link Token}. - * - * @param o The token or token stream to be converted. - * @param language The name of current language. - * @returns The HTML representation of the token or token stream. - */ -function stringify (o: string | Token | TokenStream, language: string, hooks: Hooks): string { - if (typeof o === 'string') { - return htmlEncode(o); - } - if (Array.isArray(o)) { - let s = ''; - o.forEach(e => { - s += stringify(e, language, hooks); - }); - return s; - } - - const env: HookEnvMap['wrap'] = { - type: o.type, - content: stringify(o.content, language, hooks), - tag: 'span', - classes: ['token', o.type], - attributes: {}, - language, - }; - - const aliases = o.alias; - if (aliases) { - if (Array.isArray(aliases)) { - env.classes.push(...aliases); - } - else { - env.classes.push(aliases); - } - } - - hooks.run('wrap', env); - - let attributes = ''; - for (const name in env.attributes) { - attributes += - ' ' + name + '="' + (env.attributes[name] || '').replace(/"/g, '"') + '"'; - } - - return ( - '<' + - env.tag + - ' class="' + - env.classes.join(' ') + - '"' + - attributes + - '>' + - env.content + - '' - ); -} - -function toGrammarToken (pattern: GrammarToken | RegExpLike): GrammarToken { - if (!pattern.pattern) { - return { pattern }; - } - else { - return pattern; - } -} - -function resolve ( - components: Registry, - reference: Grammar | string | null | undefined -): Grammar | undefined { - if (reference) { - if (typeof reference === 'string') { - return components.getLanguage(reference); - } - return reference; - } - return undefined; -} +import Prism from "./prism-class"; /** * Prism singleton. diff --git a/src/core/registry.ts b/src/core/registry.ts index e53900b0a5..22fc6380a9 100644 --- a/src/core/registry.ts +++ b/src/core/registry.ts @@ -1,7 +1,7 @@ import { extend } from '../shared/language-util'; import { forEach, kebabToCamelCase } from '../shared/util'; import type { ComponentProto, Grammar } from '../types'; -import type { Prism } from './prism'; +import type { Prism } from './prism-class'; interface Entry { proto: ComponentProto; diff --git a/src/global.ts b/src/global.ts index 44e5a8d040..6ef69a00e5 100644 --- a/src/global.ts +++ b/src/global.ts @@ -1,4 +1,4 @@ -import globalPrism, { Prism } from './core/prism'; +import globalPrism, { Prism } from './core/prism-class'; declare global { var Prism: Prism | undefined diff --git a/src/types.d.ts b/src/types.d.ts index 3db3270e6f..c597bb22e2 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -1,4 +1,4 @@ -import type { Prism } from './core/prism'; +import type { Prism } from './core/prism-class'; import type { TokenStream } from './core/token'; import type { KnownPlugins } from './known-plugins'; import type { rest, tokenize } from './shared/symbols'; From d1097d82ccce8ff686cedf699e1b7df93402740f Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sat, 26 Apr 2025 17:40:03 -0400 Subject: [PATCH 005/101] Finish removing HookState --- src/core/hooks.ts | 2 +- src/core/prism-class.ts | 2 -- src/plugins/command-line/prism-command-line.ts | 9 +++------ src/plugins/keep-markup/prism-keep-markup.ts | 1 - 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/core/hooks.ts b/src/core/hooks.ts index 0cfe80db41..2305ae2c00 100644 --- a/src/core/hooks.ts +++ b/src/core/hooks.ts @@ -56,4 +56,4 @@ export class Hooks { } } -export type HookCallback = (env?: Record) => void; +export type HookCallback = (env: Record) => void; diff --git a/src/core/prism-class.ts b/src/core/prism-class.ts index ddff195980..26fb5116c2 100644 --- a/src/core/prism-class.ts +++ b/src/core/prism-class.ts @@ -1,7 +1,6 @@ import { getLanguage, setLanguage } from '../shared/dom-util'; import { rest, tokenize } from '../shared/symbols'; import { htmlEncode } from '../shared/util'; -import { HookState } from './hook-state'; import { Hooks } from './hooks'; import { LinkedList } from './linked-list'; import { Registry } from './registry'; @@ -78,7 +77,6 @@ export default class Prism { language, grammar, code, - state: new HookState(), }; const insertHighlightedCode = (highlightedCode: string) => { diff --git a/src/plugins/command-line/prism-command-line.ts b/src/plugins/command-line/prism-command-line.ts index 1d77db0d3f..3b8c6692e6 100644 --- a/src/plugins/command-line/prism-command-line.ts +++ b/src/plugins/command-line/prism-command-line.ts @@ -1,14 +1,11 @@ import { getParentPre } from '../../shared/dom-util'; import { addHooks } from '../../shared/hooks-util'; import { htmlEncode } from '../../shared/util'; -import type { StateKey } from '../../core/hook-state'; import type { PluginProto } from '../../types'; const CLASS_PATTERN = /(?:^|\s)command-line(?:\s|$)/; const PROMPT_CLASS = 'command-line-prompt'; -const commandLineKey: StateKey = 'command-line data'; - interface CommandLineInfo { complete?: boolean; numberOfLines?: number; @@ -21,7 +18,7 @@ export default { effect(Prism) { return addHooks(Prism.hooks, { 'before-highlight': (env) => { - const commandLine = env.state.get(commandLineKey, {}); + const commandLine = env.commandLine ?? {}; if (commandLine.complete || !env.code) { commandLine.complete = true; @@ -110,7 +107,7 @@ export default { env.code = codeLines.join('\n'); }, 'before-insert': (env) => { - const commandLine = env.state.get(commandLineKey, {}); + const commandLine = env.commandLine ?? {}; if (commandLine.complete) { return; } @@ -133,7 +130,7 @@ export default { env.highlightedCode = codeLines.join('\n'); }, 'complete': (env) => { - const commandLine = env.state.get(commandLineKey, {}); + const commandLine = env.commandLine ?? {}; if (commandLine.complete) { return; } diff --git a/src/plugins/keep-markup/prism-keep-markup.ts b/src/plugins/keep-markup/prism-keep-markup.ts index 6c104bac14..2d32774cea 100644 --- a/src/plugins/keep-markup/prism-keep-markup.ts +++ b/src/plugins/keep-markup/prism-keep-markup.ts @@ -1,6 +1,5 @@ import { isActive } from '../../shared/dom-util'; import { addHooks } from '../../shared/hooks-util'; -import type { StateKey } from '../../core/hook-state'; import type { PluginProto } from '../../types'; function isElement(child: ChildNode): child is Element { From e3e5db21ffeac441dc42547c3120b90eac2bd89e Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sat, 26 Apr 2025 17:41:18 -0400 Subject: [PATCH 006/101] Update prism-class.ts --- src/core/prism-class.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/core/prism-class.ts b/src/core/prism-class.ts index 26fb5116c2..fc48a6c63b 100644 --- a/src/core/prism-class.ts +++ b/src/core/prism-class.ts @@ -23,14 +23,7 @@ export default class Prism { plugins: Partial & KnownPlugins> = {}; /** - * This is the most high-level function in Prism’s API. - * It queries all the elements that have a `.language-xxxx` class and then calls {@link Prism#highlightElement} on - * each one of them. - * - * The following hooks will be run: - * 1. `before-highlightall` - * 2. `before-all-elements-highlight` - * 3. All hooks of {@link Prism#highlightElement} for each element. + * See {@link highlightAll}. */ highlightAll (options: HighlightAllOptions = {}) { return highlightAll.call(this, options); From c8662614ee88cef5d5272ff7e56d6b0fc455d250 Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sat, 26 Apr 2025 17:42:18 -0400 Subject: [PATCH 007/101] Delete hook-state.ts --- src/core/hook-state.ts | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 src/core/hook-state.ts diff --git a/src/core/hook-state.ts b/src/core/hook-state.ts deleted file mode 100644 index 011422cd4c..0000000000 --- a/src/core/hook-state.ts +++ /dev/null @@ -1,25 +0,0 @@ -export type StateKey = (string | symbol) & { __keyType?: T }; - -/** - * A simple typed map from some key to its data. - */ -export class HookState { - private _data = new Map(); - - has (key: StateKey<{}>): boolean { - return this._data.has(key); - } - - get (key: StateKey, defaultValue: T) { - let current = this._data.get(key); - if (current === undefined) { - current = defaultValue; - this._data.set(key, current); - } - return current as T; - } - - set (key: StateKey, value: T): void { - this._data.set(key, value); - } -} From 8ce6c695f0b852bed81ff39e9ba24bcc0c7ee211 Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sat, 26 Apr 2025 18:34:36 -0400 Subject: [PATCH 008/101] Move more functions out of prism-class --- src/core/highlight-all.ts | 43 ++++------- src/core/highlight-element.ts | 114 +++++++++++++++++++++++++++++ src/core/highlight.ts | 55 ++++++++++++++ src/core/prism-class.ts | 134 ++-------------------------------- src/core/prism.ts | 3 + src/core/stringify.ts | 60 +++++++++++++++ 6 files changed, 253 insertions(+), 156 deletions(-) create mode 100644 src/core/highlight-element.ts create mode 100644 src/core/highlight.ts create mode 100644 src/core/stringify.ts diff --git a/src/core/highlight-all.ts b/src/core/highlight-all.ts index 7fb9506f1b..9bbd1ca086 100644 --- a/src/core/highlight-all.ts +++ b/src/core/highlight-all.ts @@ -1,6 +1,5 @@ -import Prism from './prism-class'; -import prism from "./prism"; -import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types'; +import singleton, { type Prism } from './prism'; +import type { HighlightElementOptions } from './highlight-element'; /** * This is the most high-level function in Prism’s API. @@ -13,46 +12,30 @@ import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types' * 3. All hooks of {@link Prism#highlightElement} for each element. */ export function highlightAll (this: Prism, options: HighlightAllOptions = {}) { - const context = this ?? prism; + const prism = this ?? singleton; const { root, async, callback } = options; - const env: Record = - { - callback, - root: root ?? document, - selector: - 'code[class*="language-"], [class*="language-"] code, code[class*="lang-"], [class*="lang-"] code', - }; + const env: Record = { + callback, + root: root ?? document, + selector: + 'code[class*="language-"], [class*="language-"] code, code[class*="lang-"], [class*="lang-"] code', + }; - context.hooks.run('before-highlightall', env); + prism.hooks.run('before-highlightall', env); env.elements = [...env.root.querySelectorAll(env.selector)]; - context.hooks.run('before-all-elements-highlight', env); + prism.hooks.run('before-all-elements-highlight', env); for (const element of env.elements) { - context.highlightElement(element, { async, callback: env.callback }); + prism.highlightElement(element, { async, callback: env.callback }); } } -export interface HighlightAllOptions { +export interface HighlightAllOptions extends HighlightElementOptions { /** * The root element, whose descendants that have a `.language-xxxx` class will be highlighted. */ root?: ParentNode; - async?: AsyncHighlighter; - /** - * An optional callback to be invoked on each element after its highlighting is done. - * - * @see HighlightElementOptions#callback - */ - callback?: (element: Element) => void; } - -export interface AsyncHighlightingData { - language: string; - code: string; - grammar: Grammar; -} -export type AsyncHighlighter = (data: AsyncHighlightingData) => Promise; - diff --git a/src/core/highlight-element.ts b/src/core/highlight-element.ts new file mode 100644 index 0000000000..a682e48b79 --- /dev/null +++ b/src/core/highlight-element.ts @@ -0,0 +1,114 @@ +import { getLanguage, setLanguage } from '../shared/dom-util'; +import { htmlEncode } from '../shared/util'; +import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types'; +import singleton, { type Prism } from './prism'; + +/** + * Highlights the code inside a single element. + * + * The following hooks will be run: + * 1. `before-sanity-check` + * 2. `before-highlight` + * 3. All hooks of {@link Prism#highlight}. These hooks will be run by an asynchronous worker if `async` is `true`. + * 4. `before-insert` + * 5. `after-highlight` + * 6. `complete` + * + * Some the above hooks will be skipped if the element doesn't contain any text or there is no grammar loaded for + * the element's language. + * + * @param element The element containing the code. + * It must have a class of `language-xxxx` to be processed, where `xxxx` is a valid language identifier. + */ +export function highlightElement ( + this: Prism, + element: Element, + options: HighlightElementOptions = {} +) { + const prism = this ?? singleton; + const { async, callback } = options; + + // Find language + const language = getLanguage(element); + const languageId = prism.components.resolveAlias(language); + const grammar = prism.components.getLanguage(languageId); + + // Set language on the element, if not present + setLanguage(element, language); + + // Set language on the parent, for styling + let parent = element.parentElement; + if (parent && parent.nodeName.toLowerCase() === 'pre') { + setLanguage(parent, language); + } + + const code = element.textContent as string; + + const env: Record = { + element, + language, + grammar, + code, + }; + + const insertHighlightedCode = (highlightedCode: string) => { + env.highlightedCode = highlightedCode; + prism.hooks.run('before-insert', env); + + env.element.innerHTML = env.highlightedCode; + + prism.hooks.run('after-highlight', env); + prism.hooks.run('complete', env); + callback?.(env.element); + }; + + prism.hooks.run('before-sanity-check', env); + + // plugins may change/add the parent/element + parent = env.element.parentElement; + if (parent && parent.nodeName.toLowerCase() === 'pre' && !parent.hasAttribute('tabindex')) { + parent.setAttribute('tabindex', '0'); + } + + if (!env.code) { + prism.hooks.run('complete', env); + callback?.(env.element); + return; + } + + prism.hooks.run('before-highlight', env); + + if (!env.grammar) { + insertHighlightedCode(htmlEncode(env.code)); + return; + } + + if (async) { + async({ + language: env.language, + code: env.code, + grammar: env.grammar, + }).then(insertHighlightedCode, error => console.log(error)); + } + else { + insertHighlightedCode(prism.highlight(env.code, env.language, { grammar: env.grammar })); + } +} + +export interface HighlightElementOptions { + async?: AsyncHighlighter; + /** + * An optional callback to be invoked after the highlighting is done. + * Mostly useful when `async` is `true`, since in that case, the highlighting is done asynchronously. + * + * @param element The element successfully highlighted. + */ + callback?: (element: Element) => void; +} + +export interface AsyncHighlightingData { + language: string; + code: string; + grammar: Grammar; +} +export type AsyncHighlighter = (data: AsyncHighlightingData) => Promise; diff --git a/src/core/highlight.ts b/src/core/highlight.ts new file mode 100644 index 0000000000..f9c0e44e03 --- /dev/null +++ b/src/core/highlight.ts @@ -0,0 +1,55 @@ +import { Token } from './token'; +import singleton, { type Prism } from './prism'; +import { stringify } from './stringify'; +import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types'; + +/** + * Low-level function, only use if you know what you’re doing. It accepts a string of text as input + * and the language definitions to use, and returns a string with the HTML produced. + * + * The following hooks will be run: + * 1. `before-tokenize` + * 2. `after-tokenize` + * 3. `wrap`: On each {@link Token}. + * + * @param text A string with the code to be highlighted. + * @param language The name of the language definition passed to `grammar`. + * @param options An object containing the tokens to use. + * + * Usually a language definition like `Prism.languages.markup`. + * @returns The highlighted HTML. + * @example + * Prism.highlight('var foo = true;', 'javascript'); + */ +export function highlight ( + this: Prism, + text: string, + language: string, + options?: HighlightOptions +): string { + const prism = this ?? singleton; + + const languageId = prism.components.resolveAlias(language); + const grammar = options?.grammar ?? prism.components.getLanguage(languageId); + + const env: Record | Record = { + code: text, + grammar, + language, + }; + + prism.hooks.run('before-tokenize', env); + + if (!env.grammar) { + throw new Error('The language "' + env.language + '" has no grammar.'); + } + + env.tokens = prism.tokenize(env.code, env.grammar); + prism.hooks.run('after-tokenize', env); + + return stringify(env.tokens, env.language, prism.hooks); +} + +export interface HighlightOptions { + grammar?: Grammar; +} diff --git a/src/core/prism-class.ts b/src/core/prism-class.ts index fc48a6c63b..0ffe453889 100644 --- a/src/core/prism-class.ts +++ b/src/core/prism-class.ts @@ -1,4 +1,3 @@ -import { getLanguage, setLanguage } from '../shared/dom-util'; import { rest, tokenize } from '../shared/symbols'; import { htmlEncode } from '../shared/util'; import { Hooks } from './hooks'; @@ -9,7 +8,9 @@ import type { KnownPlugins } from '../known-plugins'; import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types'; import type { LinkedListHeadNode, LinkedListMiddleNode, LinkedListTailNode } from './linked-list'; import type { TokenStream } from './token'; -import {highlightAll, HighlightAllOptions} from './highlight-all'; +import { highlightAll, type HighlightAllOptions } from './highlight-all'; +import { highlightElement, type HighlightElementOptions } from './highlight-element'; +import { highlight, type HighlightOptions } from './highlight'; /** * Prism: Lightweight, robust, elegant syntax highlighting @@ -30,130 +31,17 @@ export default class Prism { } /** - * Highlights the code inside a single element. - * - * The following hooks will be run: - * 1. `before-sanity-check` - * 2. `before-highlight` - * 3. All hooks of {@link Prism#highlight}. These hooks will be run by an asynchronous worker if `async` is `true`. - * 4. `before-insert` - * 5. `after-highlight` - * 6. `complete` - * - * Some the above hooks will be skipped if the element doesn't contain any text or there is no grammar loaded for - * the element's language. - * - * @param element The element containing the code. - * It must have a class of `language-xxxx` to be processed, where `xxxx` is a valid language identifier. + * See {@link highlightElement} */ highlightElement (element: Element, options: HighlightElementOptions = {}) { - const { async, callback } = options; - - // Find language - const language = getLanguage(element); - const languageId = this.components.resolveAlias(language); - const grammar = this.components.getLanguage(languageId); - - // Set language on the element, if not present - setLanguage(element, language); - - // Set language on the parent, for styling - let parent = element.parentElement; - if (parent && parent.nodeName.toLowerCase() === 'pre') { - setLanguage(parent, language); - } - - const code = element.textContent as string; - - const env: HookEnvMap['before-sanity-check'] = { - element, - language, - grammar, - code, - }; - - const insertHighlightedCode = (highlightedCode: string) => { - assertEnv<'before-insert'>(env); - env.highlightedCode = highlightedCode; - this.hooks.run('before-insert', env); - - env.element.innerHTML = env.highlightedCode; - - this.hooks.run('after-highlight', env); - this.hooks.run('complete', env); - callback?.(env.element); - }; - - this.hooks.run('before-sanity-check', env); - - // plugins may change/add the parent/element - parent = env.element.parentElement; - if (parent && parent.nodeName.toLowerCase() === 'pre' && !parent.hasAttribute('tabindex')) { - parent.setAttribute('tabindex', '0'); - } - - if (!env.code) { - this.hooks.run('complete', env); - callback?.(env.element); - return; - } - - this.hooks.run('before-highlight', env); - - if (!env.grammar) { - insertHighlightedCode(htmlEncode(env.code)); - return; - } - - if (async) { - async({ - language: env.language, - code: env.code, - grammar: env.grammar, - }).then(insertHighlightedCode, error => console.log(error)); - } - else { - insertHighlightedCode(this.highlight(env.code, env.language, { grammar: env.grammar })); - } + return highlightElement.call(this, element, options); } /** - * Low-level function, only use if you know what you’re doing. It accepts a string of text as input - * and the language definitions to use, and returns a string with the HTML produced. - * - * The following hooks will be run: - * 1. `before-tokenize` - * 2. `after-tokenize` - * 3. `wrap`: On each {@link Token}. - * - * @param text A string with the code to be highlighted. - * @param language The name of the language definition passed to `grammar`. - * @param options An object containing the tokens to use. - * - * Usually a language definition like `Prism.languages.markup`. - * @returns The highlighted HTML. - * @example - * Prism.highlight('var foo = true;', 'javascript'); + * See {@link highlight} */ highlight (text: string, language: string, options?: HighlightOptions): string { - const languageId = this.components.resolveAlias(language); - const grammar = options?.grammar ?? this.components.getLanguage(languageId); - - const env: HookEnvMap['before-tokenize'] | HookEnvMap['after-tokenize'] = { - code: text, - grammar, - language, - }; - this.hooks.run('before-tokenize', env); - if (!env.grammar) { - throw new Error('The language "' + env.language + '" has no grammar.'); - } - - assertEnv<'after-tokenize'>(env); - env.tokens = this.tokenize(env.code, env.grammar); - this.hooks.run('after-tokenize', env); - - return stringify(env.tokens, env.language, this.hooks); + return highlight.call(this, text, language, options); } /** @@ -379,16 +267,10 @@ export interface AsyncHighlightingData { } export type AsyncHighlighter = (data: AsyncHighlightingData) => Promise; - - export interface HighlightOptions { grammar?: Grammar; } -function assertEnv (env: unknown): asserts env is HookEnvMap[T] { - /* noop */ -} - function matchPattern (pattern: RegExp, pos: number, text: string, lookbehind: boolean) { pattern.lastIndex = pos; const match = pattern.exec(text); @@ -423,7 +305,7 @@ function stringify (o: string | Token | TokenStream, language: string, hooks: Ho return s; } - const env: HookEnvMap['wrap'] = { + const env: Record = { type: o.type, content: stringify(o.content, language, hooks), tag: 'span', diff --git a/src/core/prism.ts b/src/core/prism.ts index c2f1c519ca..c02a689607 100644 --- a/src/core/prism.ts +++ b/src/core/prism.ts @@ -9,3 +9,6 @@ import Prism from "./prism-class"; * Any imported plugins and languages will automatically be added to this instance. */ export default new Prism(); + +/** Re-export Prism class so they can be imported together */ +export { Prism }; diff --git a/src/core/stringify.ts b/src/core/stringify.ts new file mode 100644 index 0000000000..2dda710378 --- /dev/null +++ b/src/core/stringify.ts @@ -0,0 +1,60 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +import { htmlEncode } from '../shared/util'; +import type { Hooks } from './hooks'; +import type { Token } from './token'; +import type { TokenStream } from './token'; + +/** + * Converts the given token or token stream to an HTML representation. + * + * The following hooks will be run: + * 1. `wrap`: On each {@link Token}. + * + * @param o The token or token stream to be converted. + * @param language The name of current language. + * @returns The HTML representation of the token or token stream. + */ +function stringify (o: string | Token | TokenStream, language: string, hooks: Hooks): string { + if (typeof o === 'string') { + return htmlEncode(o); + } + if (Array.isArray(o)) { + let s = ''; + o.forEach(e => { + s += stringify(e, language, hooks); + }); + return s; + } + + const env: Record & { classes: string[]; attributes: Record } = { + type: o.type, + content: stringify(o.content, language, hooks), + tag: 'span', + classes: ['token', o.type], + attributes: {}, + language, + }; + + const aliases = o.alias; + if (aliases) { + if (Array.isArray(aliases)) { + env.classes.push(...aliases); + } + else { + env.classes.push(aliases); + } + } + + hooks.run('wrap', env); + + const attributes = + Object.entries(env.attributes) + .map(([name, value]) => ` ${name}=${(value ?? '').replace(/"/g, '"')}"`) + .join('') || ''; + + return `<${env.tag} class="${env.classes.join(' ')}"${attributes}>${env.content}`; +} + +export { stringify }; +export default stringify; From 52fca5dc8c385235aeb1bcae9e578d61c81cd2bf Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sat, 26 Apr 2025 18:44:31 -0400 Subject: [PATCH 009/101] Move tokenize --- src/core/prism-class.ts | 333 +--------------------------------------- src/core/stringify.ts | 2 - src/core/tokenize.ts | 256 ++++++++++++++++++++++++++++++ 3 files changed, 262 insertions(+), 329 deletions(-) create mode 100644 src/core/tokenize.ts diff --git a/src/core/prism-class.ts b/src/core/prism-class.ts index 0ffe453889..63af20bb59 100644 --- a/src/core/prism-class.ts +++ b/src/core/prism-class.ts @@ -1,16 +1,12 @@ -import { rest, tokenize } from '../shared/symbols'; -import { htmlEncode } from '../shared/util'; import { Hooks } from './hooks'; -import { LinkedList } from './linked-list'; import { Registry } from './registry'; -import { Token } from './token'; -import type { KnownPlugins } from '../known-plugins'; -import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types'; -import type { LinkedListHeadNode, LinkedListMiddleNode, LinkedListTailNode } from './linked-list'; -import type { TokenStream } from './token'; import { highlightAll, type HighlightAllOptions } from './highlight-all'; import { highlightElement, type HighlightElementOptions } from './highlight-element'; import { highlight, type HighlightOptions } from './highlight'; +import { tokenize } from './tokenize'; +import type { KnownPlugins } from '../known-plugins'; +import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types'; +import type { TokenStream } from './token'; /** * Prism: Lightweight, robust, elegant syntax highlighting @@ -45,326 +41,9 @@ export default class Prism { } /** - * This is the heart of Prism, and the most low-level function you can use. It accepts a string of text as input - * and the language definitions to use, and returns an array with the tokenized code. - * - * When the language definition includes nested tokens, the function is called recursively on each of these tokens. - * - * This method could be useful in other contexts as well, as a very crude parser. - * - * @param text A string with the code to be highlighted. - * @param grammar An object containing the tokens to use. - * - * Usually a language definition like `Prism.languages.markup`. - * @returns An array of strings and tokens, a token stream. - * @example - * let code = `var foo = 0;`; - * let tokens = Prism.tokenize(code, Prism.getLanguage('javascript')); - * tokens.forEach(token => { - * if (token instanceof Token && token.type === 'number') { - * console.log(`Found numeric literal: ${token.content}`); - * } - * }); + * See {@link tokenize} */ tokenize (text: string, grammar: Grammar): TokenStream { - const customTokenize = grammar[tokenize]; - if (customTokenize) { - return customTokenize(text, grammar, this); - } - - let restGrammar = resolve(this.components, grammar[rest]); - while (restGrammar) { - grammar = { ...grammar, ...restGrammar }; - restGrammar = resolve(this.components, restGrammar[rest]); - } - - const tokenList = new LinkedList(); - tokenList.addAfter(tokenList.head, text); - - this._matchGrammar(text, tokenList, grammar, tokenList.head, 0); - - return tokenList.toArray(); - } - - private _matchGrammar ( - text: string, - tokenList: LinkedList, - grammar: GrammarTokens, - startNode: LinkedListHeadNode | LinkedListMiddleNode, - startPos: number, - rematch?: RematchOptions - ): void { - for (const token in grammar) { - const tokenValue = grammar[token]; - if (!grammar.hasOwnProperty(token) || !tokenValue) { - continue; - } - - const patterns = Array.isArray(tokenValue) ? tokenValue : [tokenValue]; - - for (let j = 0; j < patterns.length; ++j) { - if (rematch && rematch.cause === `${token},${j}`) { - return; - } - - const patternObj = toGrammarToken(patterns[j]); - let { pattern, lookbehind = false, greedy = false, alias, inside } = patternObj; - const insideGrammar = resolve(this.components, inside); - - if (greedy && !pattern.global) { - // Without the global flag, lastIndex won't work - patternObj.pattern = pattern = RegExp(pattern.source, pattern.flags + 'g'); - } - - for ( - // iterate the token list and keep track of the current token/string position - let currentNode = startNode.next, pos = startPos; - currentNode.next !== null; - pos += currentNode.value.length, currentNode = currentNode.next - ) { - if (rematch && pos >= rematch.reach) { - break; - } - - let str = currentNode.value; - - if (tokenList.length > text.length) { - // Something went terribly wrong, ABORT, ABORT! - return; - } - - if (str instanceof Token) { - continue; - } - - let removeCount = 1; // this is the to parameter of removeBetween - let match; - - if (greedy) { - match = matchPattern(pattern, pos, text, lookbehind); - if (!match || match.index >= text.length) { - break; - } - - const from = match.index; - const to = match.index + match[0].length; - let p = pos; - - // find the node that contains the match - p += currentNode.value.length; - while (from >= p) { - currentNode = currentNode.next; - if (currentNode.next === null) { - throw new Error( - 'The linked list and the actual text have become de-synced' - ); - } - p += currentNode.value.length; - } - // adjust pos (and p) - p -= currentNode.value.length; - pos = p; - - // the current node is a Token, then the match starts inside another Token, which is invalid - if (currentNode.value instanceof Token) { - continue; - } - - // find the last node which is affected by this match - let k: - | LinkedListMiddleNode - | LinkedListTailNode = currentNode; - for ( - ; - k.next !== null && (p < to || typeof k.value === 'string'); - k = k.next - ) { - removeCount++; - p += k.value.length; - } - removeCount--; - - // replace with the new match - str = text.slice(pos, p); - match.index -= pos; - } - else { - match = matchPattern(pattern, 0, str, lookbehind); - if (!match) { - continue; - } - } - - // eslint-disable-next-line no-redeclare - const from = match.index; - const matchStr = match[0]; - const before = str.slice(0, from); - const after = str.slice(from + matchStr.length); - - const reach = pos + str.length; - if (rematch && reach > rematch.reach) { - rematch.reach = reach; - } - - let removeFrom = currentNode.prev; - - if (before) { - removeFrom = tokenList.addAfter(removeFrom, before); - pos += before.length; - } - - tokenList.removeRange(removeFrom, removeCount); - - const wrapped = new Token( - token, - insideGrammar ? this.tokenize(matchStr, insideGrammar) : matchStr, - alias, - matchStr - ); - currentNode = tokenList.addAfter(removeFrom, wrapped); - - if (after) { - tokenList.addAfter(currentNode, after); - } - - if (removeCount > 1) { - // at least one Token object was removed, so we have to do some rematching - // this can only happen if the current pattern is greedy - - const nestedRematch: RematchOptions = { - cause: `${token},${j}`, - reach, - }; - this._matchGrammar( - text, - tokenList, - grammar, - currentNode.prev, - pos, - nestedRematch - ); - - // the reach might have been extended because of the rematching - if (rematch && nestedRematch.reach > rematch.reach) { - rematch.reach = nestedRematch.reach; - } - } - } - } - } - } -} - -interface RematchOptions { - cause: string; - reach: number; -} - -export interface AsyncHighlightingData { - language: string; - code: string; - grammar: Grammar; -} -export type AsyncHighlighter = (data: AsyncHighlightingData) => Promise; - -export interface HighlightOptions { - grammar?: Grammar; -} - -function matchPattern (pattern: RegExp, pos: number, text: string, lookbehind: boolean) { - pattern.lastIndex = pos; - const match = pattern.exec(text); - if (match && lookbehind && match[1]) { - // change the match to remove the text matched by the Prism lookbehind group - const lookbehindLength = match[1].length; - match.index += lookbehindLength; - match[0] = match[0].slice(lookbehindLength); - } - return match; -} - -/** - * Converts the given token or token stream to an HTML representation. - * - * The following hooks will be run: - * 1. `wrap`: On each {@link Token}. - * - * @param o The token or token stream to be converted. - * @param language The name of current language. - * @returns The HTML representation of the token or token stream. - */ -function stringify (o: string | Token | TokenStream, language: string, hooks: Hooks): string { - if (typeof o === 'string') { - return htmlEncode(o); - } - if (Array.isArray(o)) { - let s = ''; - o.forEach(e => { - s += stringify(e, language, hooks); - }); - return s; - } - - const env: Record = { - type: o.type, - content: stringify(o.content, language, hooks), - tag: 'span', - classes: ['token', o.type], - attributes: {}, - language, - }; - - const aliases = o.alias; - if (aliases) { - if (Array.isArray(aliases)) { - env.classes.push(...aliases); - } - else { - env.classes.push(aliases); - } - } - - hooks.run('wrap', env); - - let attributes = ''; - for (const name in env.attributes) { - attributes += - ' ' + name + '="' + (env.attributes[name] || '').replace(/"/g, '"') + '"'; - } - - return ( - '<' + - env.tag + - ' class="' + - env.classes.join(' ') + - '"' + - attributes + - '>' + - env.content + - '' - ); -} - -function toGrammarToken (pattern: GrammarToken | RegExpLike): GrammarToken { - if (!pattern.pattern) { - return { pattern }; - } - else { - return pattern; - } -} - -function resolve ( - components: Registry, - reference: Grammar | string | null | undefined -): Grammar | undefined { - if (reference) { - if (typeof reference === 'string') { - return components.getLanguage(reference); - } - return reference; + return tokenize.call(this, text, grammar); } - return undefined; } diff --git a/src/core/stringify.ts b/src/core/stringify.ts index 2dda710378..6309c6fd20 100644 --- a/src/core/stringify.ts +++ b/src/core/stringify.ts @@ -1,5 +1,3 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -/* eslint-disable @typescript-eslint/no-unsafe-call */ import { htmlEncode } from '../shared/util'; import type { Hooks } from './hooks'; import type { Token } from './token'; diff --git a/src/core/tokenize.ts b/src/core/tokenize.ts new file mode 100644 index 0000000000..25267903fe --- /dev/null +++ b/src/core/tokenize.ts @@ -0,0 +1,256 @@ +import { rest, tokenize as tokenizer } from '../shared/symbols'; +import { LinkedList } from './linked-list'; +import singleton, { type Prism } from './prism'; +import { Token } from './token'; +import type { Registry } from './registry'; +import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types'; +import type { LinkedListHeadNode, LinkedListMiddleNode, LinkedListTailNode } from './linked-list'; +import type { TokenStream } from './token'; + +/** + * This is the heart of Prism, and the most low-level function you can use. It accepts a string of text as input + * and the language definitions to use, and returns an array with the tokenized code. + * + * When the language definition includes nested tokens, the function is called recursively on each of these tokens. + * + * This method could be useful in other contexts as well, as a very crude parser. + * + * @param text A string with the code to be highlighted. + * @param grammar An object containing the tokens to use. + * + * Usually a language definition like `Prism.languages.markup`. + * @returns An array of strings and tokens, a token stream. + * @example + * let code = `var foo = 0;`; + * let tokens = Prism.tokenize(code, Prism.getLanguage('javascript')); + * tokens.forEach(token => { + * if (token instanceof Token && token.type === 'number') { + * console.log(`Found numeric literal: ${token.content}`); + * } + * }); + */ +export function tokenize (this: Prism, text: string, grammar: Grammar): TokenStream { + const prism = this ?? singleton; + const customTokenize = grammar[tokenizer]; + if (customTokenize) { + return customTokenize(text, grammar, prism); + } + + let restGrammar = resolve(prism.components, grammar[rest]); + while (restGrammar) { + grammar = { ...grammar, ...restGrammar }; + restGrammar = resolve(prism.components, restGrammar[rest]); + } + + const tokenList = new LinkedList(); + tokenList.addAfter(tokenList.head, text); + + _matchGrammar.call(prism, text, tokenList, grammar, tokenList.head, 0); + + return tokenList.toArray(); +} + +function _matchGrammar ( + this: Prism, + text: string, + tokenList: LinkedList, + grammar: GrammarTokens, + startNode: LinkedListHeadNode | LinkedListMiddleNode, + startPos: number, + rematch?: RematchOptions +): void { + for (const token in grammar) { + const tokenValue = grammar[token]; + if (!grammar.hasOwnProperty(token) || !tokenValue) { + continue; + } + + const patterns = Array.isArray(tokenValue) ? tokenValue : [tokenValue]; + + for (let j = 0; j < patterns.length; ++j) { + if (rematch && rematch.cause === `${token},${j}`) { + return; + } + + const patternObj = toGrammarToken(patterns[j]); + let { pattern, lookbehind = false, greedy = false, alias, inside } = patternObj; + const insideGrammar = resolve(this.components, inside); + + if (greedy && !pattern.global) { + // Without the global flag, lastIndex won't work + patternObj.pattern = pattern = RegExp(pattern.source, pattern.flags + 'g'); + } + + for ( + // iterate the token list and keep track of the current token/string position + let currentNode = startNode.next, pos = startPos; + currentNode.next !== null; + pos += currentNode.value.length, currentNode = currentNode.next + ) { + if (rematch && pos >= rematch.reach) { + break; + } + + let str = currentNode.value; + + if (tokenList.length > text.length) { + // Something went terribly wrong, ABORT, ABORT! + return; + } + + if (str instanceof Token) { + continue; + } + + let removeCount = 1; // this is the to parameter of removeBetween + let match; + + if (greedy) { + match = matchPattern(pattern, pos, text, lookbehind); + if (!match || match.index >= text.length) { + break; + } + + const from = match.index; + const to = match.index + match[0].length; + let p = pos; + + // find the node that contains the match + p += currentNode.value.length; + while (from >= p) { + currentNode = currentNode.next; + if (currentNode.next === null) { + throw new Error( + 'The linked list and the actual text have become de-synced' + ); + } + p += currentNode.value.length; + } + // adjust pos (and p) + p -= currentNode.value.length; + pos = p; + + // the current node is a Token, then the match starts inside another Token, which is invalid + if (currentNode.value instanceof Token) { + continue; + } + + // find the last node which is affected by this match + let k: + | LinkedListMiddleNode + | LinkedListTailNode = currentNode; + for (; k.next !== null && (p < to || typeof k.value === 'string'); k = k.next) { + removeCount++; + p += k.value.length; + } + removeCount--; + + // replace with the new match + str = text.slice(pos, p); + match.index -= pos; + } + else { + match = matchPattern(pattern, 0, str, lookbehind); + if (!match) { + continue; + } + } + + // eslint-disable-next-line no-redeclare + const from = match.index; + const matchStr = match[0]; + const before = str.slice(0, from); + const after = str.slice(from + matchStr.length); + + const reach = pos + str.length; + if (rematch && reach > rematch.reach) { + rematch.reach = reach; + } + + let removeFrom = currentNode.prev; + + if (before) { + removeFrom = tokenList.addAfter(removeFrom, before); + pos += before.length; + } + + tokenList.removeRange(removeFrom, removeCount); + + const wrapped = new Token( + token, + insideGrammar ? this.tokenize(matchStr, insideGrammar) : matchStr, + alias, + matchStr + ); + currentNode = tokenList.addAfter(removeFrom, wrapped); + + if (after) { + tokenList.addAfter(currentNode, after); + } + + if (removeCount > 1) { + // at least one Token object was removed, so we have to do some rematching + // this can only happen if the current pattern is greedy + + const nestedRematch: RematchOptions = { + cause: `${token},${j}`, + reach, + }; + _matchGrammar.call( + this, + text, + tokenList, + grammar, + currentNode.prev, + pos, + nestedRematch + ); + + // the reach might have been extended because of the rematching + if (rematch && nestedRematch.reach > rematch.reach) { + rematch.reach = nestedRematch.reach; + } + } + } + } + } +} + +function matchPattern (pattern: RegExp, pos: number, text: string, lookbehind: boolean) { + pattern.lastIndex = pos; + const match = pattern.exec(text); + if (match && lookbehind && match[1]) { + // change the match to remove the text matched by the Prism lookbehind group + const lookbehindLength = match[1].length; + match.index += lookbehindLength; + match[0] = match[0].slice(lookbehindLength); + } + return match; +} + +function toGrammarToken (pattern: GrammarToken | RegExpLike): GrammarToken { + if (!pattern.pattern) { + return { pattern }; + } + else { + return pattern; + } +} + +function resolve ( + components: Registry, + reference: Grammar | string | null | undefined +): Grammar | undefined { + if (reference) { + if (typeof reference === 'string') { + return components.getLanguage(reference); + } + return reference; + } + return undefined; +} + +interface RematchOptions { + cause: string; + reach: number; +} From 1158aa93bc38b4d17826d068d57172dcf341a73b Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sun, 27 Apr 2025 14:51:18 -0400 Subject: [PATCH 010/101] Update src/core.ts Co-authored-by: Dmitry Sharabin --- src/core.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core.ts b/src/core.ts index 7997036216..32ed9ac105 100644 --- a/src/core.ts +++ b/src/core.ts @@ -1,2 +1,2 @@ -export { Prism } from './core/prism-class'; +export { default as Prism } from './core/prism-class'; export { Token } from './core/token'; From 9acac36e14d4d8ebdce2900e8b67bfafbef64ee1 Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sun, 27 Apr 2025 14:53:53 -0400 Subject: [PATCH 011/101] Update src/types.d.ts Co-authored-by: Dmitry Sharabin --- src/types.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types.d.ts b/src/types.d.ts index c597bb22e2..aa3666b68d 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -1,4 +1,4 @@ -import type { Prism } from './core/prism-class'; +import type Prism from './core/prism-class'; import type { TokenStream } from './core/token'; import type { KnownPlugins } from './known-plugins'; import type { rest, tokenize } from './shared/symbols'; From 8a8bc188de6b0d9ba332edae027a4b3634ef66c0 Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sun, 27 Apr 2025 15:01:16 -0400 Subject: [PATCH 012/101] Update src/global.ts Co-authored-by: Dmitry Sharabin --- src/global.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/global.ts b/src/global.ts index 6ef69a00e5..dbe1d8efc7 100644 --- a/src/global.ts +++ b/src/global.ts @@ -1,4 +1,4 @@ -import globalPrism, { Prism } from './core/prism-class'; +import globalPrism, { type Prism } from './core/prism'; declare global { var Prism: Prism | undefined From e0ca584d947605ae7f6f18c1e45c5c9f405c1700 Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sun, 27 Apr 2025 15:02:46 -0400 Subject: [PATCH 013/101] Update hooks.ts --- src/core/hooks.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/hooks.ts b/src/core/hooks.ts index 2305ae2c00..ff96cd9d87 100644 --- a/src/core/hooks.ts +++ b/src/core/hooks.ts @@ -56,4 +56,4 @@ export class Hooks { } } -export type HookCallback = (env: Record) => void; +export type HookCallback = (env: Record) => void; From 2727307a437c2c22250ac05e89d029b9845a27c2 Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sun, 27 Apr 2025 15:06:19 -0400 Subject: [PATCH 014/101] Update hooks.ts --- src/core/hooks.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/hooks.ts b/src/core/hooks.ts index ff96cd9d87..9ed0c90f6d 100644 --- a/src/core/hooks.ts +++ b/src/core/hooks.ts @@ -1,6 +1,6 @@ export class Hooks { // eslint-disable-next-line func-call-spacing - private _all = new Map void)[]>(); + private _all = new Map(); /** * Adds the given callback to the list of callbacks for the given hook and returns a function that @@ -17,7 +17,7 @@ export class Hooks { * @param name The name of the hook. * @param callback The callback function which is given environment variables. */ - add (name: Name, callback: HookCallback): () => void { + add (name: Name, callback: HookCallback): () => void { let hooks = this._all.get(name); if (hooks === undefined) { hooks = []; From 96e93ea1b04982cf6bca670381a4573006f196fd Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sun, 27 Apr 2025 15:06:50 -0400 Subject: [PATCH 015/101] Update src/core/prism-class.ts Co-authored-by: Dmitry Sharabin --- src/core/prism-class.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/prism-class.ts b/src/core/prism-class.ts index 63af20bb59..70a579bd71 100644 --- a/src/core/prism-class.ts +++ b/src/core/prism-class.ts @@ -36,7 +36,7 @@ export default class Prism { /** * See {@link highlight} */ - highlight (text: string, language: string, options?: HighlightOptions): string { + highlight (text: string, language: string, options: HighlightOptions = {}): string { return highlight.call(this, text, language, options); } From dbbb0dd0d0ab67840b400cabf340d1815df57603 Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sun, 27 Apr 2025 15:07:35 -0400 Subject: [PATCH 016/101] Update src/global.ts Co-authored-by: Dmitry Sharabin --- src/global.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/global.ts b/src/global.ts index dbe1d8efc7..88c52576d8 100644 --- a/src/global.ts +++ b/src/global.ts @@ -1,7 +1,7 @@ import globalPrism, { type Prism } from './core/prism'; declare global { - var Prism: Prism | undefined + const Prism: Prism | undefined; } /** From 4dfff4c38beb42a1168442bfbbdf11586a98411d Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sun, 27 Apr 2025 15:09:49 -0400 Subject: [PATCH 017/101] Update registry.ts --- src/core/registry.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/registry.ts b/src/core/registry.ts index 22fc6380a9..806ca0a573 100644 --- a/src/core/registry.ts +++ b/src/core/registry.ts @@ -1,7 +1,7 @@ import { extend } from '../shared/language-util'; import { forEach, kebabToCamelCase } from '../shared/util'; import type { ComponentProto, Grammar } from '../types'; -import type { Prism } from './prism-class'; +import type Prism from './prism-class'; interface Entry { proto: ComponentProto; From a2a5d2361ba881f68e9e70118640b41443d88b1a Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sun, 27 Apr 2025 15:36:27 -0400 Subject: [PATCH 018/101] Update eslint.config.mjs --- eslint.config.mjs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/eslint.config.mjs b/eslint.config.mjs index f0638a3489..941a711a86 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -17,13 +17,7 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); export default defineConfig([ { - ignores: [ - 'benchmark/downloads', - 'benchmark/remotes', - 'dist', - 'node_modules', - 'types', - ], + ignores: ['benchmark/downloads', 'benchmark/remotes', 'dist', 'node_modules', 'types'], }, js.configs.recommended, { @@ -180,6 +174,9 @@ export default defineConfig([ '@typescript-eslint/no-unsafe-call': 'off', '@typescript-eslint/no-explicit-any': 'off', + '@typescript-eslint/no-unsafe-assignment': 'off', + '@typescript-eslint/no-unsafe-member-access': 'off', + '@typescript-eslint/no-unsafe-return': 'off', }, }, { From 1e31274bc39bc0964eb9ad789527824f53e0eead Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sun, 27 Apr 2025 15:36:38 -0400 Subject: [PATCH 019/101] Remove known plugins --- src/core/prism-class.ts | 9 +++------ src/known-plugins.d.ts | 23 ----------------------- src/types.d.ts | 5 +---- 3 files changed, 4 insertions(+), 33 deletions(-) delete mode 100644 src/known-plugins.d.ts diff --git a/src/core/prism-class.ts b/src/core/prism-class.ts index 70a579bd71..a5e1ed124e 100644 --- a/src/core/prism-class.ts +++ b/src/core/prism-class.ts @@ -4,20 +4,17 @@ import { highlightAll, type HighlightAllOptions } from './highlight-all'; import { highlightElement, type HighlightElementOptions } from './highlight-element'; import { highlight, type HighlightOptions } from './highlight'; import { tokenize } from './tokenize'; -import type { KnownPlugins } from '../known-plugins'; import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types'; import type { TokenStream } from './token'; /** - * Prism: Lightweight, robust, elegant syntax highlighting - * - * @license MIT - * @author Lea Verou and contributors + * Prism class, to create Prism instances with different settings. + * In most use cases, you just need the pre-existing Prism instance, see {@link prism}. */ export default class Prism { hooks = new Hooks(); components = new Registry(this); - plugins: Partial & KnownPlugins> = {}; + plugins: Record = {}; /** * See {@link highlightAll}. diff --git a/src/known-plugins.d.ts b/src/known-plugins.d.ts deleted file mode 100644 index 19cd5baba3..0000000000 --- a/src/known-plugins.d.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { Autoloader } from './plugins/autoloader/prism-autoloader'; -import type { CustomClass } from './plugins/custom-class/prism-custom-class'; -import type { FileHighlight } from './plugins/file-highlight/prism-file-highlight'; -import type { FilterHighlightAll } from './plugins/filter-highlight-all/prism-filter-highlight-all'; -import type { JsonpHighlight } from './plugins/jsonp-highlight/prism-jsonp-highlight'; -import type { LineHighlight } from './plugins/line-highlight/prism-line-highlight'; -import type { LineNumbers } from './plugins/line-numbers/prism-line-numbers'; -import type { NormalizeWhitespace } from './plugins/normalize-whitespace/prism-normalize-whitespace'; -import type { PreviewerCollection } from './plugins/previewers/prism-previewers'; -import type { Toolbar } from './plugins/toolbar/prism-toolbar'; - -declare interface KnownPlugins { - autoloader: Autoloader; - customClass: CustomClass; - fileHighlight: FileHighlight; - filterHighlightAll: FilterHighlightAll; - jsonpHighlight: JsonpHighlight; - lineHighlight: LineHighlight; - lineNumbers: LineNumbers; - normalizeWhitespace: NormalizeWhitespace; - previewers: PreviewerCollection; - toolbar: Toolbar; -} diff --git a/src/types.d.ts b/src/types.d.ts index aa3666b68d..8141df9656 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -1,6 +1,5 @@ import type Prism from './core/prism-class'; import type { TokenStream } from './core/token'; -import type { KnownPlugins } from './known-plugins'; import type { rest, tokenize } from './shared/symbols'; export interface GrammarOptions { @@ -19,9 +18,7 @@ export interface LanguageProto extends ComponentProt grammar: Grammar | ((options: GrammarOptions) => Grammar); plugin?: undefined; } -type PluginType = Name extends keyof KnownPlugins - ? KnownPlugins[Name] - : unknown; +type PluginType = unknown; export interface PluginProto extends ComponentProtoBase { grammar?: undefined; plugin?: ( From ca6df670be45ab0e8d6e559e3ec050c1be690095 Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sun, 27 Apr 2025 15:36:42 -0400 Subject: [PATCH 020/101] Update prism.ts --- src/core/prism.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/core/prism.ts b/src/core/prism.ts index c02a689607..ea1181e4ac 100644 --- a/src/core/prism.ts +++ b/src/core/prism.ts @@ -1,4 +1,10 @@ -import Prism from "./prism-class"; +/** + * Prism: Lightweight, robust, elegant syntax highlighting + * + * @license MIT + * @author Lea Verou and contributors + */ +import Prism from './prism-class'; /** * Prism singleton. @@ -8,7 +14,8 @@ import Prism from "./prism-class"; * In global builds, it will also be the Prism global variable. * Any imported plugins and languages will automatically be added to this instance. */ -export default new Prism(); +const prism = new Prism(); +export default prism; -/** Re-export Prism class so they can be imported together */ +/** See {@link Prism} */ export { Prism }; From 051e0e650aae1224e7989b9cde3e93a55fcda002 Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sun, 27 Apr 2025 15:37:12 -0400 Subject: [PATCH 021/101] Update src/core/highlight.ts Co-authored-by: Dmitry Sharabin --- src/core/highlight.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/highlight.ts b/src/core/highlight.ts index f9c0e44e03..2c79460e4b 100644 --- a/src/core/highlight.ts +++ b/src/core/highlight.ts @@ -25,7 +25,7 @@ export function highlight ( this: Prism, text: string, language: string, - options?: HighlightOptions + options: HighlightOptions = {} ): string { const prism = this ?? singleton; From 2147caef90fddb15c3f4da2f916bde02bb4fd94b Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sun, 27 Apr 2025 15:37:20 -0400 Subject: [PATCH 022/101] Update src/core/highlight.ts Co-authored-by: Dmitry Sharabin --- src/core/highlight.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/highlight.ts b/src/core/highlight.ts index 2c79460e4b..0c43adc9aa 100644 --- a/src/core/highlight.ts +++ b/src/core/highlight.ts @@ -30,7 +30,7 @@ export function highlight ( const prism = this ?? singleton; const languageId = prism.components.resolveAlias(language); - const grammar = options?.grammar ?? prism.components.getLanguage(languageId); + const grammar = options.grammar ?? prism.components.getLanguage(languageId); const env: Record | Record = { code: text, From de0b0bd2bd0e0592537cc110d68ebb71b1687f3d Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sun, 27 Apr 2025 15:53:00 -0400 Subject: [PATCH 023/101] Update src/plugins/keep-markup/prism-keep-markup.ts Co-authored-by: Dmitry Sharabin --- src/plugins/keep-markup/prism-keep-markup.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/keep-markup/prism-keep-markup.ts b/src/plugins/keep-markup/prism-keep-markup.ts index 2d32774cea..08735d2d1c 100644 --- a/src/plugins/keep-markup/prism-keep-markup.ts +++ b/src/plugins/keep-markup/prism-keep-markup.ts @@ -80,7 +80,7 @@ export default { } }, 'after-highlight': (env) => { - const data = env.markupdata ?? []; + const data = env.markupData ?? []; if (data.length) { type End = [node: Text, pos: number] From 83fad951a8c844186d6a114489e4fe6a3d816299 Mon Sep 17 00:00:00 2001 From: Dmitry Sharabin Date: Sun, 27 Apr 2025 21:58:32 +0200 Subject: [PATCH 024/101] [build] Delete `known-plugins.d.ts` --- scripts/build.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build.ts b/scripts/build.ts index 23d1169284..f6761c8dcf 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -329,7 +329,7 @@ async function buildTypes() { await mkdir('./types'); // Copy existing type definitions - const typeFiles = ['types.d.ts', 'known-plugins.d.ts']; + const typeFiles = ['types.d.ts']; await Promise.all( typeFiles.map(file => copyFile(path.join(SRC_DIR, file), path.join('./types', file))) From b4366b2b78d05878bcb41bb15be9d6860c6c884f Mon Sep 17 00:00:00 2001 From: Lea Verou Date: Sun, 27 Apr 2025 22:01:58 -0400 Subject: [PATCH 025/101] [WIP] Make language definitions more declarative, simplify registry, better modularity --- eslint.config.mjs | 5 +- scripts/build.ts | 16 ++- src/auto-start.ts | 65 --------- src/config.ts | 65 +++++++++ src/core.ts | 2 +- src/core/{ => classes}/hooks.ts | 0 src/core/classes/language-registry.ts | 30 ++++ src/core/classes/language.ts | 78 +++++++++++ src/core/classes/plugin-registry.ts | 8 ++ src/core/classes/prism.ts | 132 ++++++++++++++++++ src/core/classes/registry.ts | 85 +++++++++++ src/core/prism-class.ts | 46 ------ src/core/prism.ts | 2 +- src/core/registry.ts | 3 +- src/core/stringify.ts | 2 +- src/languages/c.ts | 130 ++++++++--------- src/languages/chaiscript.ts | 1 + src/languages/cpp.ts | 1 + src/languages/java.ts | 1 + .../diff-highlight/prism-diff-highlight.ts | 2 +- .../prism-normalize-whitespace.ts | 4 +- src/plugins/toolbar/prism-toolbar.ts | 2 +- src/shared/hooks-util.ts | 2 +- src/types.d.ts | 9 +- src/util.ts | 35 +++++ 25 files changed, 528 insertions(+), 198 deletions(-) delete mode 100644 src/auto-start.ts create mode 100644 src/config.ts rename src/core/{ => classes}/hooks.ts (100%) create mode 100644 src/core/classes/language-registry.ts create mode 100644 src/core/classes/language.ts create mode 100644 src/core/classes/plugin-registry.ts create mode 100644 src/core/classes/prism.ts create mode 100644 src/core/classes/registry.ts delete mode 100644 src/core/prism-class.ts create mode 100644 src/util.ts diff --git a/eslint.config.mjs b/eslint.config.mjs index 50b25975d9..40945ec3ad 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -58,7 +58,7 @@ const config = [ 'object-shorthand': ['warn', 'always', { avoidQuotes: true }], 'one-var': ['warn', 'never'], 'prefer-arrow-callback': 'warn', - 'prefer-const': ['warn', { 'destructuring': 'all' }], + 'prefer-const': 'off', 'prefer-spread': 'warn', // JSDoc @@ -174,6 +174,7 @@ const config = [ '@typescript-eslint/no-unsafe-assignment': 'off', '@typescript-eslint/no-unsafe-member-access': 'off', '@typescript-eslint/no-unsafe-return': 'off', + '@typescript-eslint/no-unsafe-argument': 'off', }, }, { @@ -188,7 +189,7 @@ const config = [ }, { // Browser-specific parts - files: ['src/auto-start.ts'], + files: ['src/global.ts'], languageOptions: { globals: { ...globals.browser, diff --git a/scripts/build.ts b/scripts/build.ts index 23d1169284..cec8117c1a 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -319,7 +319,7 @@ async function clean() { ]); } -async function copyComponentsJson () { +async function copyComponentsJson() { const from = path.join(SRC_DIR, 'components.json'); const to = path.join(__dirname, '../dist/components.json'); await copyFile(from, to); @@ -410,11 +410,11 @@ async function buildJS() { rollupOptions: { ...defaultRollupOptions, input: { - 'prism': path.join(SRC_DIR, 'auto-start.ts'), - } + 'prism': path.join(SRC_DIR, 'global.ts'), + }, }, outputOptions: defaultOutputOptions, - } + }, }; try { @@ -430,4 +430,10 @@ async function buildJS() { } } -runTask(series(clean, parallel(buildTypes, buildJS, series(treeviewIconFont, minifyCSS)), copyComponentsJson)); +runTask( + series( + clean, + parallel(buildTypes, buildJS, series(treeviewIconFont, minifyCSS)), + copyComponentsJson + ) +); diff --git a/src/auto-start.ts b/src/auto-start.ts deleted file mode 100644 index bbbaecf7ad..0000000000 --- a/src/auto-start.ts +++ /dev/null @@ -1,65 +0,0 @@ -import Prism from './global'; -import autoloader from './plugins/autoloader/prism-autoloader'; - -Prism.components.add(autoloader); - -export const PrismConfig = { - // TODO: Update docs - /** - * By default, Prism will attempt to highlight all code elements (by calling {@link Prism#highlightAll}) on the - * current page after the page finished loading. This might be a problem if e.g. you wanted to asynchronously load - * additional languages or plugins yourself. - * - * By setting this value to `true`, Prism will not automatically highlight all code elements on the page. - * - * You obviously have to change this value before the automatic highlighting started. To do this, you can add an - * empty Prism object into the global scope before loading the Prism script like this: - * - * ```js - * window.Prism = window.Prism || {}; - * Prism.manual = true; - * // add a new ``` Besides, they have access to the Prism instance via `globalThis`. * Move Prism config to a separate file and fix the build options of the global bundle Co-Authored-By: Dmitry Sharabin * Fix some code after merging --------- Co-authored-by: Lea Verou --- scripts/build.ts | 112 +++++++-- scripts/components.ts | 2 +- src/auto-start.ts | 13 + src/components.json | 25 +- src/core/classes/hooks.ts | 17 +- src/core/classes/token.ts | 3 - src/core/highlight-element.ts | 3 +- src/global.ts | 3 +- src/languages/asciidoc.ts | 2 +- src/languages/css.ts | 2 +- src/languages/elixir.ts | 2 +- src/languages/ftl.ts | 2 +- src/languages/icu-message-format.ts | 2 +- src/languages/inform7.ts | 4 +- src/languages/javascript.ts | 2 +- src/languages/js-templates.ts | 4 +- src/languages/jsx.ts | 2 +- src/languages/lilypond.ts | 4 +- src/languages/lisp.ts | 4 +- src/languages/livescript.ts | 4 +- src/languages/naniscript.ts | 6 +- src/languages/php.ts | 6 +- src/languages/pug.ts | 4 +- src/languages/pure.ts | 8 +- src/languages/python.ts | 4 +- src/languages/rust.ts | 4 +- src/languages/sas.ts | 4 +- src/languages/scss.ts | 2 +- src/languages/stylus.ts | 16 +- src/languages/treeview.ts | 4 +- src/languages/wiki.ts | 4 +- src/plugins/autolinker/README.md | 53 ++++ src/plugins/autolinker/prism-autolinker.ts | 1 - src/plugins/autoloader/README.md | 132 ++++++++++ src/plugins/autoloader/demo.js | 58 +++++ src/plugins/autoloader/prism-autoloader.ts | 3 +- src/plugins/command-line/README.md | 233 ++++++++++++++++++ src/plugins/copy-to-clipboard/README.md | 214 ++++++++++++++++ .../prism-copy-to-clipboard.ts | 4 +- src/plugins/custom-class/README.md | 186 ++++++++++++++ .../custom-class/prism-custom-class.ts | 4 +- src/plugins/data-uri-highlight/README.md | 36 +++ src/plugins/diff-highlight/README.md | 84 +++++++ .../diff-highlight/prism-diff-highlight.ts | 2 +- src/plugins/download-button/README.md | 39 +++ .../download-button/prism-download-button.ts | 4 +- src/plugins/file-highlight/README.md | 59 +++++ .../file-highlight/prism-file-highlight.ts | 3 +- src/plugins/filter-highlight-all/README.md | 107 ++++++++ src/plugins/filter-highlight-all/demo.js | 8 + .../prism-filter-highlight-all.ts | 4 +- src/plugins/highlight-keywords/README.md | 48 ++++ src/plugins/inline-color/README.md | 51 ++++ src/plugins/jsonp-highlight/README.md | 114 +++++++++ src/plugins/jsonp-highlight/demo.js | 7 + .../jsonp-highlight/prism-jsonp-highlight.ts | 5 +- src/plugins/keep-markup/README.md | 67 +++++ src/plugins/keep-markup/prism-keep-markup.ts | 3 +- src/plugins/line-highlight/README.md | 84 +++++++ .../line-highlight/prism-line-highlight.ts | 20 +- src/plugins/line-numbers/README.md | 76 ++++++ .../line-numbers/prism-line-numbers.ts | 3 +- src/plugins/match-braces/README.md | 62 +++++ .../match-braces/prism-match-braces.ts | 3 +- src/plugins/normalize-whitespace/README.md | 172 +++++++++++++ src/plugins/normalize-whitespace/demo.md | 53 ++++ src/plugins/previewers/README.md | 221 +++++++++++++++++ src/plugins/previewers/prism-previewers.ts | 3 +- src/plugins/show-invisibles/README.md | 20 ++ src/plugins/show-language/README.md | 40 +++ .../show-language/prism-show-language.ts | 4 +- src/plugins/toolbar/README.md | 99 ++++++++ src/plugins/toolbar/demo.js | 32 +++ src/plugins/toolbar/prism-toolbar.ts | 11 +- src/plugins/treeview-icons/README.md | 63 +++++ src/plugins/unescaped-markup/README.md | 159 ++++++++++++ src/plugins/wpd/README.md | 43 ++++ themes/prism-coy.css => src/themes/coy.css | 7 +- themes/prism-dark.css => src/themes/dark.css | 7 +- .../prism-funky.css => src/themes/funky.css | 7 +- .../themes/okaidia.css | 9 +- {themes => src/themes}/prism.css | 7 +- .../themes/solarizedlight.css | 14 +- .../themes/tomorrow.css | 9 +- .../themes/twilight.css | 6 +- src/util/iterables.ts | 1 - .../custom-class/basic-functionality.ts | 16 +- tsconfig.json | 3 +- 88 files changed, 2908 insertions(+), 149 deletions(-) create mode 100644 src/auto-start.ts create mode 100644 src/plugins/autolinker/README.md create mode 100644 src/plugins/autoloader/README.md create mode 100644 src/plugins/autoloader/demo.js create mode 100644 src/plugins/command-line/README.md create mode 100644 src/plugins/copy-to-clipboard/README.md create mode 100644 src/plugins/custom-class/README.md create mode 100644 src/plugins/data-uri-highlight/README.md create mode 100644 src/plugins/diff-highlight/README.md create mode 100644 src/plugins/download-button/README.md create mode 100644 src/plugins/file-highlight/README.md create mode 100644 src/plugins/filter-highlight-all/README.md create mode 100644 src/plugins/filter-highlight-all/demo.js create mode 100644 src/plugins/highlight-keywords/README.md create mode 100644 src/plugins/inline-color/README.md create mode 100644 src/plugins/jsonp-highlight/README.md create mode 100644 src/plugins/jsonp-highlight/demo.js create mode 100644 src/plugins/keep-markup/README.md create mode 100644 src/plugins/line-highlight/README.md create mode 100644 src/plugins/line-numbers/README.md create mode 100644 src/plugins/match-braces/README.md create mode 100644 src/plugins/normalize-whitespace/README.md create mode 100644 src/plugins/normalize-whitespace/demo.md create mode 100644 src/plugins/previewers/README.md create mode 100644 src/plugins/show-invisibles/README.md create mode 100644 src/plugins/show-language/README.md create mode 100644 src/plugins/toolbar/README.md create mode 100644 src/plugins/toolbar/demo.js create mode 100644 src/plugins/treeview-icons/README.md create mode 100644 src/plugins/unescaped-markup/README.md create mode 100644 src/plugins/wpd/README.md rename themes/prism-coy.css => src/themes/coy.css (94%) rename themes/prism-dark.css => src/themes/dark.css (93%) rename themes/prism-funky.css => src/themes/funky.css (94%) rename themes/prism-okaidia.css => src/themes/okaidia.css (90%) rename {themes => src/themes}/prism.css (96%) rename themes/prism-solarizedlight.css => src/themes/solarizedlight.css (91%) rename themes/prism-tomorrow.css => src/themes/tomorrow.css (89%) rename themes/prism-twilight.css => src/themes/twilight.css (98%) diff --git a/scripts/build.ts b/scripts/build.ts index c570702492..ada382020c 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -19,11 +19,18 @@ import type { OutputOptions, Plugin, RollupBuild, RollupOptions, SourceMapInput const __dirname = path.dirname(fileURLToPath(import.meta.url)); const SRC_DIR = path.join(__dirname, '../src/'); +const DIST_DIR = path.join(__dirname, '../dist'); + const languageIds = fs .readdirSync(path.join(SRC_DIR, 'languages')) .map(f => f.slice(0, -'.js'.length)) .sort(); const pluginIds = fs.readdirSync(path.join(SRC_DIR, 'plugins')).sort(); +const themeIds = fs + .readdirSync(path.join(SRC_DIR, 'themes')) + .filter(f => /\.css$/i.test(f)) + .map(f => f.slice(0, -'.css'.length)) + .sort(); async function loadComponent (id: string) { let file; @@ -40,10 +47,8 @@ async function loadComponent (id: string) { async function minifyCSS () { const input: Record = {}; - const THEMES_DIR = path.join(__dirname, '../themes'); - const themes = await readdir(THEMES_DIR); - for (const theme of themes.filter(f => /\.css$/i.test(f))) { - input[`themes/${theme}`] = path.join(THEMES_DIR, theme); + for (const id of themeIds) { + input[`themes/${id}.css`] = path.join(SRC_DIR, `themes/${id}.css`); } for (const id of pluginIds) { @@ -53,8 +58,6 @@ async function minifyCSS () { } } - const DIST = path.join(__dirname, '../dist'); - const clean = new CleanCSS({}); await Promise.all( @@ -68,7 +71,7 @@ async function minifyCSS () { console.warn(`${file}: ${warn}`); } - const targetFile = path.join(DIST, target); + const targetFile = path.join(DIST_DIR, target); await mkdir(path.dirname(targetFile), { recursive: true }); await writeFile(targetFile, output.styles, 'utf-8'); }) @@ -276,7 +279,7 @@ const inlineRegexSourcePlugin: Plugin = { */ const lazyGrammarPlugin: Plugin = { name: 'lazy-grammar', - renderChunk(code) { + renderChunk (code) { const str = new MagicString(code); str.replace( /^(?[ \t]+)grammar: (\{[\s\S]*?^\k\})/m, @@ -286,7 +289,7 @@ const lazyGrammarPlugin: Plugin = { }, }; -function toRenderedChunk(s: MagicString): { code: string; map: SourceMapInput } { +function toRenderedChunk (s: MagicString): { code: string; map: SourceMapInput } { return { code: s.toString(), map: s.generateMap({ hires: true }) as SourceMapInput, @@ -309,7 +312,7 @@ const terserPlugin = rollupTerser({ keep_classnames: true, }); -async function clean() { +async function clean () { const outputDir = path.join(__dirname, '../dist'); const typesDir = path.join(__dirname, '../types'); await Promise.all([ @@ -324,7 +327,7 @@ async function copyComponentsJson() { await copyFile(from, to); } -async function buildTypes() { +async function buildTypes () { await mkdir('./types'); // Copy existing type definitions @@ -358,7 +361,7 @@ async function buildTypes() { program.emit(); } -async function buildJS() { +async function buildJS () { const input: Record = { 'index': path.join(SRC_DIR, 'index.ts'), 'shared': path.join(SRC_DIR, 'shared.ts'), @@ -372,7 +375,13 @@ async function buildJS() { const defaultRollupOptions: RollupOptions = { input, - plugins: [rollupTypescript({ module: 'esnext' })], + plugins: [ + rollupTypescript({ module: 'esnext' }), + lazyGrammarPlugin, + dataInsertPlugin, + inlineRegexSourcePlugin, + terserPlugin, + ], }; const defaultOutputOptions: OutputOptions = { @@ -380,7 +389,6 @@ async function buildJS() { chunkFileNames: '_chunks/[name]-[hash].js', validate: true, sourcemap: 'hidden', - plugins: [lazyGrammarPlugin, dataInsertPlugin, inlineRegexSourcePlugin, terserPlugin], }; const bundles: Record< @@ -409,10 +417,16 @@ async function buildJS() { rollupOptions: { ...defaultRollupOptions, input: { - 'prism': path.join(SRC_DIR, 'global.ts'), + 'prism': path.join(SRC_DIR, 'auto-start.ts'), }, }, - outputOptions: defaultOutputOptions, + outputOptions: { + ...defaultOutputOptions, + format: 'iife', + name: 'Prism', + exports: 'default', + extend: true, + }, }, }; @@ -429,10 +443,74 @@ async function buildJS() { } } +// Helper to get file size in bytes, or 0 if not found +const getFileSize = async (filePath: string) => { + try { + const stat = await fs.promises.stat(filePath); + return stat.size; + } + catch { + return 0; + } +}; + +async function calculateFileSizes () { + type FileSizes = { + css?: number; + js?: number; + }; + + const ret: Record> = { + core: {}, + themes: {}, + languages: {}, + plugins: {}, + }; + + ret.core.js = await getFileSize(path.join(DIST_DIR, 'index.js')); + + for (const category of ['themes', 'languages', 'plugins']) { + let ids = themeIds; + if (category === 'languages') { + ids = languageIds; + } + else if (category === 'plugins') { + ids = pluginIds; + } + + for (const id of ids) { + ret[category][id] = {}; + + for (const ext of ['js', 'css']) { + if ( + (ext === 'css' && + (category === 'languages' || components[category][id].noCSS)) || + (category === 'themes' && ext === 'js') + ) { + continue; + } + + const filePath = path.join( + DIST_DIR, + category, + category === 'plugins' ? `prism-${id}` : id + ); + ret[category][id][ext as 'css' | 'js'] = await getFileSize(`${filePath}.${ext}`); + } + } + } + + await fs.promises.writeFile( + path.join(DIST_DIR, 'file-sizes.json'), + JSON.stringify(ret, null, '\t') + ); +} + runTask( series( clean, parallel(buildTypes, buildJS, series(treeviewIconFont, minifyCSS)), - copyComponentsJson + copyComponentsJson, + calculateFileSizes ) ); diff --git a/scripts/components.ts b/scripts/components.ts index 8211f09178..726198aa13 100644 --- a/scripts/components.ts +++ b/scripts/components.ts @@ -7,4 +7,4 @@ const __dirname = path.dirname(__filename); export const components = JSON.parse( fs.readFileSync(path.join(__dirname, '../src/components.json'), 'utf-8') -) as Record }>>; +) as Record; noCSS?: boolean }>>; diff --git a/src/auto-start.ts b/src/auto-start.ts new file mode 100644 index 0000000000..f066c31e1b --- /dev/null +++ b/src/auto-start.ts @@ -0,0 +1,13 @@ +import Prism from './global'; +import autoloader from './plugins/autoloader/prism-autoloader'; +import { documentReady } from './util/async'; + +Prism.components.add(autoloader); + +void documentReady().then(() => { + if (!Prism.config.manual) { + Prism.highlightAll(); + } +}); + +export default Prism; diff --git a/src/components.json b/src/components.json index 2a86f60c61..769e4987e1 100644 --- a/src/components.json +++ b/src/components.json @@ -1,7 +1,7 @@ { "core": { "meta": { - "path": "components/prism-core.js", + "path": "index.js", "option": "mandatory" }, "core": "Core" @@ -14,27 +14,34 @@ }, "prism": { "title": "Default", - "option": "default" + "option": "default", + "owner": "LeaVerou" + }, + "dark": { + "title": "Dark", + "owner": "LeaVerou" + }, + "funky": { + "title": "Funky", + "owner": "LeaVerou" }, - "prism-dark": "Dark", - "prism-funky": "Funky", - "prism-okaidia": { + "okaidia": { "title": "Okaidia", "owner": "ocodia" }, - "prism-twilight": { + "twilight": { "title": "Twilight", "owner": "remybach" }, - "prism-coy": { + "coy": { "title": "Coy", "owner": "tshedor" }, - "prism-solarizedlight": { + "solarizedlight": { "title": "Solarized Light", "owner": "hectormatos2011 " }, - "prism-tomorrow": { + "tomorrow": { "title": "Tomorrow Night", "owner": "Rosey" } diff --git a/src/core/classes/hooks.ts b/src/core/classes/hooks.ts index be8523081c..ec28d71c0b 100644 --- a/src/core/classes/hooks.ts +++ b/src/core/classes/hooks.ts @@ -26,20 +26,23 @@ export class Hooks { ): () => void { if (Array.isArray(name)) { // One function, multiple hooks - for (let n of name) { + for (const n of name) { this.add(n, callback); } } else if (typeof name === 'object') { // Multiple hooks - let hooks = name; + const hooks = name; - for (let name in hooks) { - this.add(name, hooks[name as string]); + for (const name in hooks) { + const callback = hooks[name]; + if (callback) { + this.add(name as string, callback as HookCallback); + } } } else { - let hooks = (this._all[name] ??= []); + const hooks = (this._all[name] ??= []); hooks.push(callback as never); } @@ -54,13 +57,13 @@ export class Hooks { ): void { if (Array.isArray(name)) { // Multiple hook names, same callback - for (let n of name) { + for (const n of name) { this.remove(n, callback); } } else if (typeof name === 'object') { // Map of hook names to callbacks - for (let n in name) { + for (const n in name) { this.remove(n, callback); } } diff --git a/src/core/classes/token.ts b/src/core/classes/token.ts index f61b8ab114..ce104f25a4 100644 --- a/src/core/classes/token.ts +++ b/src/core/classes/token.ts @@ -1,6 +1,3 @@ -// eslint-disable-next-line @typescript-eslint/no-unused-vars -import type { Grammar, GrammarToken } from '../../types'; - type StandardTokenName = | 'atrule' | 'attr-name' diff --git a/src/core/highlight-element.ts b/src/core/highlight-element.ts index ed8d660ad4..db83bc8d25 100644 --- a/src/core/highlight-element.ts +++ b/src/core/highlight-element.ts @@ -1,7 +1,8 @@ import { getLanguage, setLanguage } from '../shared/dom-util'; import { htmlEncode } from '../shared/util'; +import singleton from './prism'; import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types'; -import singleton, { type Prism } from './prism'; +import type { Prism } from './prism'; /** * Highlights the code inside a single element. diff --git a/src/global.ts b/src/global.ts index 076377d3bb..29a9c0e598 100644 --- a/src/global.ts +++ b/src/global.ts @@ -1,4 +1,5 @@ -import globalPrism, { type Prism } from './core/prism'; +import globalPrism from './core/prism'; +import type { Prism } from './core/prism'; declare global { interface globalThis { diff --git a/src/languages/asciidoc.ts b/src/languages/asciidoc.ts index 6f9e309e74..2f38d390bd 100644 --- a/src/languages/asciidoc.ts +++ b/src/languages/asciidoc.ts @@ -214,7 +214,7 @@ export default { function copyFromAsciiDoc (...keys: (keyof typeof asciidoc)[]) { const o: Record = {}; for (const key of keys) { - o[key] = asciidoc[key]; + o[key] = asciidoc[key] as GrammarToken; } return o as Grammar; } diff --git a/src/languages/css.ts b/src/languages/css.ts index 31a2b8160c..9c7184ffba 100644 --- a/src/languages/css.ts +++ b/src/languages/css.ts @@ -33,7 +33,7 @@ export default { lookbehind: true, }, $rest: 'css', - }, + } as unknown as Grammar, }, 'url': { // https://drafts.csswg.org/css-values-3/#urls diff --git a/src/languages/elixir.ts b/src/languages/elixir.ts index 5eff210ee9..b1e765b8f1 100644 --- a/src/languages/elixir.ts +++ b/src/languages/elixir.ts @@ -12,7 +12,7 @@ export default { alias: 'punctuation', }, $rest: 'elixir', - }, + } as unknown as Grammar, }, }; diff --git a/src/languages/ftl.ts b/src/languages/ftl.ts index 85ef6de9c1..e7b61a5299 100644 --- a/src/languages/ftl.ts +++ b/src/languages/ftl.ts @@ -68,7 +68,7 @@ export default { 'punctuation': /[,;.:()[\]{}]/, }; - stringInterpolation.inside.interpolation.inside.$rest = ftl as Grammar; + stringInterpolation.inside.interpolation.inside.$rest = ftl as Grammar['$rest']; return { 'ftl-comment': { diff --git a/src/languages/icu-message-format.ts b/src/languages/icu-message-format.ts index 377c56bd5b..8f57844ba7 100644 --- a/src/languages/icu-message-format.ts +++ b/src/languages/icu-message-format.ts @@ -137,7 +137,7 @@ export default { alias: 'string', }, 'punctuation': /,/, - }, + } as unknown as Grammar, }, 'argument-delimiter': { pattern: /./, diff --git a/src/languages/inform7.ts b/src/languages/inform7.ts index 53f0b88473..841b05f63a 100644 --- a/src/languages/inform7.ts +++ b/src/languages/inform7.ts @@ -14,7 +14,7 @@ export default { pattern: /\[|\]/, alias: 'punctuation', }, - $rest: null as Grammar[typeof rest], + $rest: null as Grammar['$rest'], }, }, }, @@ -70,7 +70,7 @@ export default { pattern: /\S(?:\s*\S)*/, alias: 'comment', }, - }; + } as Grammar['$rest']; return inform7; }, diff --git a/src/languages/javascript.ts b/src/languages/javascript.ts index 15efa55e92..77a91bd3bb 100644 --- a/src/languages/javascript.ts +++ b/src/languages/javascript.ts @@ -206,7 +206,7 @@ export default { greedy: true, alias: 'property', }, - }, + } as unknown as Grammar, 'operator': { 'literal-property': { pattern: diff --git a/src/languages/js-templates.ts b/src/languages/js-templates.ts index 12ffe87ee1..d29a5626fb 100644 --- a/src/languages/js-templates.ts +++ b/src/languages/js-templates.ts @@ -1,6 +1,6 @@ import { embeddedIn } from '../shared/languages/templating'; import javascript, { JS_TEMPLATE, JS_TEMPLATE_INTERPOLATION } from './javascript'; -import type { GrammarToken, LanguageProto } from '../types'; +import type { Grammar, GrammarToken, LanguageProto } from '../types'; /** * Creates a new pattern to match a template string with a special tag. @@ -42,7 +42,7 @@ function createTemplate (language: string, tag: string): GrammarToken { $tokenize: embeddedIn(language), }, }, - }, + } as unknown as Grammar, }; } diff --git a/src/languages/jsx.ts b/src/languages/jsx.ts index c85c41d922..5d5e72e5fb 100644 --- a/src/languages/jsx.ts +++ b/src/languages/jsx.ts @@ -157,7 +157,7 @@ export default { alias: 'punctuation', }, $rest: 'jsx', - }, + } as unknown as Grammar, }, }); diff --git a/src/languages/lilypond.ts b/src/languages/lilypond.ts index 54bcf23616..fcada75fc0 100644 --- a/src/languages/lilypond.ts +++ b/src/languages/lilypond.ts @@ -1,5 +1,5 @@ import scheme from './scheme'; -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'lilypond', @@ -51,7 +51,7 @@ export default { }, }, 'punctuation': /#/, - }, + } as unknown as Grammar, }, 'string': { pattern: /"(?:[^"\\]|\\.)*"/, diff --git a/src/languages/lisp.ts b/src/languages/lisp.ts index 28abac83ac..235d6f3249 100644 --- a/src/languages/lisp.ts +++ b/src/languages/lisp.ts @@ -1,4 +1,4 @@ -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'lisp', @@ -202,6 +202,6 @@ export default { lookbehind: true, }, ], - }; + } as unknown as Grammar; }, } as LanguageProto<'lisp'>; diff --git a/src/languages/livescript.ts b/src/languages/livescript.ts index 85d31940f7..5d43a71cc2 100644 --- a/src/languages/livescript.ts +++ b/src/languages/livescript.ts @@ -1,4 +1,4 @@ -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'livescript', @@ -36,7 +36,7 @@ export default { }, }, 'string': /[\s\S]+/, - }, + } as unknown as Grammar, }, 'string': [ { diff --git a/src/languages/naniscript.ts b/src/languages/naniscript.ts index 8fde370258..5105e16700 100644 --- a/src/languages/naniscript.ts +++ b/src/languages/naniscript.ts @@ -1,6 +1,6 @@ import { getTextContent } from '../core/classes/token'; import { withoutTokenize } from '../util/without-tokenize'; -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto, Prism } from '../types'; function isBracketsBalanced (input: string): boolean { const brackets = '[]{}'; @@ -134,7 +134,7 @@ export default { }, }, - $tokenize (code, grammar, Prism) { + $tokenize (code: string, grammar: Grammar, Prism: Prism) { const tokens = Prism.tokenize(code, withoutTokenize(grammar)); tokens.forEach(token => { if (typeof token !== 'string' && token.type === 'generic-text') { @@ -147,6 +147,6 @@ export default { }); return tokens; }, - }; + } as unknown as Grammar; }, } as LanguageProto<'naniscript'>; diff --git a/src/languages/php.ts b/src/languages/php.ts index 81783fa44a..19b566b68d 100644 --- a/src/languages/php.ts +++ b/src/languages/php.ts @@ -1,6 +1,6 @@ import { embeddedIn } from '../shared/languages/templating'; import markup from './markup'; -import type { Grammar, LanguageProto } from '../types'; +import type { Grammar, LanguageProto, Prism } from '../types'; export default { id: 'php', @@ -370,13 +370,13 @@ export default { }, }, }, - $tokenize: (code, grammar, Prism) => { + $tokenize: (code: string, grammar: Grammar, Prism: Prism) => { if (!/<\?/.test(code)) { return Prism.tokenize(code, php); } return embedded(code, grammar, Prism); }, - }; + } as unknown as Grammar; }, } as LanguageProto<'php'>; diff --git a/src/languages/pug.ts b/src/languages/pug.ts index 2681fb26f5..45b03277e9 100644 --- a/src/languages/pug.ts +++ b/src/languages/pug.ts @@ -1,7 +1,7 @@ import { insertBefore } from '../util/insert'; import javascript from './javascript'; import markup from './markup'; -import type { GrammarTokens, LanguageProto } from '../types'; +import type { Grammar, GrammarTokens, LanguageProto } from '../types'; export default { id: 'pug', @@ -153,7 +153,7 @@ export default { }, ], 'punctuation': /[.\-!=|]+/, - }; + } as unknown as Grammar; const filter_pattern = /(^([\t ]*)):(?:(?:\r?\n|\r(?!\n))(?:\2[\t ].+|\s*?(?=\r?\n|\r)))+/.source; diff --git a/src/languages/pure.ts b/src/languages/pure.ts index b79f2298c7..ae15c121d7 100644 --- a/src/languages/pure.ts +++ b/src/languages/pure.ts @@ -1,5 +1,5 @@ import { insertBefore } from '../util/insert'; -import type { LanguageProto } from '../types'; +import type { Grammar, GrammarToken, LanguageProto } from '../types'; export default { id: 'pure', @@ -59,7 +59,7 @@ export default { /(?:[!"#$%&'*+,\-.\/:<=>?@\\^`|~\u00a1-\u00bf\u00d7-\u00f7\u20d0-\u2bff]|\b_+\b)+|\b(?:and|div|mod|not|or)\b/, // FIXME: How can we prevent | and , to be highlighted as operator when they are used alone? 'punctuation': /[(){}\[\];,|]/, - }; + } as unknown as Grammar; const inlineLanguages = ['c', { lang: 'c++', alias: 'cpp' }, 'fortran']; const inlineLanguageRe = /%< *-\*- *\d* *-\*-[\s\S]+?%>/.source; @@ -85,11 +85,11 @@ export default { 'i' ), inside: { - ...pure['inline-lang'].inside, + ...((pure['inline-lang'] as GrammarToken).inside as Grammar), $rest: alias, }, }, - }); + } as unknown as Grammar); }); return pure; diff --git a/src/languages/python.ts b/src/languages/python.ts index e8589178f3..e37d634085 100644 --- a/src/languages/python.ts +++ b/src/languages/python.ts @@ -1,4 +1,4 @@ -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'python', @@ -67,5 +67,5 @@ export default { /\b0(?:b(?:_?[01])+|o(?:_?[0-7])+|x(?:_?[a-f0-9])+)\b|(?:\b\d+(?:_\d+)*(?:\.(?:\d+(?:_\d+)*)?)?|\B\.\d+(?:_\d+)*)(?:e[+-]?\d+(?:_\d+)*)?j?(?!\w)/i, 'operator': /[-+%=]=?|!=|:=|\*\*?=?|\/\/?=?|<[<=>]?|>[=>]?|[&|^~]/, 'punctuation': /[{}[\];(),.:]/, - }, + } as unknown as Grammar, } as LanguageProto<'python'>; diff --git a/src/languages/rust.ts b/src/languages/rust.ts index bbbf384c95..319edd87cd 100644 --- a/src/languages/rust.ts +++ b/src/languages/rust.ts @@ -1,4 +1,4 @@ -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'rust', @@ -53,7 +53,7 @@ export default { alias: 'punctuation', }, $rest: 'rust', - }, + } as unknown as Grammar, }, 'lifetime-annotation': { diff --git a/src/languages/sas.ts b/src/languages/sas.ts index ed2f7b513e..f5f92e4d64 100644 --- a/src/languages/sas.ts +++ b/src/languages/sas.ts @@ -1,4 +1,4 @@ -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'sas', @@ -276,7 +276,7 @@ export default { 'numeric-constant': numericConstant, 'punctuation': punctuation, 'string': string, - }, + } as unknown as Grammar, }, 'proc-args': { diff --git a/src/languages/scss.ts b/src/languages/scss.ts index f593803cc7..0618978b94 100644 --- a/src/languages/scss.ts +++ b/src/languages/scss.ts @@ -15,7 +15,7 @@ export default { inside: { 'rule': /@[\w-]+/, $rest: 'scss', - }, + } as unknown as Grammar, }, // url, compassified 'url': /(?:[-a-z]+-)?url(?=\()/i, diff --git a/src/languages/stylus.ts b/src/languages/stylus.ts index ec35b52e00..d09250eefb 100644 --- a/src/languages/stylus.ts +++ b/src/languages/stylus.ts @@ -34,14 +34,14 @@ export default { pattern: /^\{|\}$/, alias: 'punctuation', }, - $rest: null as Grammar[typeof rest], + $rest: null as Grammar['$rest'], }, }, 'func': { pattern: /[\w-]+\([^)]*\).*/, inside: { 'function': /^[^(]+/, - $rest: null as Grammar[typeof rest], + $rest: null as Grammar['$rest'], }, }, 'important': /\B!(?:important|optional)\b/i, @@ -75,8 +75,8 @@ export default { 'punctuation': /[{}()\[\];:,]/, }; - inside['interpolation'].inside.$rest = inside; - inside['func'].inside.$rest = inside; + inside['interpolation'].inside.$rest = inside as Grammar['$rest']; + inside['func'].inside.$rest = inside as Grammar['$rest']; return { 'atrule-declaration': { @@ -84,7 +84,7 @@ export default { lookbehind: true, inside: { 'atrule': /^@[\w-]+/, - $rest: inside, + $rest: inside as Grammar['$rest'], }, }, 'variable-declaration': { @@ -92,7 +92,7 @@ export default { lookbehind: true, inside: { 'variable': /^\S+/, - $rest: inside, + $rest: inside as Grammar['$rest'], }, }, @@ -101,7 +101,7 @@ export default { lookbehind: true, inside: { 'keyword': /^\S+/, - $rest: inside, + $rest: inside as Grammar['$rest'], }, }, @@ -118,7 +118,7 @@ export default { 'interpolation': inside.interpolation, }, }, - $rest: inside, + $rest: inside as Grammar['$rest'], }, }, diff --git a/src/languages/treeview.ts b/src/languages/treeview.ts index 3036180282..0dae024bd4 100644 --- a/src/languages/treeview.ts +++ b/src/languages/treeview.ts @@ -1,6 +1,6 @@ import { getTextContent } from '../core/classes/token'; import { withoutTokenize } from '../util/without-tokenize'; -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto, Prism } from '../types'; export default { id: 'treeview', @@ -44,7 +44,7 @@ export default { }, }, }, - $tokenize (code, grammar, Prism) { + $tokenize (code: string, grammar: Grammar, Prism: Prism) { const tokens = Prism.tokenize(code, withoutTokenize(grammar)); for (const token of tokens) { diff --git a/src/languages/wiki.ts b/src/languages/wiki.ts index 1307003539..ebffd8db7d 100644 --- a/src/languages/wiki.ts +++ b/src/languages/wiki.ts @@ -1,5 +1,5 @@ import markup from './markup'; -import type { GrammarToken, LanguageProto } from '../types'; +import type { Grammar, GrammarToken, LanguageProto } from '../types'; export default { id: 'wiki', @@ -69,7 +69,7 @@ export default { alias: 'punctuation', }, $rest: tag.inside, - }, + } as Grammar, }, 'punctuation': /^(?:\{\||\|\}|\|-|[*#:;!|])|\|\||!!/m, $insert: { diff --git a/src/plugins/autolinker/README.md b/src/plugins/autolinker/README.md new file mode 100644 index 0000000000..282935d0dd --- /dev/null +++ b/src/plugins/autolinker/README.md @@ -0,0 +1,53 @@ +--- +title: Autolinker +description: Converts URLs and emails in code to clickable links. Parses Markdown links in comments. +owner: LeaVerou +--- + +
+ +# How to use + +URLs and emails will be linked automatically, you don’t need to do anything. To link some text inside a comment to a certain URL, you may use the Markdown syntax: + +```markdown +[Text you want to see](https://url-goes-here.com) +``` +
+ +
+ +# Examples + +## JavaScript + +```js +/** + * Prism: Lightweight, robust, elegant syntax highlighting + * MIT license https://www.opensource.org/licenses/mit-license.php/ + * @author Lea Verou https://lea.verou.me + * Reach Lea at fake@email.com (no, not really) + * And this is [a Markdown link](https://prismjs.com). Sweet, huh? + */ +let foo = 5; +// And a single line comment https://google.com +``` + +## CSS + +```css +@font-face { + src: url(https://lea.verou.me/logo.otf); + font-family: 'LeaVerou'; +} +``` + +## HTML + +```html + +In attributes too! +

Autolinking in raw text: https://prismjs.com

+``` +
diff --git a/src/plugins/autolinker/prism-autolinker.ts b/src/plugins/autolinker/prism-autolinker.ts index c3a34b450e..33519adafa 100644 --- a/src/plugins/autolinker/prism-autolinker.ts +++ b/src/plugins/autolinker/prism-autolinker.ts @@ -1,4 +1,3 @@ -import { HookEnv, HookCallback } from '../../core/classes/hooks'; import { tokenizeStrings } from '../../shared/tokenize-strings'; import type { PluginProto } from '../../types'; diff --git a/src/plugins/autoloader/README.md b/src/plugins/autoloader/README.md new file mode 100644 index 0000000000..a895548f2a --- /dev/null +++ b/src/plugins/autoloader/README.md @@ -0,0 +1,132 @@ +--- +title: Autoloader +description: Automatically loads the needed languages to highlight the code blocks. +owner: Golmote +noCSS: true +resources: + - https://dev.prismjs.com/components.js + - https://prismjs.com/assets/vendor/jszip.min.js + - https://prismjs.com/assets/vendor/FileSaver.min.js + - ./demo.js +--- + + + +
+ +# How to use + +The plugin will automatically handle missing grammars and load them for you. To do this, you need to provide a URL to a directory of all the grammars you want. This can be the path to a local directory with all grammars or a CDN URL. + +You can download all the available grammars by clicking on the following link: .
+Alternatively, you can also clone the GitHub repo and take the `components` folder from there. Read our [usage section](https://prismjs.com/index.html#basic-usage-cdn) to use a CDN. + +You can then download Prism core and any plugins from the [Download](https://prismjs.com/download.html) page, without checking any languages (or just check the languages you want to load by default, e.g. if you're using a language a lot, then you probably want to save the extra HTTP request). + +A couple of additional options are available through the configuration object `Prism.plugins.autoloader`. + +## Specifying the grammars path + +By default, the plugin will look for the missing grammars in the `components` folder. If your files are in a different location, you can specify it using the `languages_path` option: + +``` +Prism.plugins.autoloader.languages_path = 'path/to/grammars/'; +``` + +_Note:_ Autoloader is pretty good at guessing this path. You most likely won't have to change this path. + +## Using development versions + +By default, the plugin uses the minified versions of the grammars. If you wish to use the development versions instead, you can set the `use_minified` option to false: + +``` +Prism.plugins.autoloader.use_minified = false; +``` + +## Specifying additional dependencies + +All default dependencies are already included in the plugin. However, there are some cases where you might want to load an additional dependency for a specific code block. To do so, just add a `data-dependencies` attribute on you `` or `
` tags, containing a list of comma-separated language aliases.
+
+```markup
+

+:less
+	foo {
+		color: @red;
+	}
+
+```
+
+## Force to reload a grammar
+
+The plugin usually doesn't reload a grammar if it already exists. In some very specific cases, you might however want to do so. If you add an exclamation mark after an alias in the `data-dependencies` attribute, this language will be reloaded.
+
+```html
+

+```
+
+
+ +
+ +# Examples + +Note that no languages are loaded on this page by default. + +Basic usage with some Perl code: + +```perl +my ($class, $filename) = @_; +``` + +Alias support with TypeScript's `ts`: + +```ts +const a: number = 0; +``` + +The Less filter used in Pug: + +```pug +:less + foo { + color: @red; + } +``` + +# Markdown + +Markdown will use the Autoloader to automatically load missing languages. + +````markdown +The C# code will be highlighted __after__ the rest of this document. + +```csharp +public class Foo : IBar { + public string Baz { get; set; } = "foo"; +} +``` + +The CSS code will be highlighted with this document because CSS has already been loaded. + +```css +a:hover { + color: green !important; +} +``` +```` + +
diff --git a/src/plugins/autoloader/demo.js b/src/plugins/autoloader/demo.js new file mode 100644 index 0000000000..608d0e51ed --- /dev/null +++ b/src/plugins/autoloader/demo.js @@ -0,0 +1,58 @@ +async function getZip (files, elt) { + let process = async () => { + elt.setAttribute('data-progress', Math.round((i / l) * 100)); + if (i < l) { + await addFile(zip, files[i][0], files[i][1]); + i++; + await process(); + } + }; + + let zip = new JSZip(); + let l = files.length; + let i = 0; + + await process(); + + return zip; +} + +async function addFile (zip, filename, filepath) { + let contents = await getFileContents(filepath); + zip.file(filename, contents); +} + +async function getFileContents (filepath) { + let response = await fetch(filepath); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + return response.text(); +} + +document.querySelector('.download-grammars').addEventListener('click', async ({ target }) => { + let btn = target; + if (btn.classList.contains('loading')) { + return; + } + btn.classList.add('loading'); + btn.setAttribute('data-progress', 0); + + let files = []; + for (let id in components.languages) { + if (id === 'meta') { + continue; + } + let basepath = + 'https://dev.prismjs.com/' + components.languages.meta.path.replace(/\{id}/g, id); + let basename = basepath.substring(basepath.lastIndexOf('/') + 1); + files.push([basename + '.js', basepath + '.js']); + files.push([basename + '.min.js', basepath + '.min.js']); + } + + let zip = await getZip(files, btn); + btn.classList.remove('loading'); + + let blob = await zip.generateAsync({ type: 'blob' }); + saveAs(blob, 'prism-components.zip'); +}); diff --git a/src/plugins/autoloader/prism-autoloader.ts b/src/plugins/autoloader/prism-autoloader.ts index 595758a1be..dd64b9c9a9 100644 --- a/src/plugins/autoloader/prism-autoloader.ts +++ b/src/plugins/autoloader/prism-autoloader.ts @@ -146,7 +146,8 @@ export default { return; } - Prism.plugins.autoloader.loadLanguages(deps).then( + const autoloader = Prism.plugins.autoloader as Autoloader; + autoloader.loadLanguages(deps).then( () => Prism.highlightElement(element), (reason) => { console.error(`Failed to load languages (${deps.join(', ')}): ${String(reason)}`); diff --git a/src/plugins/command-line/README.md b/src/plugins/command-line/README.md new file mode 100644 index 0000000000..7c087c3f92 --- /dev/null +++ b/src/plugins/command-line/README.md @@ -0,0 +1,233 @@ +--- +title: Command Line +description: Display a command line with a prompt and, optionally, the output/response from the commands. +owner: chriswells0 +resources: + - https://dev.prismjs.com/components/prism-bash.js + - https://dev.prismjs.com/components/prism-powershell.js + - https://dev.prismjs.com/components/prism-sql.js +--- + +
+ +# How to use + +This is intended for code blocks (`
`) and not for inline code.
+
+Add class **command-line** to your `
`. For a server command line, specify the user and host names using the `data-user` and `data-host` attributes. The resulting prompt displays a **#** for the root user and **$** for all other users. For any other command line, such as a Windows prompt, you may specify the entire prompt using the `data-prompt` attribute.
+
+## Optional: Command output (positional)
+
+You may specify the lines to be presented as output (no prompt and no highlighting) through the `data-output` attribute on the `
` element in the following simple format:
+
+- A single number refers to the line with that number
+- Ranges are denoted by two numbers, separated with a hyphen (-)
+- Multiple line numbers or ranges are separated by commas.
+- Whitespace is allowed anywhere and will be stripped off.
+
+Examples:
+
+5
+
+: The 5th line
+
+1-5
+
+: Lines 1 through 5
+
+1,4
+
+: Line 1 and line 4
+
+1-2, 5, 9-20
+
+: Lines 1 through 2, line 5, lines 9 through 20
+
+## Optional: Command output (prefix)
+
+To automatically present some lines as output, you can prefix those lines with any string and specify the prefix using the `data-filter-output` attribute on the `
` element. For example, `data-filter-output="(out)"` will treat lines beginning with `(out)` as output and remove the prefix.
+
+A blank line will render as an empty line with a prompt. If you want an empty line without a prompt then you can use a line containing just the output prefix, e.g. `(out)`. See the blank lines in the examples below.
+
+Output lines are user selectable by default, so if you select the whole content of the code block, it will select the shell commands and any output lines. This may not be desirable if you want to copy/paste just the commands and not the output. If you want to make the output not user selectable then add the following to your CSS:
+
+```css
+.command-line span.token.output {
+	user-select: none;
+}
+```
+
+## Optional: Multi-line commands
+
+You can configure the plugin to handle multi-line commands. This can be done in two ways; setting a line continuation string (as in Bash); or explicitly marking continuation lines with a prefix for languages that do not have a continuation string/character, e.g. SQL, Scala, etc..
+
+
+`data-continuation-str`
+
+: Set this attribute to the line continuation string/character, e.g. for bash `data-continuation-str="\"`
+
+`data-filter-continuation`
+
+: This works in a similar way to `data-filter-output`. Prefix all continuation lines with the value of `data-filter-continuation` and they will be displayed with the prompt set in `data-continuation-prompt`. For example, `data-filter-continuation="(con)"` will treat lines beginning with `(con)` as continuation lines and remove the prefix.
+
+`data-continuation-prompt`
+
+: Set this attribute to define the prompt to be displayed when the command has continued beyond the first line (whether using line continuation or command termination), e.g. for MySQL `data-continuation-prompt="->"`. If this attribute is not set then a default of `>` will be used.
+
+
+ +
+ +# Examples + +## Default Use Without Output + +```html +
+```
+
+```bash { .command-line }
+cd ~/.vim
+
+vim vimrc
+```
+
+## Root User Without Output
+
+```html
+
+```
+
+```bash { .command-line data-user="root" data-host="localhost" }
+cd /usr/local/etc
+cp php.ini php.ini.bak
+vi php.ini
+```
+
+## Non-Root User With Output
+
+```html
+
+```
+
+```bash { .command-line data-user="chris" data-host="remotehost" data-output="2, 4-8" }
+pwd
+/usr/home/chris/bin
+ls -la
+total 2
+drwxr-xr-x   2 chris  chris     11 Jan 10 16:48 .
+drwxr--r-x  45 chris  chris     92 Feb 14 11:10 ..
+-rwxr-xr-x   1 chris  chris    444 Aug 25  2013 backup
+-rwxr-xr-x   1 chris  chris    642 Jan 17 14:42 deploy
+```
+
+## Windows PowerShell With Output
+
+```html
+
+```
+
+```powershell { .command-line data-prompt="PS C:\Users\Chris>" data-output="2-19" }
+dir
+
+
+    Directory: C:\Users\Chris
+
+
+Mode                LastWriteTime     Length Name
+----                -------------     ------ ----
+d-r--        10/14/2015   5:06 PM            Contacts
+d-r--        12/12/2015   1:47 PM            Desktop
+d-r--         11/4/2015   7:59 PM            Documents
+d-r--        10/14/2015   5:06 PM            Downloads
+d-r--        10/14/2015   5:06 PM            Favorites
+d-r--        10/14/2015   5:06 PM            Links
+d-r--        10/14/2015   5:06 PM            Music
+d-r--        10/14/2015   5:06 PM            Pictures
+d-r--        10/14/2015   5:06 PM            Saved Games
+d-r--        10/14/2015   5:06 PM            Searches
+d-r--        10/14/2015   5:06 PM            Videos
+```
+
+## Line continuation with Output (bash)
+
+```html
+
+```
+
+```bash { .command-line data-filter-output="(out)" data-continuation-str="\" }
+export MY_VAR=123
+echo "hello"
+(out)hello
+echo one \
+two \
+three
+(out)one two three
+(out)
+echo "goodbye"
+(out)goodbye
+```
+
+## Line continuation with Output (PowerShell)
+
+```html
+
+```
+
+
Write-Host `
+'Hello' `
+'from' `
+'PowerShell!'
+(out)Hello from PowerShell!
+Write-Host 'Goodbye from PowerShell!'
+(out)Goodbye from PowerShell!

+
+## Line continuation using prefix (MySQL/SQL)
+
+```html
+
+```
+
+```sql { .command-line data-prompt="mysql>" data-continuation-prompt="->" data-filter-output="(out)" data-filter-continuation="(con)" }
+set @my_var = 'foo';
+set @my_other_var = 'bar';
+(out)
+CREATE TABLE people (
+(con)first_name VARCHAR(30) NOT NULL,
+(con)last_name VARCHAR(30) NOT NULL
+(con));
+(out)Query OK, 0 rows affected (0.09 sec)
+(out)
+insert into people
+(con)values ('John', 'Doe');
+(out)Query OK, 1 row affected (0.02 sec)
+(out)
+select *
+(con)from people
+(con)order by last_name;
+(out)+------------+-----------+
+(out)| first_name | last_name |
+(out)+------------+-----------+
+(out)| John       | Doe       |
+(out)+------------+-----------+
+(out)1 row in set (0.00 sec)
+```
+
+
diff --git a/src/plugins/copy-to-clipboard/README.md b/src/plugins/copy-to-clipboard/README.md new file mode 100644 index 0000000000..7268a4a664 --- /dev/null +++ b/src/plugins/copy-to-clipboard/README.md @@ -0,0 +1,214 @@ +--- +title: Copy to Clipboard +description: Add a button that copies the code block to the clipboard when clicked. +owner: mAAdhaTTah +require: toolbar +noCSS: true +body_classes: language-text +resources: + - ../autoloader/prism-autoloader.js { type="module" } + - ../toolbar/prism-toolbar.css + - ../toolbar/prism-toolbar.js { type="module" } +--- + +
+ +# How to use + +The plugin depends on the Prism [Toolbar](../toolbar) plugin. In addition to including the plugin file with your PrismJS build, ensure it is loaded before the plugin. + +
+ +
+ +# Settings + +By default, the plugin shows messages in English and sets a 5-second timeout after a click. You can use the following HTML5 data attributes to override the default settings: + +- `data-prismjs-copy`{ .token .attr-name } — default message displayed by Copy to Clipboard; +- `data-prismjs-copy-error`{ .token .attr-name } — a message displayed after failing copying, prompting the user to press Ctrl+C; +- `data-prismjs-copy-success`{ .token .attr-name } — a message displayed after a successful copying; +- `data-prismjs-copy-timeout`{ .token .attr-name } — a timeout (in milliseconds) after copying. Once the timeout passed, the success or error message will revert back to the default message. The value should be a non-negative integer. + +The plugin traverses up the DOM tree to find each of these attributes. The search starts at every `pre code`{ .token .tag } element and stops at the closest ancestor element that has a desired attribute or at the worst case, at the `html`{ .token .tag } element. + +**Warning!** Although possible, you definitely shouldn't add these attributes to the `html`{ .token .tag } element, because a human-readable text should be placed _after_ the character encoding declaration (``), and the latter [must be](https://html.spec.whatwg.org/multipage/semantics.html#charset) serialized completely within the first 512 (in older browsers) or 1024 bytes of the document. Consider using the `body`{ .token .tag } element or one of its descendants. + +
+ +
+ +# Styling + +This plugin supports customizing the style of the copy button. To understand how this is done, let's look at the HTML structure of the copy button: + +```html + +``` + +The `copy-to-clipboard-button` class can be used to select the button. The `data-copy-state` attribute indicates the current state of the plugin with the 3 possible states being: + +- `data-copy-state="copy"` — default state; +- `data-copy-state="copy-error"` — the state after failing copying; +- `data-copy-state="copy-success"` — the state after successful copying; + +These 3 states should be conveyed to the user either by different styling or displaying the button text. + +
+ +
+ +# Examples + +## Sharing + +The following code blocks show modified messages and both use a half-second timeout. The other settings are set to default. + +Source code: + +```html + + 
console.log('Hello, world!');
+ + 
int main() {
+	return 0;
+}
+ +``` + +Output: + +
+ +```js { data-prismjs-copy="Copy the JavaScript snippet!" } +console.log('Hello, world!'); +``` + +```c { data-prismjs-copy="Copy the C snippet!" } +int main() { + return 0; +} +``` + +
+ +## Inheritance + +The plugin always use the closest ancestor element that has a desired attribute, so it's possible to override any setting on any descendant. In the following example, the `baz`{ .token .attr-value } message is used. The other settings are set to default. + +Source code: + +```html + +
+ 
int main() {
+	return 0;
+}
+
+ +``` + +Output: + +
+
+ +```c { data-prismjs-copy="baz" } +int main() { + return 0; +} +``` + +
+
+ +## i18n + +You can use the data attributes for internationalization. + +The following code blocks use shared messages in Russian and the default 5-second timeout. + +Source code: + +```html + + + + + 
int main() {
+	return 0;
+}
+ + 
console.log('Hello, world!');
+ + +``` + +Output: + +
+ +```c +int main() { + return 0; +} +``` + +```js +console.log('Hello, world!'); +``` + +
+ +The next HTML document is in English, but some code blocks show messages in Russian and simplified Mainland Chinese. The other settings are set to default. + +Source code: + +```html + + + + 
console.log('Hello, world!');
+ + 
console.log('Привет, мир!');
+ + 
console.log('你好，世界！');
+ + +``` + +Output: + +```js +console.log('Hello, world!'); +``` + +```js { lang="ru" data-prismjs-copy="Скопировать" data-prismjs-copy-error="Нажмите Ctrl+C, чтобы скопировать" data-prismjs-copy-success="Скопировано!" } +console.log('Привет, мир!'); +``` + +```js { lang="zh-Hans-CN" data-prismjs-copy="复制文本" data-prismjs-copy-error="按Ctrl+C复制" data-prismjs-copy-success="文本已复制！" } +console.log('你好，世界！'); +``` + +
diff --git a/src/plugins/copy-to-clipboard/prism-copy-to-clipboard.ts b/src/plugins/copy-to-clipboard/prism-copy-to-clipboard.ts index 4d38b8412f..0d0ae86f6a 100644 --- a/src/plugins/copy-to-clipboard/prism-copy-to-clipboard.ts +++ b/src/plugins/copy-to-clipboard/prism-copy-to-clipboard.ts @@ -1,5 +1,6 @@ import toolbar from '../toolbar/prism-toolbar'; import type { PluginProto } from '../../types'; +import type { Toolbar } from '../toolbar/prism-toolbar'; interface CopyInfo { getText: () => string; @@ -118,8 +119,7 @@ export default { id: 'copy-to-clipboard', require: toolbar, effect(Prism) { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const toolbar = Prism.plugins.toolbar!; + const toolbar = Prism.plugins.toolbar as Toolbar; return toolbar.registerButton('copy-to-clipboard', (env) => { const element = env.element; diff --git a/src/plugins/custom-class/README.md b/src/plugins/custom-class/README.md new file mode 100644 index 0000000000..18ede3e6e7 --- /dev/null +++ b/src/plugins/custom-class/README.md @@ -0,0 +1,186 @@ +--- +title: Custom Class +description: This plugin allows you to prefix Prism's default classes (`.comment` can become `.namespace--comment`) or replace them with your defined ones (like `.editor__comment`). You can even add new classes. +owner: dvkndn +noCSS: true +body_classes: language-javascript +--- + +
+ +# Motivation + +Prism default classes are sensible but fixed and too generic. This plugin provide some ways to customize those classes to suit your needs. Example usages: + +- You want to add namespace for all of them (like `.prism--comment`) to avoid conflict with your existing classes. +- You use a naming convention (like [BEM](https://en.bem.info/method)). You want to write classes like `.editor__comment`. +- You use [CSS Modules](https://github.com/css-modules/css-modules). You want to use your hashed classes, like `.comment_7sh3a`. +- You need more granular control about the classes of certain tokens. You can define functions which will add new classes to tokens, so selectively change the highlighting of certain parts of your code. + +
+ +
+ +# How to use + +## Prefix all Prism classes + +``` +Prism.plugins.customClass.prefix('prism--') +``` + +## Replace some Prism classes with ones you defined + +```js +Prism.plugins.customClass.map({ + keyword: 'special-keyword', + string: 'string_ch29s', + comment: 'comment_93jsa' +}); +``` + +Object's keys are the classes you want to replace (eg: `comment`), with their values being the classes you want to use (eg: `my-comment`). Classes which are not specified will stay as they are. + +Alternatively you can also pass a function that takes the original class and returns the mapped class. This function can also be used implement language specific mapped classes. +Example: + +```js +Prism.plugins.customClass.map((className, language) => { + if (language === 'css') { + return cssSpecificMap[className] || className; + } else { + return className; + } +}); +``` + +## Add new classes + +You can add new classes with per-token and per-language precision. + +```js +Prism.plugins.customClass.add(({content, type, language}) => { + if (content === 'content' && type === 'property' && language === 'css') { + return 'content-property'; + } +}); +``` + +**Note:** The given `content` is the inner HTML of the current token. All `<` and `&` characters are escaped and it might contain the HTML code of nested tokens. + +
+ +
+ +# Notes + +- Feature functions must be called **AFTER** Prism and this plugin. For example: + + ```html + + + + + + + ``` + +- In most cases, using 1 feature is enough. However, it is possible to use both of them together if you want (Result will be like `.my-namespace--comment_93jsa`). + +## CSS Modules Usage + +The initial purpose of this plugin is to be used with CSS Modules. It works perfectly with the class map object returned by CSS Modules. For example: + +```js +import Prism from 'prismjs'; +import classMap from 'styles/editor-class-map.css'; +Prism.plugins.customClass.map(classMap) +``` + +**Note:** This plugin only affects generated token elements (usually of the form `span.token`). The classes of `code` and `pre` elements as well as all elements generated by other plugins (e.g. [Toolbar](../toolbar) elements and [line number](../line-numbers) elements) will not be changed. + +
+ +
+ +# Example + +## Prefix and map classes + +Input + +```html +

+	var foo = 'bar';
+
+``` + +Options + +```js +Prism.plugins.customClass.map({ + keyword: 'special-keyword', + string: 'my-string' +}); +Prism.plugins.customClass.prefix('pr-'); +``` + +Output + +```html +

+	var
+	foo
+	=
+	'bar'
+	;
+
+``` + +Note that this plugin only affects tokens. The classes of the `code` and `pre` elements won't be prefixed. + +## Add new classes + +Input + +```html +

+a::after {
+	content: '\2b00 ';
+	opacity: .7;
+}
+
+``` + +Options + +```js +Prism.plugins.customClass.add(({language, type, content}) => { + if (content === 'content' && type === 'property' && language === 'css') { + return 'content-property'; + } +}); +``` + +Output + +```html +

+a::after
+{
+	content
+	:
+	'\2b00 '
+	;
+	opacity
+	:
+	 .7
+	;
+}
+
+``` + +
diff --git a/src/plugins/custom-class/prism-custom-class.ts b/src/plugins/custom-class/prism-custom-class.ts index e95487cd46..8710010236 100644 --- a/src/plugins/custom-class/prism-custom-class.ts +++ b/src/plugins/custom-class/prism-custom-class.ts @@ -53,7 +53,7 @@ export default { return new CustomClass(); }, effect(Prism) { - const customClass = Prism.plugins.customClass; + const customClass = Prism.plugins.customClass as CustomClass; return Prism.hooks.add('wrap', (env) => { if (customClass['adder']) { @@ -74,7 +74,7 @@ export default { return; } - env.classes = env.classes.map((c) => customClass.apply(c)); + env.classes = env.classes.map((c: string) => customClass.apply(c)); }); } } as PluginProto<'custom-class'>; diff --git a/src/plugins/data-uri-highlight/README.md b/src/plugins/data-uri-highlight/README.md new file mode 100644 index 0000000000..a57700c9c1 --- /dev/null +++ b/src/plugins/data-uri-highlight/README.md @@ -0,0 +1,36 @@ +--- +title: Data URI Highlight +description: Highlights data-URI contents. +owner: Golmote +noCSS: true +resources: ../autolinker/prism-autolinker.css +--- + +
+ +# How to use + +Data-URIs will be highlighted automatically, provided the needed grammar is loaded. The grammar to use is guessed using the MIME type information. + +
+ +
+ +# Example + +```css +div { + border: 40px solid transparent; + border-image: 33.334% url('data:image/svg+xml, \ + \ + \ + \ + \ + '); + padding: 1em; + max-width: 20em; + font: 130%/1.6 Baskerville, Palatino, serif; +} +``` + +
diff --git a/src/plugins/diff-highlight/README.md b/src/plugins/diff-highlight/README.md new file mode 100644 index 0000000000..72ef8ffd93 --- /dev/null +++ b/src/plugins/diff-highlight/README.md @@ -0,0 +1,84 @@ +--- +title: Diff Highlight +description: Highlight the code inside diff blocks. +owner: RunDevelopment +require: diff +resources: ../autoloader/prism-autoloader.js { type="module" } +--- + +
+ +# How to use + +Replace the `language-diff` of your code block with a `language-diff-xxxx` class to enable syntax highlighting for diff blocks. + +Optional: +You can add the `diff-highlight` class to your code block to indicate changes using the background color of a line rather than the color of the text. + +## Autoloader + +The [Autoloader plugin](../autoloader) understands the `language-diff-xxxx` format and will ensure that the language definitions for both Diff and the code language are loaded. + +
+ +
+ +# Example + +Using `class="language-diff"`: + +```diff +@@ -4,6 +4,5 @@ +- let foo = bar.baz([1, 2, 3]); +- foo = foo + 1; ++ const foo = bar.baz([1, 2, 3]) + 1; + console.log(`foo: ${foo}`); +``` + +Using `class="language-diff diff-highlight"`: + +```diff { .diff-highlight } +@@ -4,6 +4,5 @@ +- let foo = bar.baz([1, 2, 3]); +- foo = foo + 1; ++ const foo = bar.baz([1, 2, 3]) + 1; + console.log(`foo: ${foo}`); +``` + +Using `class="language-diff-javascript"`: + +```diff-javascript +@@ -4,6 +4,5 @@ +- let foo = bar.baz([1, 2, 3]); +- foo = foo + 1; ++ const foo = bar.baz([1, 2, 3]) + 1; + console.log(`foo: ${foo}`); +``` + +Using `class="language-diff-javascript diff-highlight"`: + +```diff-javascript { .diff-highlight } +@@ -4,6 +4,5 @@ +- let foo = bar.baz([1, 2, 3]); +- foo = foo + 1; ++ const foo = bar.baz([1, 2, 3]) + 1; + console.log(`foo: ${foo}`); +``` + +Using `class="language-diff-rust diff-highlight"`: +(Autoloader is used to load the Rust language definition.) + +```diff-rust { .diff-highlight } +@@ -111,6 +114,9 @@ + nasty_btree_map.insert(i, MyLeafNode(i)); + } + ++ let mut zst_btree_map: BTreeMap<(), ()> = BTreeMap::new(); ++ zst_btree_map.insert((), ()); ++ + // VecDeque + let mut vec_deque = VecDeque::new(); + vec_deque.push_back(5); +``` + +
diff --git a/src/plugins/diff-highlight/prism-diff-highlight.ts b/src/plugins/diff-highlight/prism-diff-highlight.ts index 07811fb3d9..398558fdec 100644 --- a/src/plugins/diff-highlight/prism-diff-highlight.ts +++ b/src/plugins/diff-highlight/prism-diff-highlight.ts @@ -44,7 +44,7 @@ export default { return new Token('prefix', PREFIXES[type], /\w+/.exec(type)?.[0]); }; - const withoutPrefixes = token.content.filter((t) => typeof t === 'string' || t.type !== 'prefix'); + const withoutPrefixes = token.content.filter((t: any) => typeof t === 'string' || t.type !== 'prefix'); const prefixCount = token.content.length - withoutPrefixes.length; const diffTokens = Prism.tokenize(getTextContent(withoutPrefixes), diffGrammar); diff --git a/src/plugins/download-button/README.md b/src/plugins/download-button/README.md new file mode 100644 index 0000000000..8600d70c18 --- /dev/null +++ b/src/plugins/download-button/README.md @@ -0,0 +1,39 @@ +--- +title: Download Button +description: A button in the toolbar of a code block adding a convenient way to download a code file. +owner: Golmote +require: toolbar +noCSS: true +resources: + - ../toolbar/prism-toolbar.css + - ../toolbar/prism-toolbar.js { type="module" } +--- + +
+ +# How to use + +Use the `data-src` and `data-download-link` attribute on a `
` elements similar to [Autoloader](../autoloader), like so:
+
+```html
+

+```
+
+Optionally, the text of the button can also be customized by using a `data-download-link-label` attribute.
+
+```html
+

+```
+
+

+
+

+
+# Examples
+
+The plugin’s JS code:
+

+
+This page:
+

+

diff --git a/src/plugins/download-button/prism-download-button.ts b/src/plugins/download-button/prism-download-button.ts
index b2d64601f6..d091ca728d 100644
--- a/src/plugins/download-button/prism-download-button.ts
+++ b/src/plugins/download-button/prism-download-button.ts
@@ -1,13 +1,13 @@
 import { getParentPre } from '../../shared/dom-util';
 import toolbar from '../toolbar/prism-toolbar';
 import type { PluginProto } from '../../types';
+import type { Toolbar } from '../toolbar/prism-toolbar';
 
 export default {
 	id: 'download-button',
 	require: toolbar,
 	effect(Prism) {
-		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-		const toolbar = Prism.plugins.toolbar!;
+		const toolbar = Prism.plugins.toolbar as Toolbar;
 
 		return toolbar.registerButton('download-file', (env) => {
 			const pre = getParentPre(env.element);
diff --git a/src/plugins/file-highlight/README.md b/src/plugins/file-highlight/README.md
new file mode 100644
index 0000000000..5500667b64
--- /dev/null
+++ b/src/plugins/file-highlight/README.md
@@ -0,0 +1,59 @@
+---
+title: File Highlight
+description: Fetch external files and highlight them with Prism. Used on the Prism website itself.
+owner: LeaVerou
+noCSS: true
+resources:
+  - ../line-numbers/prism-line-numbers.css
+  - ../line-numbers/prism-line-numbers.js { type="module" }
+---
+
+

+
+# How to use
+
+Use the `data-src` attribute on empty `
` elements, like so:
+
+```
+

+```
+
+You don’t need to specify the language, it’s automatically determined by the file extension. If, however, the language cannot be determined from the file extension or the file extension is incorrect, you may specify a language as well (with the usual class name way).
+
+Use the `data-range` attribute to display only a selected range of lines from the file, like so:
+
+```
+

+```
+
+Lines start at 1, so `"1,5"` will display line 1 up to and including line 5. It's also possible to specify just a single line (e.g. `"5"` for just line 5) and open ranges (e.g. `"3,"` for all lines starting at line 3). Negative integers can be used to specify the n-th last line, e.g. `-2` for the second last line.
+
+When `data-range` is used in conjunction with the [Line Numbers plugin](../line-numbers), this plugin will add the proper `data-start` according to the specified range. This behavior can be overridden by setting the `data-start` attribute manually.
+
+Please note that the files are fetched with XMLHttpRequest. This means that if the file is on a different origin, fetching it will fail, unless CORS is enabled on that website.
+
+

+
+

+
+# Examples
+
+The plugin’s JS code:
+
+

+
+This page:
+
+

+
+File that doesn’t exist:
+
+

+
+With line numbers, and `data-range="12,111"`:
+
+

+
+For more examples, browse around the Prism website. Most large code samples are actually files fetched with this plugin.
+
+

diff --git a/src/plugins/file-highlight/prism-file-highlight.ts b/src/plugins/file-highlight/prism-file-highlight.ts
index 394a277472..8f3f98c923 100644
--- a/src/plugins/file-highlight/prism-file-highlight.ts
+++ b/src/plugins/file-highlight/prism-file-highlight.ts
@@ -1,6 +1,7 @@
 import { setLanguage } from '../../shared/dom-util';
 import type { Prism } from '../../core';
 import type { PluginProto } from '../../types';
+import type { Autoloader } from '../autoloader/prism-autoloader';
 
 const FAILURE_MESSAGE = (status: number, message: string) => {
 	return `✖ Error ${status} while fetching file: ${message}`;
@@ -145,7 +146,7 @@ export default {
 				setLanguage(pre, language);
 
 				// preload the language
-				const autoloader = Prism.plugins.autoloader;
+				const autoloader = Prism.plugins.autoloader as Autoloader;
 				if (autoloader) {
 					autoloader.preloadLanguages(language);
 				}
diff --git a/src/plugins/filter-highlight-all/README.md b/src/plugins/filter-highlight-all/README.md
new file mode 100644
index 0000000000..1bc89e8f91
--- /dev/null
+++ b/src/plugins/filter-highlight-all/README.md
@@ -0,0 +1,107 @@
+---
+title: Filter highlightAll
+description: Filters the elements the `highlightAll` and `highlightAllUnder` methods actually highlight.
+owner: RunDevelopment
+noCSS: true
+resources:
+  - https://dev.prismjs.com/components/prism-typescript.js
+  - ./demo.js { defer }
+---
+
+
+
+

+
+# How to use
+
+Filter highlightAll provides you with ways to filter the element the `highlightAll` and `highlightAllUnder` methods actually highlight. This can be very useful when you use Prism's automatic highlighting when loading the page but want to exclude certain code blocks.
+
+

+
+

+
+# API
+
+In `Prism.plugins.filterHighlightAll` you can find the following:
+
+`add(condition: (value: { element, language: string }) => boolean): void`
+
+: Adds a new filter which will only allow an element to be highlighted if the given function returns `true` for that element.  
+This can be used to define a custom language filter.
+
+`addSelector(selector: string): void`
+
+: Adds a new filter which will only allow an element to be highlighted if the element matches the given CSS selector.
+
+`reject.add(condition: (value: { element, language: string }) => boolean): void`
+
+: Same as `add`, but only elements which do **not** fulfill the condition will be highlighted.
+
+`reject.addSelector(selector: string): void`
+
+: Same as `addSelector`, but only elements which do **not** match the selector will be highlighted.
+
+`filterKnown: boolean = false`
+
+: Set this to `true` to only allow known languages. Code blocks without a set language or an unknown language will not be highlighted.
+
+An element will only be highlighted by the `highlightAll` and `highlightAllUnder` methods if all of the above accept the element.
+
+## Attributes
+
+You can also add the following `data-*`{ .language-none } attributes to the script which contains the Filter highlightAll plugin.
+
+`
+
+
+
+```
+
+And later in your HTML:
+
+```html
+
+

+
+
+

+```
+
+Finally, unlike like the [File Highlight](../file-highlight) plugin, you _do_ need to supply the appropriate `class` with the language to highlight. This could have been auto-detected, but since you're not actually linking to a file it's not always possible (see below in the example using GitHub status). Furthermore, if you're linking to files with a `.xaml` extension for example, this plugin then needs to somehow map that to highlight as `markup`, which just means more bloat. You know what you're trying to highlight, just say so. 🙂
+
+## Caveat for Gists
+
+There's a bit of a catch with gists, as they can actually contain multiple files. There are two options to handle this:
+
+1. If your gist only contains one file, you don't need to to anything; the one and only file will automatically be chosen and highlighted
+2. If your file contains multiple files, the first one will be chosen by default. However, you can supply the filename in the `data-filename` attribute, and this file will be highlighted instead:
+
+```html
+

+```
+
+

+
+

+
+# Examples
+
+The plugin’s JS code (from GitHub):
+
+

+
+GitHub Gist (gist contains a single file, automatically selected):
+
+

+
+GitHub Gist (gist contains a multiple files, file to load specified):
+
+

+
+Bitbucket API:
+
+

+
+Custom adapter (JSON.stringify showing the GitHub REST API for [Prism's repository](https://api.github.com/repos/PrismJS/prism)):
+
+

+
+Registered adapter (as above, but without explicitly declaring the `data-adapter` attribute):
+
+

+
+

diff --git a/src/plugins/jsonp-highlight/demo.js b/src/plugins/jsonp-highlight/demo.js
new file mode 100644
index 0000000000..6a23863938
--- /dev/null
+++ b/src/plugins/jsonp-highlight/demo.js
@@ -0,0 +1,7 @@
+function dump_json (x) {
+	return `using dump_json: ${JSON.stringify(x, null, 2)}`;
+}
+
+Prism.plugins.jsonphighlight.registerAdapter(
+	x => `using registerAdapter: ${JSON.stringify(x, null, 2)}`
+);
diff --git a/src/plugins/jsonp-highlight/prism-jsonp-highlight.ts b/src/plugins/jsonp-highlight/prism-jsonp-highlight.ts
index 533880100b..4712cc949c 100644
--- a/src/plugins/jsonp-highlight/prism-jsonp-highlight.ts
+++ b/src/plugins/jsonp-highlight/prism-jsonp-highlight.ts
@@ -1,5 +1,6 @@
 import type { Prism } from '../../core';
 import type { PluginProto } from '../../types';
+import type { Autoloader } from '../autoloader/prism-autoloader';
 
 function getGlobal(): Record {
 	return typeof window === 'object' ? window as never : {};
@@ -241,7 +242,7 @@ export default {
 		return config;
 	},
 	effect(Prism) {
-		const config = Prism.plugins.jsonpHighlight;
+		const config = Prism.plugins.jsonpHighlight as JsonpHighlight;
 
 
 		const LOADING_MESSAGE = 'Loading…';
@@ -282,7 +283,7 @@ export default {
 				code.className = 'language-' + language;
 
 				// preload the language
-				const autoloader = Prism.plugins.autoloader;
+				const autoloader = Prism.plugins.autoloader as Autoloader;
 				if (autoloader) {
 					autoloader.preloadLanguages(language);
 				}
diff --git a/src/plugins/keep-markup/README.md b/src/plugins/keep-markup/README.md
new file mode 100644
index 0000000000..94be40d6bb
--- /dev/null
+++ b/src/plugins/keep-markup/README.md
@@ -0,0 +1,67 @@
+---
+title: Keep Markup
+description: Prevents custom markup from being dropped out during highlighting.
+owner: Golmote
+optional: normalize-whitespace
+noCSS: true
+---
+
+
+
+

+
+# How to use
+
+You have nothing to do. The plugin is active by default. With this plugin loaded, all markup inside code will be kept.
+
+However, you can deactivate the plugin for certain code element by adding the `no-keep-markup` class to it. You can also deactivate the plugin for the whole page by adding the `no-keep-markup` class to the body of the page and then selectively activate it again by adding the `keep-markup` class to code elements.
+
+## Double highlighting
+
+Some plugins (e.g. [Autoloader](../autoloader)) need to re-highlight code blocks. This is a problem for Keep Markup because it will keep the markup of the first highlighting pass resulting in a lot of unnecessary DOM nodes and causing problems for themes and other plugins.
+
+This problem can be fixed by adding a `drop-tokens` class to a code block or any of its ancestors. If `drop-tokens` is present, Keep Markup will ignore all `span.token`{ .language-css } elements created by Prism.
+
+

+
+

+
+# Examples
+
+The following source code
+
+```html
+

+@media screen {
+	div {
+		text-decoration: underline;
+		background: url('foo.png');
+	}
+}

+```
+
+would render like this:
+
+

+@media screen {
+	div {
+		text-decoration: underline;
+		background: url('foo.png');
+	}
+}

+
+

+	It also works for inline code:
+	var bar = function () { /* foo */ };
+

+
+

diff --git a/src/plugins/keep-markup/prism-keep-markup.ts b/src/plugins/keep-markup/prism-keep-markup.ts
index 955e25e291..0ccc5e4d55 100644
--- a/src/plugins/keep-markup/prism-keep-markup.ts
+++ b/src/plugins/keep-markup/prism-keep-markup.ts
@@ -13,7 +13,6 @@ interface NodeData {
 	posOpen: number;
 	posClose: number;
 }
-const markupData: StateKey = 'keep-markup data';
 
 export default {
 	id: 'keep-markup',
@@ -79,7 +78,7 @@ export default {
 				}
 			},
 			'after-highlight': (env) => {
-				const data = env.markupData ?? [];
+				const data: NodeData[] = env.markupData ?? [];
 				if (data.length) {
 					type End = [node: Text, pos: number]
 
diff --git a/src/plugins/line-highlight/README.md b/src/plugins/line-highlight/README.md
new file mode 100644
index 0000000000..6a4074dc53
--- /dev/null
+++ b/src/plugins/line-highlight/README.md
@@ -0,0 +1,84 @@
+---
+title: Line Highlight
+description: Highlights specific lines and/or line ranges.
+owner: LeaVerou
+resources:
+  - ../line-numbers/prism-line-numbers.css
+  - ../line-numbers/prism-line-numbers.js { type="module" }
+---
+
+

+
+# How to use
+
+Obviously, this only works on code blocks (`
`) and not for inline code.
+
+You specify the lines to be highlighted through the `data-line` attribute on the `
` element, in the following simple format:
+
+- A single number refers to the line with that number
+- Ranges are denoted by two numbers, separated with a hyphen (-)
+- Multiple line numbers or ranges are separated by commas.
+- Whitespace is allowed anywhere and will be stripped off.
+
+Examples:
+
+5
+
+: The 5th line
+
+1-5
+
+: Lines 1 through 5
+
+1,4
+
+: Line 1 and line 4
+
+1-2, 5, 9-20
+
+: Lines 1 through 2, line 5, lines 9 through 20
+
+In case you want the line numbering to be offset by a certain number (for example, you want the 1st line to be number 41 instead of 1, which is an offset of 40), you can additionally use the `data-line-offset` attribute.
+
+You can also link to specific lines on any code snippet, by using the following as a url hash: `#{element-id}.{lines}` where `{element-id}` is the id of the `
` element and `{lines}` is one or more lines or line ranges that follow the format outlined above. For example, if there is an element with `id="play"` on the page, you can link to lines 5-6 by linking to [#play.5-6](#play.5-6)
+
+If line numbers are also enabled for a code block and the `
` element has an id, you can add the `linkable-line-numbers` class to the `
` element. This will make all line numbers clickable and when clicking any line number, it will change the hash of the current page to link to that specific line.
+
+

+
+

+
+# Examples
+
+## Line 2
+
+

+
+## Lines 15-25
+
+

+
+## Line 1 and lines 3-4 and line 42
+
+

+
+## Line 43, starting from line 41
+
+

+
+[Linking example](#play.50-55,60)
+
+## Compatible with [Line numbers](../line-numbers)
+
+

+
+Even with some extra content before the `code` element.
+
+
Some content

+
+
+## With linkable line numbers
+
+

+
+

diff --git a/src/plugins/line-highlight/prism-line-highlight.ts b/src/plugins/line-highlight/prism-line-highlight.ts
index 96648465a8..3c575c5e06 100644
--- a/src/plugins/line-highlight/prism-line-highlight.ts
+++ b/src/plugins/line-highlight/prism-line-highlight.ts
@@ -3,6 +3,7 @@ import { combineCallbacks } from '../../util/combine-callbacks';
 import { lazy, noop } from '../../shared/util';
 import type { Prism } from '../../core';
 import type { PluginProto } from '../../types';
+import type { LineNumbers } from '../line-numbers/prism-line-numbers';
 
 const LINE_NUMBERS_CLASS = 'line-numbers';
 const LINKABLE_LINE_NUMBERS_CLASS = 'linkable-line-numbers';
@@ -126,8 +127,9 @@ export class LineHighlight {
 
 			// if the line-numbers plugin is enabled, then there is no reason for this plugin to display the line numbers
 			if (hasLineNumbers && this.Prism.plugins.lineNumbers) {
-				const startNode = this.Prism.plugins.lineNumbers.getLine(pre, start);
-				const endNode = this.Prism.plugins.lineNumbers.getLine(pre, end);
+				const lineNumbers = this.Prism.plugins.lineNumbers as LineNumbers;
+				const startNode = lineNumbers.getLine(pre, start);
+				const endNode = lineNumbers.getLine(pre, end);
 
 				if (startNode) {
 					const top = `${startNode.offsetTop + codePreOffset}px`;
@@ -185,7 +187,8 @@ export class LineHighlight {
 			const start = parseInt(pre.getAttribute('data-start') || '1');
 
 			// iterate all line number spans
-			this.Prism.plugins.lineNumbers.getLines(pre)?.forEach((lineSpan, i) => {
+			const lineNumbers = this.Prism.plugins.lineNumbers as LineNumbers;
+			lineNumbers.getLines(pre)?.forEach((lineSpan, i) => {
 				const lineNumber = i + start;
 				lineSpan.onclick = () => {
 					const hash = `${id}.${lineNumber}`;
@@ -266,10 +269,11 @@ export default {
 				pre.setAttribute('data-line', '');
 			}
 
-			const mutateDom = Prism.plugins.lineHighlight.highlightLines(pre, range, 'temporary ');
+			const lineHighlight = Prism.plugins.lineHighlight as LineHighlight;
+			const mutateDom = lineHighlight.highlightLines(pre, range, 'temporary ');
 			mutateDom();
 
-			if (Prism.plugins.lineHighlight.scrollIntoView) {
+			if (lineHighlight.scrollIntoView) {
 				document.querySelector('.temporary.line-highlight')?.scrollIntoView();
 			}
 		};
@@ -277,7 +281,8 @@ export default {
 			$$('pre')
 				.filter(isActiveFor)
 				.map((pre) => {
-					return Prism.plugins.lineHighlight.highlightLines(pre);
+					const lineHighlight = Prism.plugins.lineHighlight as LineHighlight;
+					return lineHighlight.highlightLines(pre);
 				})
 				.forEach(callFunction);
 		};
@@ -326,7 +331,8 @@ export default {
 				clearTimeout(fakeTimer as never);
 			}
 
-			const mutateDom = Prism.plugins.lineHighlight.highlightLines(pre);
+			const lineHighlight = Prism.plugins.lineHighlight as LineHighlight;
+			const mutateDom = lineHighlight.highlightLines(pre);
 			mutateDom();
 			fakeTimer = setTimeout(applyHash, 1);
 		});
diff --git a/src/plugins/line-numbers/README.md b/src/plugins/line-numbers/README.md
new file mode 100644
index 0000000000..320f7d904d
--- /dev/null
+++ b/src/plugins/line-numbers/README.md
@@ -0,0 +1,76 @@
+---
+title: Line Numbers
+description: Line number at the beginning of code lines.
+owner: kuba-kubula
+---
+
+

+
+# How to use
+
+Obviously, this is supposed to work only for code blocks (`
`) and not for inline code.
+
+Add the `line-numbers` class to your desired `
` or any of its ancestors, and the Line Numbers plugin will take care of the rest. To give all code blocks line numbers, add the `line-numbers` class to the `` of the page. This is part of a general activation mechanism where adding the `line-numbers` (or `no-line-numbers`) class to any element will enable (or disable) the Line Numbers plugin for all code blocks in that element.  
+Example:
+
+```html
+ 
+
+	
+	
...

+	
+	
...

+
+	
 
+
+		
+		
...

+		
+		
...

+
+	

+
+```
+
+Optional: You can specify the `data-start` (Number) attribute on the `
` element. It will shift the line counter.
+
+Optional: To support multiline line numbers using soft wrap, apply the CSS `white-space: pre-line;` or `white-space: pre-wrap;` to your desired `
`.
+
+

+
+

+
+# Examples
+
+## JavaScript
+
+

+
+## CSS
+
+Please note that this `
` does not have the `line-numbers` class but its parent does.
+
+

+
+## HTML
+
+Please note the `data-start="-5"` in the code below.
+
+

+
+## Unknown languages
+
+```{ .language-none .line-numbers }
+This raw text
+is not highlighted
+but it still has
+line numbers
+```
+
+## Soft wrap support
+
+Please note the `style="white-space: pre-wrap;"` in the code below.
+
+

+
+

diff --git a/src/plugins/line-numbers/prism-line-numbers.ts b/src/plugins/line-numbers/prism-line-numbers.ts
index 3b76a98301..bc5d389f70 100644
--- a/src/plugins/line-numbers/prism-line-numbers.ts
+++ b/src/plugins/line-numbers/prism-line-numbers.ts
@@ -203,7 +203,8 @@ export default {
 
 		let lastWidth = NaN;
 		const listener = () => {
-			if (Prism.plugins.lineNumbers.assumeViewportIndependence && lastWidth === window.innerWidth) {
+			const lineNumbers = Prism.plugins.lineNumbers as LineNumbers;
+			if (lineNumbers.assumeViewportIndependence && lastWidth === window.innerWidth) {
 				return;
 			}
 			lastWidth = window.innerWidth;
diff --git a/src/plugins/match-braces/README.md b/src/plugins/match-braces/README.md
new file mode 100644
index 0000000000..98b0900593
--- /dev/null
+++ b/src/plugins/match-braces/README.md
@@ -0,0 +1,62 @@
+---
+title: Match braces
+description: Highlights matching braces.
+owner: RunDevelopment
+resources: ../autoloader/prism-autoloader.js { type="module" }
+---
+
+

+
+# How to use
+
+To enable this plugin add the `match-braces` class to a code block:
+
+```html
+
...

+```
+
+Just like `language-xxxx`, the `match-braces` class is inherited, so you can add the class to the `` to enable the plugin for the whole page.
+
+The plugin will highlight brace pairs when the cursor hovers over one of the braces. The highlighting effect will disappear as soon as the cursor leaves the brace pair.  
+The hover effect can be disabled by adding the `no-brace-hover` to the code block. This class can also be inherited.
+
+You can also click on a brace to select the brace pair. To deselect the pair, click anywhere within the code block or select another pair.  
+The selection effect can be disabled by adding the `no-brace-select` to the code block. This class can also be inherited.
+
+## Rainbow braces 🌈
+
+To enable rainbow braces, simply add the `rainbow-braces` class to a code block. This class can also get inherited.
+
+

+
+

+
+# Examples
+
+## JavaScript
+
+

+
+```js
+const func = (a, b) => {
+	return `${a}:${b}`;
+}
+```
+
+## Lisp
+
+```lisp
+(defun factorial (n)
+	(if (= n 0) 1
+		(* n (factorial (- n 1)))))
+```
+
+## Lisp with rainbow braces 🌈 but without hover
+
+```lisp { .rainbow-braces .no-brace-hover }
+(defun factorial (n)
+	(if (= n 0) 1
+		(* n (factorial (- n 1)))))
+```
+
+

diff --git a/src/plugins/match-braces/prism-match-braces.ts b/src/plugins/match-braces/prism-match-braces.ts
index 6c7b045859..88ce35e261 100644
--- a/src/plugins/match-braces/prism-match-braces.ts
+++ b/src/plugins/match-braces/prism-match-braces.ts
@@ -1,11 +1,12 @@
 import { getParentPre, isActive } from '../../shared/dom-util';
 import type { PluginProto } from '../../types';
+import type { CustomClass } from '../custom-class/prism-custom-class';
 
 export default {
 	id: 'match-braces',
 	effect(Prism) {
 		function mapClassName(name: string) {
-			const customClass = Prism.plugins.customClass;
+			const customClass = Prism.plugins.customClass as CustomClass;
 			if (customClass) {
 				return customClass.apply(name);
 			} else {
diff --git a/src/plugins/normalize-whitespace/README.md b/src/plugins/normalize-whitespace/README.md
new file mode 100644
index 0000000000..cfdfec4633
--- /dev/null
+++ b/src/plugins/normalize-whitespace/README.md
@@ -0,0 +1,172 @@
+---
+title: Normalize Whitespace
+description: Supports multiple operations to normalize whitespace in code blocks.
+owner: zeitgeist87
+optional: unescaped-markup
+noCSS: true
+body_classes: language-markup
+resources: ../keep-markup/prism-keep-markup.js { type="module" }
+---
+
+
+
+

+
+# How to use
+
+Obviously, this is supposed to work only for code blocks (`
`) and not for inline code.
+
+By default the plugin trims all leading and trailing whitespace of every code block. It also removes extra indents and trailing whitespace on every line.
+
+The plugin can be disabled for a particular code block by adding the class `no-whitespace-normalization` to either the `
` or `` tag.
+
+The default settings can be overridden with the `setDefaults()`{ .language-javascript } method like so:
+
+```js
+Prism.plugins.NormalizeWhitespace.setDefaults({
+	"remove-trailing": true,
+	"remove-indent": true,
+	"left-trim": true,
+	"right-trim": true,
+	/*"break-lines": 80,
+	"indent": 2,
+	"remove-initial-line-feed": false,
+	"tabs-to-spaces": 4,
+	"spaces-to-tabs": 4*/
+});
+```
+
+The following settings are available and can be set via the `data-[setting]` attribute on the `
` element:
+
+`remove-trailing`
+
+: Removes trailing whitespace on all lines.
+
+`remove-indent`
+
+: If the whole code block is indented too much it removes the extra indent.
+
+`left-trim`
+
+: Removes all whitespace from the top of the code block.
+
+`right-trim`
+
+: Removes all whitespace from the bottom of the code block.
+
+`break-lines`
+
+: Simple way of breaking long lines at a certain length (default is 80 characters).
+
+`indent`
+
+: Adds a certain number of tabs to every line.
+
+`remove-initial-line-feed`
+
+: Less aggressive version of left-trim. It only removes a single line feed from the top of the code block.
+
+`tabs-to-spaces`
+
+: Converts all tabs to a certain number of spaces (default is 4 spaces).
+
+`spaces-to-tabs`
+
+: Converts a certain number of spaces to a tab (default is 4 spaces).
+
+

+
+

+
+# Examples
+
+The following example demonstrates the use of this plugin:
+
+

+
+The result looks like this:
+
+
+
+	
+
+
+		let example = {
+			foo: true,
+
+			bar: false
+		};
+
+
+		let
+		there_is_a_very_very_very_very_long_line_it_can_break_it_for_you
+		 = true;
+		
+		if 
+		(there_is_a_very_very_very_very_long_line_it_can_break_it_for_you
+		 === true) {
+		};
+
+
+	
+
+

+
+It is also compatible with the [keep-markup](../keep-markup) plugin:
+
+

+
+
+@media screen {
+	div {
+		text-decoration: underline;
+		background: url('foo.png');
+	}
+}

+
+This plugin can also be used on the server or on the command line with Node.js:
+
+```js
+let Prism = require("prismjs");
+let Normalizer = require("prismjs/plugins/normalize-whitespace/prism-normalize-whitespace");
+// Create a new Normalizer object
+let nw = new Normalizer({
+	"remove-trailing": true,
+	"remove-indent": true,
+	"left-trim": true,
+	"right-trim": true,
+	/*"break-lines": 80,
+	"indent": 2,
+	"remove-initial-line-feed": false,
+	"tabs-to-spaces": 4,
+	"spaces-to-tabs": 4*/
+});
+
+// ..or use the default object from Prism
+nw = Prism.plugins.NormalizeWhitespace;
+
+// The code snippet you want to highlight, as a string
+let code = "\t\t\tlet data = 1;    ";
+
+// Removes leading and trailing whitespace
+// and then indents by 1 tab
+code = nw.normalize(code, {
+	// Extra settings
+	indent: 1
+});
+
+// Returns a highlighted HTML string
+let html = Prism.highlight(code, Prism.languages.javascript);
+```
+
+

diff --git a/src/plugins/normalize-whitespace/demo.md b/src/plugins/normalize-whitespace/demo.md
new file mode 100644
index 0000000000..c14c6aee2d
--- /dev/null
+++ b/src/plugins/normalize-whitespace/demo.md
@@ -0,0 +1,53 @@
+---
+layout: null
+eleventyExcludeFromCollections: true
+---
+
+
+

+
+
+
+	
+
+
+		let example = {
+			foo: true,
+
+			bar: false
+		};
+
+
+	
+
+

+
+
+
+	
+
+
+		let there_is_a_very_very_very_very_long_line_it_can_break_it_for_you = true;
+
+		if (there_is_a_very_very_very_very_long_line_it_can_break_it_for_you === true) {
+		};
+
+
+	
+
+

+
+

+
+
+
+
+
diff --git a/src/plugins/previewers/README.md b/src/plugins/previewers/README.md
new file mode 100644
index 0000000000..2cbd1cc9a5
--- /dev/null
+++ b/src/plugins/previewers/README.md
@@ -0,0 +1,221 @@
+---
+title: Previewers
+description: Previewers for angles, colors, gradients, easing and time.
+owner: Golmote
+require: css-extras
+resources:
+  - https://dev.prismjs.com/components/prism-css-extras.js
+  - https://dev.prismjs.com/components/prism-less.js
+  - https://dev.prismjs.com/components/prism-sass.js
+  - https://dev.prismjs.com/components/prism-scss.js
+  - https://dev.prismjs.com/components/prism-stylus.js
+---
+
+

+
+# How to use
+
+You don't need to do anything. With this plugin loaded, a previewer will appear on hovering some values in code blocks. The following previewers are supported:
+
+- `angle` for angles
+- `color` for colors
+- `gradient` for gradients
+- `easing` for easing functions
+- `time` for durations
+
+This plugin is compatible with CSS, Less, Markup attributes, Sass, Scss and Stylus.
+
+

+
+

+
+# Examples
+
+## CSS
+
+```css
+.example-gradient {
+	background: -webkit-linear-gradient(left,     #cb60b3 0%, #c146a1 50%, #a80077 51%, #db36a4 100%); /* Chrome10+, Safari5.1+ */
+	background:    -moz-linear-gradient(left,     #cb60b3 0%, #c146a1 50%, #a80077 51%, #db36a4 100%); /* FF3.6+ */
+	background:     -ms-linear-gradient(left,     #cb60b3 0%, #c146a1 50%, #a80077 51%, #db36a4 100%); /* IE10+ */
+	background:      -o-linear-gradient(left,     #cb60b3 0%, #c146a1 50%, #a80077 51%, #db36a4 100%); /* Opera 11.10+ */
+	background:         linear-gradient(to right, #cb60b3 0%, #c146a1 50%, #a80077 51%, #db36a4 100%); /* W3C */
+}
+.example-angle {
+	transform: rotate(10deg);
+}
+.example-color {
+	color: rgba(255, 0, 0, 0.2);
+	background: purple;
+	border: 1px solid hsl(100, 70%, 40%);
+}
+.example-easing {
+	transition-timing-function: linear;
+}
+.example-time {
+	transition-duration: 3s;
+}
+```
+
+## Markup attributes
+
+```html
+
+
+```
+
+## Less
+
+```less
+@gradient: linear-gradient(135deg, #9dd53a 0%, #a1d54f 50%, #80c217 51%, #7cbc0a 100%);
+.example-gradient {
+	background: -webkit-linear-gradient(-45deg, #9dd53a 0%, #a1d54f 50%, #80c217 51%, #7cbc0a 100%); /* Chrome10+, Safari5.1+ */
+	background:    -moz-linear-gradient(-45deg, #9dd53a 0%, #a1d54f 50%, #80c217 51%, #7cbc0a 100%); /* FF3.6+ */
+	background:     -ms-linear-gradient(-45deg, #9dd53a 0%, #a1d54f 50%, #80c217 51%, #7cbc0a 100%); /* IE10+ */
+	background:      -o-linear-gradient(-45deg, #9dd53a 0%, #a1d54f 50%, #80c217 51%, #7cbc0a 100%); /* Opera 11.10+ */
+	background:         linear-gradient(135deg, #9dd53a 0%, #a1d54f 50%, #80c217 51%, #7cbc0a 100%); /* W3C */
+}
+@angle: 3rad;
+.example-angle {
+	transform: rotate(.4turn)
+}
+@nice-blue: #5B83AD;
+.example-color {
+	color: hsla(102, 53%, 42%, 0.4);
+}
+@easing: cubic-bezier(0.1, 0.3, 1, .4);
+.example-easing {
+	transition-timing-function: ease;
+}
+@time: 1s;
+.example-time {
+	transition-duration: 2s;
+}
+```
+
+## Sass
+
+```sass
+$gradient: linear-gradient(135deg, #9dd53a 0%, #a1d54f 50%, #80c217 51%, #7cbc0a 100%)
+@mixin example-gradient
+	background: -moz-radial-gradient(center, ellipse cover, #f2f6f8 0%, #d8e1e7 50%, #b5c6d0 51%, #e0eff9 100%)
+	background: radial-gradient(ellipse at center, #f2f6f8 0%, #d8e1e7 50%, #b5c6d0 51%, #e0eff9 100%)
+$angle: 380grad
+@mixin example-angle
+	transform: rotate(-120deg)
+.example-angle
+	transform: rotate(18rad)
+$color: blue
+@mixin example-color
+	color: rgba(147, 32, 34, 0.8)
+.example-color
+	color: pink
+$easing: ease-out
+.example-easing
+	transition-timing-function: ease-in-out
+$time: 3s
+@mixin example-time
+	transition-duration: 800ms
+.example-time
+	transition-duration: 0.8s
+```
+
+## Scss
+
+```scss
+$gradient: linear-gradient(135deg, #9dd53a 0%, #a1d54f 50%, #80c217 51%, #7cbc0a 100%);
+$attr: background;
+.example-gradient {
+	#{$attr}-image: repeating-linear-gradient(10deg, rgba(255, 0, 0, 0), rgba(255, 0, 0, 1) 10px, rgba(255, 0, 0, 0) 20px);
+}
+$angle: 1.8turn;
+.example-angle {
+	transform: rotate(-3rad)
+}
+$color: blue;
+.example-color {
+	#{$attr}-color: rgba(255, 255, 0, 0.75);
+}
+$easing: linear;
+.example-easing {
+	transition-timing-function: cubic-bezier(0.9, 0.1, .2, .4);
+}
+$time: 1s;
+.example-time {
+	transition-duration: 10s
+}
+```
+
+## Stylus
+
+```stylus
+gradient = linear-gradient(135deg, #9dd53a 0%, #a1d54f 50%, #80c217 51%, #7cbc0a 100%)
+.example-gradient
+	background-image: repeating-radial-gradient(circle, rgba(255, 0, 0, 0), rgba(255, 0, 0, 1) 10px, rgba(255, 0, 0, 0) 20px)
+angle = 357deg
+.example-angle
+	transform: rotate(100grad)
+color = olive
+.example-color
+	color: #000
+easing = ease-in
+.example-easing
+	transition-timing-function: ease-out
+time = 3s
+.example-time
+	transition-duration: 0.5s
+```
+
+
+
+

+
+# Disabling a previewer
+
+All previewers are enabled by default. To enable only a subset of them, a `data-previewers` attribute can be added on a code block or any ancestor. Its value should be a space-separated list of previewers representing the subset.
+
+For example:
+
+```html
+
div {
+	/* Only the previewer for color and time are enabled */
+	color: red;
+	transition-duration: 1s;
+	/* The previewer for angles is not enabled. */
+	transform: rotate(10deg);
+}

+```
+
+will give the following result:
+
+```css { data-previewers="color time" }
+div {
+	/* Only the previewers for color and time are enabled */
+	color: red;
+	transition-duration: 1s;
+	/* The previewer for angles is not enabled. */
+	transform: rotate(10deg);
+}
+```
+
+

+
+

+
+# API
+
+This plugins provides a constructor that can be accessed through `Prism.plugins.Previewer`.
+
+Once a previewer has been instantiated, an HTML element is appended to the document body. This element will appear when specific tokens are hovered.
+
+## `new Prism.plugins.Previewer(type, updater, supportedLanguages)`
+
+- `type`: the token type this previewer is associated to. The previewer will be shown when hovering tokens of this type.
+
+- `updater`: the function that will be called each time an associated token is hovered. This function takes the text content of the token as its only parameter. The previewer HTML element can be accessed through the keyword `this`. This function must return `true` for the previewer to be shown.
+
+- `supportedLanguages`: an optional array of supported languages. The previewer will be available only for those. Defaults to `'*'`, which means every languages.
+
+- `initializer`: an optional function. This function will be called when the previewer is initialized, right after the HTML element has been appended to the document body.
+
+

diff --git a/src/plugins/previewers/prism-previewers.ts b/src/plugins/previewers/prism-previewers.ts
index 2a05e48da5..f944157fed 100644
--- a/src/plugins/previewers/prism-previewers.ts
+++ b/src/plugins/previewers/prism-previewers.ts
@@ -717,7 +717,8 @@ export default {
 		*/
 
 		return Prism.hooks.add('after-highlight', (env) => {
-			Prism.plugins.previewers.initEvents(env.element, env.language);
+			const previewers = Prism.plugins.previewers as PreviewerCollection;
+			previewers.initEvents(env.element, env.language);
 		});
 	}
 } as PluginProto<'previewers'>;
diff --git a/src/plugins/show-invisibles/README.md b/src/plugins/show-invisibles/README.md
new file mode 100644
index 0000000000..c9c86ae921
--- /dev/null
+++ b/src/plugins/show-invisibles/README.md
@@ -0,0 +1,20 @@
+---
+title: Show Invisibles
+description: Show hidden characters such as tabs and line breaks.
+owner: LeaVerou
+optional:
+  - autolinker
+  - data-uri-highlight
+---
+
+

+
+# Examples
+
+

+
+

+
+

+
+

diff --git a/src/plugins/show-language/README.md b/src/plugins/show-language/README.md
new file mode 100644
index 0000000000..62604c1aa6
--- /dev/null
+++ b/src/plugins/show-language/README.md
@@ -0,0 +1,40 @@
+---
+title: Show Language
+description: Display the highlighted language in code blocks (inline code does not show the label).
+owner: nauzilus
+require: toolbar
+noCSS: true
+resources:
+  - ../toolbar/prism-toolbar.css
+  - ../toolbar/prism-toolbar.js { type="module" }
+---
+
+

+
+# Examples
+
+## JavaScript
+
+

+
+## CSS
+
+

+
+## HTML (Markup)
+
+

+
+## SVG
+
+The `data-language`{ .language-markup } attribute can be used to display a specific label whether it has been defined as a language or not.
+
+

+
+## Plain text
+
+```none
+Just some text (aka. not code).
+```
+
+

diff --git a/src/plugins/show-language/prism-show-language.ts b/src/plugins/show-language/prism-show-language.ts
index daeef836df..34d90ac99f 100644
--- a/src/plugins/show-language/prism-show-language.ts
+++ b/src/plugins/show-language/prism-show-language.ts
@@ -2,13 +2,13 @@ import { getParentPre } from '../../shared/dom-util';
 import { getTitle } from '../../shared/meta/title-data';
 import toolbar from '../toolbar/prism-toolbar';
 import type { PluginProto } from '../../types';
+import type { Toolbar } from '../toolbar/prism-toolbar';
 
 export default {
 	id: 'show-language',
 	require: toolbar,
 	effect(Prism) {
-		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-		const toolbar = Prism.plugins.toolbar!;
+		const toolbar = Prism.plugins.toolbar as Toolbar;
 
 		return toolbar.registerButton('show-language', (env) => {
 			const pre = getParentPre(env.element);
diff --git a/src/plugins/toolbar/README.md b/src/plugins/toolbar/README.md
new file mode 100644
index 0000000000..9e8d684710
--- /dev/null
+++ b/src/plugins/toolbar/README.md
@@ -0,0 +1,99 @@
+---
+title: Toolbar
+description: Attach a toolbar for plugins to easily register buttons on the top of a code block.
+owner: mAAdhaTTah
+body_classes: language-markup
+resources: ./demo.js { defer }
+---
+
+

+
+# How to use
+
+The Toolbar plugin allows for several methods to register your button, using the `Prism.plugins.toolbar.registerButton` function.
+
+The simplest method is through the HTML API. Add a `data-label` attribute to the `pre` element, and the Toolbar plugin will read the value of that attribute and append a label to the code snippet.
+
+```html { data-label="Hello World!" }
+

+```
+
+If you want to provide arbitrary HTML to the label, create a `template` element with the HTML you want in the label, and provide the `template` element's `id` to `data-label`. The Toolbar plugin will use the template's content for the button. You can also use to declare your event handlers inline:
+
+```html { data-label="my-label-button" }
+

+```
+
+```html
+
+```
+
+## Registering buttons
+
+For more flexibility, the Toolbar exposes a JavaScript function that can be used to register new buttons or labels to the Toolbar, `Prism.plugins.toolbar.registerButton`.
+
+The function accepts a key for the button and an object with a `text` property string and an optional `onClick` function or a `url` string. The `onClick` function will be called when the button is clicked, while the `url` property will be set to the anchor tag's `href`.
+
+```js
+Prism.plugins.toolbar.registerButton("hello-world", {
+	text: "Hello World!", // required
+	onClick: function (env) {
+		// optional
+		alert(`This code snippet is written in ${env.language}.`);
+	},
+});
+```
+
+See how the above code registers the `Hello World!` button? You can use this in your plugins to register your own buttons with the toolbar.
+
+If you need more control, you can provide a function to `registerButton` that returns either a `span`, `a`, or `button` element.
+
+```js
+Prism.plugins.toolbar.registerButton("select-code", env => {
+	let button = document.createElement("button");
+	button.innerHTML = "Select Code";
+
+	button.addEventListener("click", () => {
+		// Source: http://stackoverflow.com/a/11128179/2757940
+		if (document.body.createTextRange) {
+			// ms
+			let range = document.body.createTextRange();
+			range.moveToElementText(env.element);
+			range.select();
+		}
+		else if (window.getSelection) {
+			// moz, opera, webkit
+			let selection = window.getSelection();
+			let range = document.createRange();
+			range.selectNodeContents(env.element);
+			selection.removeAllRanges();
+			selection.addRange(range);
+		}
+	});
+
+	return button;
+});
+```
+
+The above function creates the Select Code button you see, and when you click it, the code gets highlighted.
+
+## Ordering buttons
+
+By default, the buttons will be added to the code snippet in the order they were registered. If more control over the order is needed, the `data-toolbar-order` attribute can be used. Given a comma-separated list of button names, it will ensure that these buttons will be displayed in the given order.  
+Buttons not listed will not be displayed. This means that buttons can be disabled using this technique.
+
+Example: The "Hello World!" button will appear before the "Select Code" button and the custom label button will not be displayed.
+
+```html { data-toolbar-order="hello-world,select-code" data-label="Hello World!" }
+

+```
+
+The `data-toolbar-order` attribute is inherited, so you can define the button order for the whole document by adding the attribute to the `body` of the page.
+
+```html
+
+```
+
+

+
+
diff --git a/src/plugins/toolbar/demo.js b/src/plugins/toolbar/demo.js
new file mode 100644
index 0000000000..bfbdb5d178
--- /dev/null
+++ b/src/plugins/toolbar/demo.js
@@ -0,0 +1,32 @@
+Prism.plugins.toolbar.registerButton("hello-world", {
+	text: "Hello World!", // required
+	onClick: function (env) {
+		// optional
+		alert(`This code snippet is written in ${env.language}.`);
+	},
+});
+
+Prism.plugins.toolbar.registerButton("select-code", env => {
+	let button = document.createElement("button");
+	button.innerHTML = "Select Code";
+
+	button.addEventListener("click", () => {
+		// Source: http://stackoverflow.com/a/11128179/2757940
+		if (document.body.createTextRange) {
+			// ms
+			let range = document.body.createTextRange();
+			range.moveToElementText(env.element);
+			range.select();
+		}
+		else if (window.getSelection) {
+			// moz, opera, webkit
+			let selection = window.getSelection();
+			let range = document.createRange();
+			range.selectNodeContents(env.element);
+			selection.removeAllRanges();
+			selection.addRange(range);
+		}
+	});
+
+	return button;
+});
diff --git a/src/plugins/toolbar/prism-toolbar.ts b/src/plugins/toolbar/prism-toolbar.ts
index 94f098a460..f725c08028 100644
--- a/src/plugins/toolbar/prism-toolbar.ts
+++ b/src/plugins/toolbar/prism-toolbar.ts
@@ -1,6 +1,6 @@
 import { getParentPre } from '../../shared/dom-util';
 import { noop } from '../../shared/util';
-import type { CompleteEnv, HookCallback } from '../../core/classes/hooks';
+import type { HookCallback, HookEnv } from '../../core/classes/hooks';
 import type { PluginProto } from '../../types';
 
 /**
@@ -33,13 +33,13 @@ export interface ButtonOptions {
 	/**
 	 * The event listener for the `click` event of the created button.
 	 */
-	onClick?: (env: CompleteEnv) => void;
+	onClick?: (env: HookEnv) => void;
 	/**
 	 * The class attribute to include with element.
 	 */
 	className?: string;
 }
-export type ButtonFactory = (env: CompleteEnv) => Node | undefined;
+export type ButtonFactory = (env: HookEnv) => Node | undefined;
 
 export class Toolbar {
 	private callbacks: ButtonFactory[] = [];
@@ -103,7 +103,7 @@ export class Toolbar {
 	/**
 	 * @package
 	 */
-	hook: HookCallback<'complete'> = (env) => {
+	hook: HookCallback = (env) => {
 		// Check if inline or actual code block (credit to line-numbers plugin)
 		const pre = getParentPre(env.element);
 		if (!pre) {
@@ -201,6 +201,7 @@ export default {
 		return toolbar;
 	},
 	effect(Prism) {
-		return Prism.hooks.add('complete', Prism.plugins.toolbar.hook);
+		const toolbar = Prism.plugins.toolbar as Toolbar;
+		return Prism.hooks.add('complete', toolbar.hook);
 	}
 } as PluginProto<'toolbar'>;
diff --git a/src/plugins/treeview-icons/README.md b/src/plugins/treeview-icons/README.md
new file mode 100644
index 0000000000..dd14042925
--- /dev/null
+++ b/src/plugins/treeview-icons/README.md
@@ -0,0 +1,63 @@
+---
+title: Treeview
+description: A language with special styles to highlight file system tree structures.
+owner: Golmote
+---
+
+

+
+# How to use
+
+You may use `tree -F` to get a compatible text structure.
+
+```treeview
+root_folder/
+|-- a first folder/
+|   |-- holidays.mov
+|   |-- javascript-file.js
+|   `-- some_picture.jpg
+|-- documents/
+|   |-- spreadsheet.xls
+|   |-- manual.pdf
+|   |-- document.docx
+|   `-- presentation.ppt
+|       `-- test
+|-- empty_folder/
+|-- going deeper/
+|   |-- going deeper/
+|   |   `-- going deeper/
+|   |        `-- going deeper/
+|   |            `-- .secret_file
+|   |-- style.css
+|   `-- index.html
+|-- music and movies/
+|   |-- great-song.mp3
+|   |-- S01E02.new.episode.avi
+|   |-- S01E02.new.episode.nfo
+|   `-- track 1.cda
+|-- .gitignore
+|-- .htaccess
+|-- .npmignore
+|-- archive 1.zip
+|-- archive 2.tar.gz
+|-- logo.svg
+`-- README.md
+```
+
+You can also use the following box-drawing characters to represent the tree:
+
+```treeview
+root_folder/
+├── a first folder/
+|   ├── holidays.mov
+|   ├── javascript-file.js
+|   └── some_picture.jpg
+├── documents/
+|   ├── spreadsheet.xls
+|   ├── manual.pdf
+|   ├── document.docx
+|   └── presentation.ppt
+└── etc.
+```
+
+

diff --git a/src/plugins/unescaped-markup/README.md b/src/plugins/unescaped-markup/README.md
new file mode 100644
index 0000000000..4165d7d3b1
--- /dev/null
+++ b/src/plugins/unescaped-markup/README.md
@@ -0,0 +1,159 @@
+---
+title: Unescaped Markup
+description: Write markup without having to escape anything.
+owner: LeaVerou
+---
+
+

+
+# How to use
+
+This plugin provides several methods of achieving the same thing:
+
+- Instead of using `
` elements, use `
+```
+
+- Use an HTML-comment to escape your code:
+
+```html
+

+```
+
+This will only work if the `code` element contains exactly one comment and nothing else (not even spaces). E.g. ` ` and `text` will not work.
+
+

+
+

+
+# Examples
+
+View source to see that the following didn’t need escaping (except for </script>, that does):
+
+
+
+
The next example uses the HTML-comment method:

+
+

+
+

+
+

+
+# FAQ
+
+Why not use the HTML `