Introduce grammar patch from the simplify branch

Authored by: Lea Verou <[email protected]>

Author: DmitrySharabin

src/core/registry.js

@@ -1,8 +1,8 @@
 import { kebabToCamelCase } from '../shared/util.js';
-import { cloneGrammar } from '../util/extend.js';
+import { extend } from '../util/extend.js';
+import { grammarPatch } from '../util/grammar-patch.js';
 import { forEach, toArray } from '../util/iterables.js';
-import { extend } from '../util/language-util.js';
-import { defineLazyProperty } from '../util/objects.js';
+import { deepClone, defineLazyProperty } from '../util/objects.js';
 
 /**
  * TODO: docs
@@ -221,7 +221,7 @@ export class Registry {
 
 		const base = entry?.proto.base;
 		// We need this so that any code modifying the base grammar doesn't affect other instances
-		const baseGrammar = base && cloneGrammar(required(base.id), base.id);
+		const baseGrammar = base && deepClone(required(base.id));
 
 		const requiredLanguages = toArray(
 			/** @type {LanguageProto | LanguageProto[] | undefined} */ (entry?.proto.require)
@@ -240,7 +240,7 @@ export class Registry {
 		else {
 			const options = {
 				getOptionalLanguage: id => this.getLanguage(id),
-				extend: (id, ref) => extend(required(id), id, ref),
+				extend: (id, ref) => extend(required(id), ref),
 				...(baseGrammar && { base: baseGrammar }),
 				...(requiredLanguages.length && { languages }),
 			};
@@ -249,10 +249,10 @@ export class Registry {
 		}
 
 		if (baseGrammar) {
-			evaluatedGrammar = extend(baseGrammar, base.id, evaluatedGrammar);
+			evaluatedGrammar = extend(baseGrammar, evaluatedGrammar);
 		}
 
-		return (entry.evaluatedGrammar = evaluatedGrammar);
+		return (entry.evaluatedGrammar = grammarPatch(evaluatedGrammar));
 	}
 }
 

src/core/tokenize/tokenize.js

@@ -42,13 +42,21 @@ export function tokenize (text, grammar) {
 	const tokenList = new LinkedList();
 	tokenList.addAfter(tokenList.head, text);
 
-	_matchGrammar.call(prism, text, tokenList, grammar, tokenList.head, 0);
+	_matchGrammar.call(
+		prism,
+		text,
+		tokenList,
+		/** @type {GrammarTokens} */ (grammar),
+		tokenList.head,
+		0
+	);
 
 	return tokenList.toArray();
 }
 
 /**
  * @typedef {import('../../types.d.ts').TokenStream} TokenStream
  * @typedef {import('../../types.d.ts').Grammar} Grammar
+ * @typedef {import('../../types.d.ts').GrammarTokens} GrammarTokens
  * @typedef {import('../prism.js').Prism} Prism
  */

src/types.d.ts

@@ -198,7 +198,35 @@ export type GrammarSpecial = {
 	$tokenize?: (code: string, grammar: Grammar, Prism: Prism) => TokenStream;
 };
 
-export type Grammar = GrammarTokens & GrammarSpecial;
+/**
+ * Tokens within $insert
+ */
+export type InsertableToken = (RegExpLike | GrammarToken | (RegExpLike | GrammarToken)[]) & {
+	$before?: TokenName | TokenName[];
+	$after?: TokenName | TokenName[];
+};
+
+/**
+ * A grammar that is defined as its delta from another grammar.
+ */
+export type GrammarPatch = {
+	$insert?: Partial<Record<TokenName, InsertableToken>>;
+	$insertBefore?: Partial<Record<TokenName, GrammarTokens>>;
+	$insertAfter?: Partial<Record<TokenName, GrammarTokens>>;
+	$delete?: TokenName[];
+	$merge?: Partial<
+		Record<TokenName, Partial<Omit<GrammarToken, 'pattern'>> & { pattern?: RegExpLike }>
+	>;
+};
+
+export interface Grammar extends GrammarSpecial, GrammarPatch {
+	[token: string]:
+		| RegExpLike
+		| GrammarToken
+		| (RegExpLike | GrammarToken)[]
+		| GrammarSpecial[keyof GrammarSpecial]
+		| GrammarPatch[keyof GrammarPatch];
+}
 
 export interface PlainObject {
 	[key: string]: unknown;

src/util/extend.js

@@ -1,3 +1,5 @@
+import { betterAssign, deepClone } from './objects.js';
+
 /**
  * Creates a deep copy of the language with the given id and appends the given tokens.
  *
@@ -13,9 +15,8 @@
  * Therefore, it is encouraged to order overwriting tokens according to the positions of the overwritten tokens.
  * Furthermore, all non-overwriting tokens should be placed after the overwriting ones.
  *
- * @param {Grammar} grammar The grammar of the language to extend.
- * @param {string} id The id of the language to extend.
- * @param {Grammar} reDef The new tokens to append.
+ * @param {Grammar} base The grammar of the language to extend.
+ * @param {Grammar} grammar The new tokens to append.
  * @returns {Grammar} The new language created.
  * @example
  * Prism.languages['css-with-colors'] = Prism.languages.extend('css', {
@@ -26,120 +27,67 @@
  *     'color': /\b(?:red|green|blue)\b/
  * });
  */
-export function extend (grammar, id, reDef) {
-	const lang = cloneGrammar(grammar, id);
+export function extend (base, grammar) {
+	const lang = deepClone(base);
+
+	for (const key in grammar) {
+		if (typeof key !== 'string' || key.startsWith('$')) {
+			// ignore special keys
+			continue;
+		}
 
-	for (const key in reDef) {
-		lang[key] = reDef[key];
+		lang[key] = grammar[key];
 	}
 
-	return lang;
-}
+	if (grammar.$insertBefore) {
+		lang.$insertBefore = betterAssign(lang.$insertBefore ?? {}, grammar.$insertBefore);
+	}
 
-/**
- * @param {Grammar} grammar
- * @param {string} id
- * @returns {Grammar}
- */
-export function cloneGrammar (grammar, id) {
-	/** @type {Grammar} */
-	const result = {};
+	if (grammar.$insertAfter) {
+		lang.$insertAfter = betterAssign(lang.$insertAfter ?? {}, grammar.$insertAfter);
+	}
 
-	/** @type {Map<Grammar, Grammar>} */
-	const visited = new Map();
+	if (grammar.$insert) {
+		// Syntactic sugar for $insertBefore/$insertAfter
+		for (const tokenName in grammar.$insert) {
+			const def = grammar.$insert[tokenName];
+			const { $before, $after, ...token } = def;
+			const relToken = $before || $after;
+			const all = $before ? '$insertBefore' : '$insertAfter';
+			lang[all] ??= {};
 
-	/**
-	 * @param {GrammarToken | RegExpLike} value
-	 */
-	function cloneToken (value) {
-		if (!value.pattern) {
-			return value;
-		}
-		else {
-			/** @type {GrammarToken} */
-			const copy = { pattern: value.pattern };
-			if (value.lookbehind) {
-				copy.lookbehind = value.lookbehind;
+			if (Array.isArray(relToken)) {
+				// Insert in multiple places
+				for (const t of relToken) {
+					lang[all][t][tokenName] = token;
+				}
 			}
-			if (value.greedy) {
-				copy.greedy = value.greedy;
+			else if (relToken) {
+				(lang[all][relToken] ??= {})[tokenName] = token;
 			}
-			if (value.alias) {
-				copy.alias = Array.isArray(value.alias) ? [...value.alias] : value.alias;
+			else {
+				lang[tokenName] = token;
 			}
-			if (value.inside) {
-				copy.inside = cloneRef(value.inside);
-			}
-			return copy;
 		}
 	}
 
-	/**
-	 * @param {GrammarTokens['string']} value
-	 */
-	function cloneTokens (value) {
-		if (!value) {
-			return undefined;
-		}
-		else if (Array.isArray(value)) {
-			return value.map(cloneToken);
+	if (grammar.$delete) {
+		if (lang.$delete) {
+			// base also had $delete
+			lang.$delete.push(...grammar.$delete);
 		}
 		else {
-			return cloneToken(value);
+			lang.$delete = [...grammar.$delete];
 		}
 	}
 
-	/**
-	 * @param {string | Grammar} ref
-	 */
-	function cloneRef (ref) {
-		if (ref === id) {
-			// self ref
-			return result;
-		}
-		else if (typeof ref === 'string') {
-			return ref;
-		}
-		else {
-			return clone(ref);
-		}
+	if (grammar.$merge) {
+		lang.$merge = betterAssign(lang.$merge ?? {}, grammar.$merge);
 	}
 
-	/**
-	 * @param {Grammar} value
-	 */
-	function clone (value) {
-		let mapped = visited.get(value);
-		if (mapped === undefined) {
-			mapped = value === grammar ? result : {};
-			visited.set(value, mapped);
-
-			// tokens
-			for (const [key, tokens] of Object.entries(value)) {
-				mapped[key] = cloneTokens(/** @type {GrammarToken[]} */ (tokens));
-			}
-
-			// rest
-			const r = value.$rest;
-			if (r != null) {
-				mapped.$rest = cloneRef(r);
-			}
-
-			// tokenize
-			const t = value.$tokenize;
-			if (t) {
-				mapped.$tokenize = t;
-			}
-		}
-		return mapped;
-	}
-
-	return clone(grammar);
+	return lang;
 }
 
 /**
  * @typedef {import('../types.d.ts').Grammar} Grammar
- * @typedef {import('../types.d.ts').GrammarToken} GrammarToken
- * @typedef {import('../types.d.ts').GrammarTokens} GrammarTokens
- * @typedef {import('../types.d.ts').RegExpLike} RegExpLike
  */

src/util/grammar-patch.js

@@ -0,0 +1,93 @@
+import { insertAfter, insertBefore } from './insert.js';
+import { deepMerge } from './objects.js';
+
+/**
+ * Apply a patch to a grammar to modify it.
+ * The patch and the grammar may be the same object.
+ *
+ * @param {Grammar} grammar
+ * @param {Grammar} [patch=grammar]
+ * @returns {Grammar}
+ */
+export function grammarPatch (grammar, patch = grammar) {
+	if (patch.$insertBefore) {
+		for (const key in patch.$insertBefore) {
+			const tokens = patch.$insertBefore[key];
+
+			if (key?.includes('/')) {
+				// Deep key
+				let path = key.split('/');
+				const lastKey = path.pop();
+				path = path.flatMap(key => [key, 'inside']); // add `inside` after each key
+				// @ts-ignore
+				const obj = path.reduce((acc, key) => acc?.[key], grammar);
+
+				if (obj) {
+					// @ts-ignore
+					insertBefore(obj, lastKey, tokens);
+				}
+			}
+			else if (tokens) {
+				// @ts-ignore
+				insertBefore(grammar, key, tokens);
+			}
+		}
+		delete grammar.$insertBefore;
+	}
+
+	if (patch.$insertAfter) {
+		for (const key in patch.$insertAfter) {
+			const tokens = patch.$insertAfter[key];
+
+			if (key?.includes('/')) {
+				// Deep key
+				let path = key.split('/');
+				const lastKey = path.pop();
+				path = path.flatMap(key => [key, 'inside']); // add `inside` after each key
+				// @ts-ignore
+				const obj = path.reduce((acc, key) => acc?.[key], grammar);
+
+				if (obj) {
+					// @ts-ignore
+					insertAfter(obj, lastKey, tokens);
+				}
+			}
+			else if (tokens) {
+				// @ts-ignore
+				insertAfter(grammar, key, tokens);
+			}
+		}
+		delete grammar.$insertAfter;
+	}
+
+	if (patch.$delete) {
+		// @ts-ignore
+		for (const key of patch.$delete) {
+			// TODO support deep keys
+			delete grammar[key];
+		}
+		delete grammar.$delete;
+	}
+
+	if (patch.$merge) {
+		for (const key in patch.$merge) {
+			const tokens = patch.$merge[key];
+
+			if (grammar[key]) {
+				deepMerge(grammar[key], tokens);
+			}
+			else {
+				grammar[key] = /** @type {GrammarTokens} */ (tokens);
+			}
+		}
+
+		delete grammar.$merge;
+	}
+
+	return grammar;
+}
+
+/**
+ * @typedef {import('../types.d.ts').Grammar} Grammar
+ * @typedef {import('../types.d.ts').GrammarTokens} GrammarTokens
+ */