Introduce grammar patch from the simplify branch

src/core/registry.js

@@ -1,8 +1,8 @@
 import { kebabToCamelCase } from '../shared/util.js';
-import { cloneGrammar } from '../util/extend.js';
+import { extend } from '../util/extend.js';
+import { grammarPatch } from '../util/grammar-patch.js';
 import { forEach, toArray } from '../util/iterables.js';
-import { extend } from '../util/language-util.js';
-import { defineLazyProperty } from '../util/objects.js';
+import { deepClone, defineLazyProperty } from '../util/objects.js';
 
 /**
  * TODO: docs
@@ -221,7 +221,7 @@ export class Registry {
 
 		const base = entry?.proto.base;
 		// We need this so that any code modifying the base grammar doesn't affect other instances
-		const baseGrammar = base && cloneGrammar(required(base.id), base.id);
+		const baseGrammar = base && deepClone(required(base.id));
 
 		const requiredLanguages = toArray(
 			/** @type {LanguageProto | LanguageProto[] | undefined} */ (entry?.proto.require)
@@ -240,7 +240,7 @@ export class Registry {
 		else {
 			const options = {
 				getOptionalLanguage: id => this.getLanguage(id),
-				extend: (id, ref) => extend(required(id), id, ref),
+				extend: (id, ref) => extend(required(id), ref),
 				...(baseGrammar && { base: baseGrammar }),
 				...(requiredLanguages.length && { languages }),
 			};
@@ -249,10 +249,10 @@ export class Registry {
 		}
 
 		if (baseGrammar) {
-			evaluatedGrammar = extend(baseGrammar, base.id, evaluatedGrammar);
+			evaluatedGrammar = extend(baseGrammar, evaluatedGrammar);
 		}
 
-		return (entry.evaluatedGrammar = evaluatedGrammar);
+		return (entry.evaluatedGrammar = grammarPatch(evaluatedGrammar));
 	}
 }
 

src/core/tokenize/tokenize.js

@@ -42,13 +42,21 @@ export function tokenize (text, grammar) {
 	const tokenList = new LinkedList();
 	tokenList.addAfter(tokenList.head, text);
 
-	_matchGrammar.call(prism, text, tokenList, grammar, tokenList.head, 0);
+	_matchGrammar.call(
+		prism,
+		text,
+		tokenList,
+		/** @type {GrammarTokens} */ (grammar),
+		tokenList.head,
+		0
+	);
 
 	return tokenList.toArray();
 }
 
 /**
  * @typedef {import('../../types.d.ts').TokenStream} TokenStream
  * @typedef {import('../../types.d.ts').Grammar} Grammar
+ * @typedef {import('../../types.d.ts').GrammarTokens} GrammarTokens
  * @typedef {import('../prism.js').Prism} Prism
  */

src/languages/c.js

@@ -73,7 +73,7 @@ export default {
 		/* OpenCL host API */
 		const extensions = getOptionalLanguage('opencl-extensions');
 		if (extensions) {
-			insertBefore(base, 'keyword', extensions);
+			insertBefore(base, 'keyword', /** @type {GrammarTokens} */ (extensions));
 			delete base['type-opencl-host-cpp'];
 		}
 
@@ -104,4 +104,5 @@ export default {
 
 /**
  * @typedef {import('../types.d.ts').GrammarToken} GrammarToken
+ * @typedef {import('../types.d.ts').GrammarTokens} GrammarTokens
  */

src/languages/chaiscript.js

@@ -1,5 +1,4 @@
 import { toArray } from '../util/iterables.js';
-import { insertBefore } from '../util/language-util.js';
 import clike from './clike.js';
 import cpp from './cpp.js';
 
@@ -8,44 +7,7 @@ export default {
 	id: 'chaiscript',
 	base: clike,
 	require: cpp,
-	grammar ({ base, languages }) {
-		insertBefore(base, 'operator', {
-			'parameter-type': {
-				// e.g. def foo(int x, Vector y) {...}
-				pattern: /([,(]\s*)\w+(?=\s+\w)/,
-				lookbehind: true,
-				alias: 'class-name',
-			},
-		});
-
-		insertBefore(base, 'string', {
-			'string-interpolation': {
-				pattern:
-					/(^|[^\\])"(?:[^"$\\]|\\[\s\S]|\$(?!\{)|\$\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\})*"/,
-				lookbehind: true,
-				greedy: true,
-				inside: {
-					'interpolation': {
-						pattern:
-							/((?:^|[^\\])(?:\\{2})*)\$\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\}/,
-						lookbehind: true,
-						inside: {
-							'interpolation-expression': {
-								pattern: /(^\$\{)[\s\S]+(?=\}$)/,
-								lookbehind: true,
-								inside: 'chaiscript',
-							},
-							'interpolation-punctuation': {
-								pattern: /^\$\{|\}$/,
-								alias: 'punctuation',
-							},
-						},
-					},
-					'string': /[\s\S]+/,
-				},
-			},
-		});
-
+	grammar ({ languages }) {
 		return {
 			'string': {
 				pattern: /(^|[^\\])'(?:[^'\\]|\\[\s\S])*'/,
@@ -66,8 +28,50 @@ export default {
 			],
 			'keyword':
 				/\b(?:attr|auto|break|case|catch|class|continue|def|default|else|finally|for|fun|global|if|return|switch|this|try|var|while)\b/,
-			'number': [...toArray(languages.cpp.number), /\b(?:Infinity|NaN)\b/],
+			'number': [
+				...toArray(
+					/** @type {import('../types.d.ts').GrammarTokens} */ (languages.cpp).number
+				),
+				/\b(?:Infinity|NaN)\b/,
+			],
 			'operator': />>=?|<<=?|\|\||&&|:[:=]?|--|\+\+|[=!<>+\-*/%|&^]=?|[?~]|`[^`\r\n]{1,4}`/,
+			$insertBefore: {
+				'operator': {
+					'parameter-type': {
+						// e.g. def foo(int x, Vector y) {...}
+						pattern: /([,(]\s*)\w+(?=\s+\w)/,
+						lookbehind: true,
+						alias: 'class-name',
+					},
+				},
+				'string': {
+					'string-interpolation': {
+						pattern:
+							/(^|[^\\])"(?:[^"$\\]|\\[\s\S]|\$(?!\{)|\$\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\})*"/,
+						lookbehind: true,
+						greedy: true,
+						inside: {
+							'interpolation': {
+								pattern:
+									/((?:^|[^\\])(?:\\{2})*)\$\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\}/,
+								lookbehind: true,
+								inside: {
+									'interpolation-expression': {
+										pattern: /(^\$\{)[\s\S]+(?=\}$)/,
+										lookbehind: true,
+										inside: 'chaiscript',
+									},
+									'interpolation-punctuation': {
+										pattern: /^\$\{|\}$/,
+										alias: 'punctuation',
+									},
+								},
+							},
+							'string': /[\s\S]+/,
+						},
+					},
+				},
+			},
 		};
 	},
 };

src/languages/cpp.js

@@ -103,7 +103,11 @@ export default {
 		/* OpenCL host API */
 		const extensions = getOptionalLanguage('opencl-extensions');
 		if (extensions) {
-			insertBefore(cpp, 'keyword', extensions);
+			insertBefore(
+				cpp,
+				'keyword',
+				/** @type {import('../types.d.ts').GrammarTokens} */ (extensions)
+			);
 		}
 
 		const baseInside = { ...cpp };

src/languages/crystal.js

@@ -1,51 +1,11 @@
 import { toArray } from '../util/iterables.js';
-import { insertBefore } from '../util/language-util.js';
 import ruby from './ruby.js';
 
 /** @type {import('../types.d.ts').LanguageProto<'crystal'>} */
 export default {
 	id: 'crystal',
 	base: ruby,
 	grammar ({ base }) {
-		insertBefore(base, 'string-literal', {
-			'attribute': {
-				pattern: /@\[.*?\]/,
-				inside: {
-					'delimiter': {
-						pattern: /^@\[|\]$/,
-						alias: 'punctuation',
-					},
-					'attribute': {
-						pattern: /^(\s*)\w+/,
-						lookbehind: true,
-						alias: 'class-name',
-					},
-					'args': {
-						pattern: /\S(?:[\s\S]*\S)?/,
-						inside: 'crystal',
-					},
-				},
-			},
-			'expansion': {
-				pattern: /\{(?:\{.*?\}|%.*?%)\}/,
-				inside: {
-					'content': {
-						pattern: /^(\{.)[\s\S]+(?=.\}$)/,
-						lookbehind: true,
-						inside: 'crystal',
-					},
-					'delimiter': {
-						pattern: /^\{[\{%]|[\}%]\}$/,
-						alias: 'operator',
-					},
-				},
-			},
-			'char': {
-				pattern: /'(?:[^\\\r\n]{1,2}|\\(?:.|u(?:[A-Fa-f0-9]{1,4}|\{[A-Fa-f0-9]{1,6}\})))'/,
-				greedy: true,
-			},
-		});
-
 		return {
 			'keyword': [
 				/\b(?:__DIR__|__END_LINE__|__FILE__|__LINE__|abstract|alias|annotation|as|asm|begin|break|case|class|def|do|else|elsif|end|ensure|enum|extend|for|fun|if|ifdef|include|instance_sizeof|lib|macro|module|next|of|out|pointerof|private|protected|ptr|require|rescue|return|select|self|sizeof|struct|super|then|type|typeof|undef|uninitialized|union|unless|until|when|while|with|yield)\b/,
@@ -56,8 +16,52 @@ export default {
 			],
 			'number':
 				/\b(?:0b[01_]*[01]|0o[0-7_]*[0-7]|0x[\da-fA-F_]*[\da-fA-F]|(?:\d(?:[\d_]*\d)?)(?:\.[\d_]*\d)?(?:[eE][+-]?[\d_]*\d)?)(?:_(?:[uif](?:8|16|32|64))?)?\b/,
-			'operator': [/->/, ...toArray(base.operator)],
+			'operator': [
+				/->/,
+				...toArray(/** @type {import('../types.d.ts').GrammarTokens} */ (base).operator),
+			],
 			'punctuation': /[(){}[\].,;\\]/,
+			$insertBefore: {
+				'string-literal': {
+					'attribute': {
+						pattern: /@\[.*?\]/,
+						inside: {
+							'delimiter': {
+								pattern: /^@\[|\]$/,
+								alias: 'punctuation',
+							},
+							'attribute': {
+								pattern: /^(\s*)\w+/,
+								lookbehind: true,
+								alias: 'class-name',
+							},
+							'args': {
+								pattern: /\S(?:[\s\S]*\S)?/,
+								inside: 'crystal',
+							},
+						},
+					},
+					'expansion': {
+						pattern: /\{(?:\{.*?\}|%.*?%)\}/,
+						inside: {
+							'content': {
+								pattern: /^(\{.)[\s\S]+(?=.\}$)/,
+								lookbehind: true,
+								inside: 'crystal',
+							},
+							'delimiter': {
+								pattern: /^\{[\{%]|[\}%]\}$/,
+								alias: 'operator',
+							},
+						},
+					},
+					'char': {
+						pattern:
+							/'(?:[^\\\r\n]{1,2}|\\(?:.|u(?:[A-Fa-f0-9]{1,4}|\{[A-Fa-f0-9]{1,6}\})))'/,
+						greedy: true,
+					},
+				},
+			},
 		};
 	},
 };