[WIP] Simplify core

src/core/classes/token.ts

@@ -103,6 +103,8 @@ export class Token {
 	}
 }
 
+export default Token;
+
 /**
  * A token stream is an array of strings and {@link Token Token} objects.
  *

src/core/tokenize/match.ts

@@ -1,8 +1,8 @@
 import { Token } from '../classes/token';
 import singleton from '../prism';
 import { tokenize } from './tokenize';
-import { resolve } from './util';
-import type { GrammarToken, GrammarTokens, RegExpLike } from '../../types';
+import { resolve, tokenizeByNamedGroups } from './util';
+import type { GrammarToken, GrammarTokens, RegExpLike, TokenStream } from '../../types';
 import type {
 	LinkedList,
 	LinkedListHeadNode,
@@ -22,11 +22,7 @@
 ): void {
 	const prism = this ?? singleton;
 
-	let restGrammar = resolve(prism.languageRegistry, grammar.$rest);
-	while (restGrammar) {
-		grammar = { ...grammar, ...restGrammar };
-		restGrammar = resolve(prism.languageRegistry, restGrammar.$rest);
-	}
+	grammar = resolve.call(prism, grammar);
 
 	for (const token in grammar) {
 		const tokenValue = grammar[token];
@@ -46,11 +42,21 @@
 
 			const patternObj = toGrammarToken(patterns[j]);
 			let { pattern, lookbehind = false, greedy = false, alias, inside } = patternObj;
-			const insideGrammar = resolve(prism.languageRegistry, inside);
+			const insideGrammar = resolve.call(prism, inside);
+
+			let flagsToAdd = '';
 
 			if (greedy && !pattern.global) {
 				// Without the global flag, lastIndex won't work
-				patternObj.pattern = pattern = RegExp(pattern.source, pattern.flags + 'g');
+				flagsToAdd += 'g';
+			}
+			if (pattern.source.indexOf('(?<') && pattern.hasIndices === false) {
+				// Has named groups, we need to be able to capture their indices
+				flagsToAdd += 'd';
+			}
+
+			if (flagsToAdd) {
+				patternObj.pattern = pattern = RegExp(pattern.source, pattern.flags + flagsToAdd);
 			}
 
 			for (
@@ -130,6 +136,7 @@
 
 				const from = match.index;
 				const matchStr = match[0];
+				let content: TokenStream | string = matchStr;
 				const before = str.slice(0, from);
 				const after = str.slice(from + matchStr.length);
 
@@ -146,13 +153,37 @@
 				}
 
 				tokenList.removeRange(removeFrom, removeCount);
+				let byGroups = match.groups ? tokenizeByNamedGroups(match) : null;
+
+				if (byGroups?.length > 1) {
+					content = byGroups.map(arg => {
+						let content = typeof arg === 'string' ? arg : arg.content;
+						let type = typeof arg === 'string' ? undefined : arg.type;
+
+						if (insideGrammar) {
+							let localInsideGrammar = insideGrammar[type] ?? insideGrammar;
+
+							if (typeof localInsideGrammar === 'function') {
+								// Late resolving
+								localInsideGrammar = resolve.call(
+									prism,
+									localInsideGrammar(match.groups)
+								);
+							}
+
+							content = tokenize.call(prism, content, localInsideGrammar);
+						}
+
+						return typeof arg === 'object' && arg.type
+							? new Token(arg.type, content)
+							: content;
+					});
+				}
+				else if (insideGrammar) {
+					content = tokenize.call(prism, content, insideGrammar);
+				}
 
-				const wrapped = new Token(
-					token,
-					insideGrammar ? tokenize.call(prism, matchStr, insideGrammar) : matchStr,
-					alias,
-					matchStr
-				);
+				const wrapped = new Token(token, content, alias, matchStr);
 				currentNode = tokenList.addAfter(removeFrom, wrapped);
 
 				if (after) {

src/core/tokenize/util.ts

@@ -1,13 +1,61 @@
-import type { Grammar } from '../../types';
-import type LanguageRegistry from '../classes/language-registry';
+import { camelToKebabCase } from '../../shared/util';
+import singleton from '../prism';
+import type { Grammar, Prism } from '../../types';
 
 export function resolve (
-	languageRegistry: LanguageRegistry,
-	reference: Grammar | string | null | undefined
-): Grammar | undefined {
-	if (typeof reference === 'string') {
-		return languageRegistry.getLanguage(reference)?.grammar;
+	this: Prism,
+	reference: Grammar | string | null | undefined | Function
+): Grammar | undefined | Function {
+	let prism = this ?? singleton;
+	let ret = reference ?? undefined;
+
+	if (typeof ret === 'string') {
+		ret = prism.languageRegistry.getLanguage(ret)?.grammar;
+	}
+
+	if (typeof ret === 'function' && ret.length === 0) {
+		// Function with no arguments, resolve eagerly
+		ret = ret.call(prism);
+	}
+
+	if (typeof ret === 'object' && ret.$rest) {
+		let restGrammar = resolve(ret.$rest, prism) ?? {};
+
+		if (typeof restGrammar === 'object') {
+			ret = { ...ret, ...restGrammar };
+		}
+	}
+
+	return ret;
+}
+
+export function tokenizeByNamedGroups (match: RegExpExecArray): ({ type; content } | string)[] {
+	let str = match[0];
+	let result: ({ type; content } | string)[] = [];
+	let i = 0;
+
+	let entries = Object.entries(match.indices?.groups || {})
+		.map(([type, [start, end]]) => ({
+			type,
+			start: start - match.index,
+			end: end - match.index,
+		}))
+		.sort((a, b) => a.start - b.start);
+
+	for (let { type, start, end } of entries) {
+		if (start > i) {
+			result.push(str.slice(i, start));
+		}
+
+		let content = str.slice(start, end);
+		type = camelToKebabCase(type);
+		result.push({ type, content });
+		i = end;
+	}
+
+	if (i < str.length) {
+		result.push(str.slice(i));
 	}
 
-	return reference ?? undefined;
+	return result;
 }

src/shared/languages/templating.ts

@@ -105,6 +105,7 @@ function insertIntoHostToken (hostTokens: TokenStream, tokenStack: TokenStack) {
 
 type GrammarRef = Grammar | string | undefined | null;
 
+// TODO use resolve() from tokenize/util
 function resolve (ref: GrammarRef, components: Registry): Grammar | undefined {
 	if (!ref) {
 		return undefined;

src/shared/util.ts

@@ -65,3 +65,7 @@ export function kebabToCamelCase<T extends string> (kebab: T): KebabToCamelCase<
 	const [first, ...others] = kebab.split(/-/);
 	return (first + others.map(capitalize).join('')) as KebabToCamelCase<T>;
 }
+
+export function camelToKebabCase (str: string) {
+	return (str + '').replace(/[A-Z]/g, l => '-' + l.toLowerCase());
+}