diff --git a/.prettierignore b/.prettierignore index 6aa58be163..2ef332c920 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1 +1,2 @@ /src/plugins +tsconfig.json diff --git a/benchmark/benchmark.ts b/benchmark/benchmark.ts index 40a87dbe9b..c4bd4870d3 100644 --- a/benchmark/benchmark.ts +++ b/benchmark/benchmark.ts @@ -1,17 +1,18 @@ -import Benchmark from 'benchmark'; -import fetch from 'cross-fetch'; import crypto from 'crypto'; import fs from 'fs'; import path from 'path'; +import { fileURLToPath } from 'url'; +import Benchmark from 'benchmark'; +import fetch from 'cross-fetch'; import { gitP } from 'simple-git'; import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; +import { toArray } from '../src/util/iterables'; import { parseLanguageNames } from '../tests/helper/test-case'; import { config as baseConfig } from './config'; -import type { Prism } from '../src/core'; +import type { Prism } from '../src/types'; import type { Config, ConfigOptions } from './config'; import type { Options, Stats } from 'benchmark'; -import { fileURLToPath } from 'url'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -129,7 +130,6 @@ async function runBenchmark (config: Config) { const name = candidates[i].name.padEnd(maxCandidateNameLength, ' '); const best = String(s.best).padStart('best'.length); const worst = String(s.worst).padStart('worst'.length); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const relative = ((s.avgRelative! / minAvgRelative).toFixed(2) + 'x').padStart( 'relative'.length ); @@ -279,8 +279,7 @@ async function getFilePath (uri: string) { // file path const hash = crypto.createHash('md5').update(uri).digest('hex'); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const localPath = path.resolve(downloadDir, hash + '-' + /[-\w\.]*$/.exec(uri)![0]); + const localPath = path.resolve(downloadDir, hash + '-' + /[-.\w]*$/.exec(uri)![0]); if (!fs.existsSync(localPath)) { // download file @@ -364,7 +363,6 @@ function createTestFunction ( if (testFunction === 'tokenize') { return code => { const grammar = Prism.components.getLanguage(mainLanguage); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion Prism.tokenize(code, grammar!); }; } @@ -406,7 +404,6 @@ async function getCandidates (config: Config): Promise { const baseGit = gitP(remoteBaseDir); for (const remote of config.remotes) { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const user = /[^/]+(?=\/prism.git)/.exec(remote.repo)![0]; const branch = remote.branch || 'main'; const remoteName = `${user}@${branch}`; @@ -440,19 +437,4 @@ async function getCandidates (config: Config): Promise { return candidates; } -/** - * A utility function that converts the given optional array-like value into an array. - */ -function toArray (value: T[] | T | undefined | null): readonly T[] { - if (Array.isArray(value)) { - return value; - } - else if (value != null) { - return [value]; - } - else { - return []; - } -} - runBenchmark(getConfig()).catch(error => console.error(error)); diff --git a/eslint.config.mjs b/eslint.config.mjs index 513bc0b158..826d0e605c 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -1,17 +1,15 @@ +import path from 'path'; +import { fileURLToPath } from 'url'; import js from '@eslint/js'; - import tsEslintPlugin from '@typescript-eslint/eslint-plugin'; import tsEslintParser from '@typescript-eslint/parser'; -import { defineConfig } from 'eslint/config'; import eslintConfigPrettier from 'eslint-config-prettier/flat'; import eslintCommentsPlugin from 'eslint-plugin-eslint-comments'; import jsdocPlugin from 'eslint-plugin-jsdoc'; import regexpPlugin from 'eslint-plugin-regexp'; +import { defineConfig } from 'eslint/config'; import globals from 'globals'; -import path from 'path'; -import { fileURLToPath } from 'url'; - const __dirname = path.dirname(fileURLToPath(import.meta.url)); const config = [ @@ -41,7 +39,7 @@ const config = [ 'object-shorthand': ['warn', 'always', { avoidQuotes: true }], 'one-var': ['warn', 'never'], 'prefer-arrow-callback': 'warn', - 'prefer-const': ['warn', { 'destructuring': 'all' }], + 'prefer-const': 'off', 'prefer-spread': 'warn', // JSDoc @@ -154,11 +152,17 @@ const config = [ '@typescript-eslint/no-unsafe-call': 'off', '@typescript-eslint/no-explicit-any': 'off', + '@typescript-eslint/no-unsafe-assignment': 'off', + '@typescript-eslint/no-unsafe-member-access': 'off', + '@typescript-eslint/no-unsafe-return': 'off', + '@typescript-eslint/no-unsafe-argument': 'off', + '@typescript-eslint/no-floating-promises': 'off', + '@typescript-eslint/await-thenable': 'off', }, }, { // Browser-specific parts - files: ['src/auto-start.ts'], + files: ['src/global.ts'], languageOptions: { globals: { ...globals.browser, diff --git a/package-lock.json b/package-lock.json index b20dd36047..d488b8e967 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,8 +43,8 @@ "mocha-chai-jest-snapshot": "^1.1.6", "npm-run-all": "^4.1.5", "prettier": "^3.5.3", - "prettier-plugin-brace-style": "^0.7.2", - "prettier-plugin-merge": "^0.7.3", + "prettier-plugin-brace-style": "^0.7.3", + "prettier-plugin-merge": "^0.7.4", "prettier-plugin-space-before-function-paren": "^0.0.8", "refa": "^0.9.1", "regexp-ast-analysis": "^0.5.1", @@ -8615,9 +8615,9 @@ } }, "node_modules/prettier-plugin-brace-style": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/prettier-plugin-brace-style/-/prettier-plugin-brace-style-0.7.2.tgz", - "integrity": "sha512-n/phxv28jzqwgezbdsoSLgFaiXRO/rD9nFAKDtkJZonsABW5yV5MIwPTNRU8tZcDa9lX6wMdI/yTfg5U9T0wzg==", + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/prettier-plugin-brace-style/-/prettier-plugin-brace-style-0.7.3.tgz", + "integrity": "sha512-dpNenn3Dm4BdDmVE0azn0SLclTTcMgWjMGYsQgT5uV/mKNToYi97bT+SiAtSYb36SRQFlZ0nEs/2eD/tUI3maA==", "dev": true, "license": "MIT", "engines": { @@ -8638,9 +8638,9 @@ } }, "node_modules/prettier-plugin-merge": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/prettier-plugin-merge/-/prettier-plugin-merge-0.7.3.tgz", - "integrity": "sha512-5b4h5MZRFFTqlGWMWS3KOFUWTfEesXdMPjaT5LbTw9Jck7Ghu1hjrhM6P0Dfrzhf0zrqdwIjyFMtF6YG90dOXg==", + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/prettier-plugin-merge/-/prettier-plugin-merge-0.7.4.tgz", + "integrity": "sha512-cxuInnqaO8d/zZx/tk8PEKtab8QetiBsOlXH+kYqBF66lSwJoy491jQT3ocOdJ/8nzom40QGGq/wUblFe7zRxw==", "dev": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index f452ddccc4..bbdb61baad 100644 --- a/package.json +++ b/package.json @@ -96,8 +96,8 @@ "mocha-chai-jest-snapshot": "^1.1.6", "npm-run-all": "^4.1.5", "prettier": "^3.5.3", - "prettier-plugin-brace-style": "^0.7.2", - "prettier-plugin-merge": "^0.7.3", + "prettier-plugin-brace-style": "^0.7.3", + "prettier-plugin-merge": "^0.7.4", "prettier-plugin-space-before-function-paren": "^0.0.8", "refa": "^0.9.1", "regexp-ast-analysis": "^0.5.1", diff --git a/scripts/build.ts b/scripts/build.ts index 7e8b9a5d40..ada382020c 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -13,7 +13,7 @@ import { webfont } from 'webfont'; import { toArray } from '../src/util/iterables'; import { components } from './components'; import { parallel, runTask, series } from './tasks'; -import type { ComponentProto } from '../src/types'; +import type { ComponentProto, LanguageProto } from '../src/types'; import type { OutputOptions, Plugin, RollupBuild, RollupOptions, SourceMapInput } from 'rollup'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -143,7 +143,7 @@ const dataToInsert = { const data = await Promise.all( [...languageIds, ...pluginIds].map(async id => { const proto = await loadComponent(id); - return { id, alias: toArray(proto.alias) }; + return { id, alias: toArray((proto as LanguageProto)?.alias) }; }) ); return Object.fromEntries(data.flatMap(({ id, alias }) => alias.map(a => [a, id]))); @@ -165,12 +165,12 @@ const dataToInsert = { const data = ( await Promise.all( languageIds.map(async id => { - const proto = await loadComponent(id); + const proto = (await loadComponent(id)) as LanguageProto; const title = rawTitles.get(id); if (!title) { throw new Error(`No title for ${id}`); } - return [id, ...toArray(proto.alias)].map(name => ({ + return [id, ...toArray(proto?.alias)].map(name => ({ name, title: rawTitles.get(id) ?? title, })); @@ -321,7 +321,7 @@ async function clean () { ]); } -async function copyComponentsJson () { +async function copyComponentsJson() { const from = path.join(SRC_DIR, 'components.json'); const to = path.join(__dirname, '../dist/components.json'); await copyFile(from, to); diff --git a/src/components.json b/src/components.json index d349a7f244..769e4987e1 100644 --- a/src/components.json +++ b/src/components.json @@ -903,10 +903,6 @@ "title": "PHPDoc", "owner": "RunDevelopment" }, - "php-extras": { - "title": "PHP Extras", - "owner": "milesj" - }, "plant-uml": { "title": "PlantUML", "owner": "RunDevelopment" @@ -1304,10 +1300,6 @@ }, "owner": "freakmaxi" }, - "xml-doc": { - "title": "XML doc (.net)", - "owner": "RunDevelopment" - }, "xojo": { "title": "Xojo (REALbasic)", "owner": "Golmote" diff --git a/src/config.ts b/src/config.ts index 364cd06d0a..a2c2c6ad93 100644 --- a/src/config.ts +++ b/src/config.ts @@ -6,7 +6,7 @@ const globalConfig: Record = function getGlobalSetting (name: string) { // eslint-disable-next-line regexp/no-unused-capturing-group - const camelCaseName = name.replace(/-([a-z])/g, g => g[1].toUpperCase()); + let camelCaseName = name.replace(/-([a-z])/g, g => g[1].toUpperCase()); if (camelCaseName in globalConfig) { return globalConfig[camelCaseName]; @@ -32,12 +32,38 @@ function getGlobalBooleanSetting (name: string, defaultValue: boolean): boolean return !(value === false || value === 'false'); } +function getGlobalArraySetting (name: string): string[] { + const value = getGlobalSetting(name); + if (value === null || value === undefined || value === false || value === 'false') { + return []; + } + else if (typeof value === 'string') { + return value.split(',').map(s => s.trim()); + } + else if (Array.isArray(value)) { + return value; + } + + return []; +} + export interface PrismConfig { manual?: boolean; + silent?: boolean; + errorHandler?: (reason: any) => PromiseLike; + plugins?: string[]; + languages?: string[]; + pluginPath?: string; + languagePath?: string; } export const globalDefaults: PrismConfig = { manual: getGlobalBooleanSetting('manual', !hasDOM), + silent: getGlobalBooleanSetting('silent', false), + languages: getGlobalArraySetting('languages'), + plugins: getGlobalArraySetting('plugins'), + languagePath: (getGlobalSetting('language-path') ?? './languages/') as string, + pluginPath: (getGlobalSetting('plugin-path') ?? './plugins/') as string, }; export default globalDefaults; diff --git a/src/core.ts b/src/core.ts index ca6229da51..d614da408d 100644 --- a/src/core.ts +++ b/src/core.ts @@ -1,2 +1,2 @@ -export { Prism } from './core/prism'; +export { default as Prism } from './core/classes/prism'; export { Token } from './core/classes/token'; diff --git a/src/core/classes/component-registry.ts b/src/core/classes/component-registry.ts new file mode 100644 index 0000000000..b99374c913 --- /dev/null +++ b/src/core/classes/component-registry.ts @@ -0,0 +1,171 @@ +import { allSettled } from '../../util/async'; +import type { ComponentProto } from '../../types'; + +export interface ComponentRegistryOptions { + /** Path to the components */ + path: string; + preload?: string[]; +} + +export interface ComponentProtoBase { + id: Id; + require?: ComponentProto | readonly ComponentProto[]; + optional?: string | readonly string[]; +} + +export interface AddEventPayload { + id: string; + type?: string; + component: T; +} + +export default class ComponentRegistry extends EventTarget { + static type: string = 'unknown'; + + /** All imported components */ + cache: Record = {}; + + /** All components that are currently being loaded */ + loading: Record> = {}; + + /** + * Same data as in loading, but as an array, used for aggregate promises. + * IMPORTANT: Do NOT overwrite this array, only modify its contents. + */ + private loadingList: Promise[] = []; + + ready: Promise; + + /** Path to the components, used for loading */ + path: string; + + options: ComponentRegistryOptions; + + constructor (options: ComponentRegistryOptions) { + super(); + this.options = options; + let { path, preload } = options; + path = path.endsWith('/') ? path : path + '/'; + this.path = path; + + if (preload) { + void this.loadAll(preload); + } + + this.ready = allSettled(this.loadingList) as Promise; + } + + /** + * Returns the component if it is already loaded or a promise that resolves when it is loaded, + * without triggering a load like `load()` would. + * + * @param id + * @returns + */ + async whenDefined (id: string): Promise { + if (this.cache[id]) { + // Already loaded + return this.cache[id]; + } + + if (this.loading[id] !== undefined) { + // Already loading + return this.loading[id]; + } + + let Self = this.constructor as typeof ComponentRegistry; + return new Promise(resolve => { + let handler = (e: CustomEvent>) => { + if (e.detail.id === id) { + resolve(e.detail.component); + this.removeEventListener('add', handler as EventListener); + } + }; + this.addEventListener('add' + Self.type, handler as EventListener); + }); + } + + /** + * Add a component to the registry. + * + * @param def Component + * @param id Component id + * @param options Options + * @param options.force Force add the component even if it is already present + * @returns true if the component was added, false if it was already present + */ + add (def: T, id: string = def.id, options?: { force?: boolean }): boolean { + let Self = this.constructor as typeof ComponentRegistry; + + if (typeof this.loading[id] !== 'undefined') { + // If it was loading, remove it from the loading list + let index = this.loadingList.indexOf(this.loading[id]); + if (index > -1) { + this.loadingList.splice(index, 1); + } + + delete this.loading[id]; + } + + if (!this.cache[id] || options?.force) { + this.cache[id] = def; + + this.dispatchEvent( + new CustomEvent>('add', { + detail: { id, type: Self.type, component: def }, + }) + ); + this.dispatchEvent( + new CustomEvent>('add' + Self.type, { + detail: { id, component: def }, + }) + ); + + return true; + } + + return false; + } + + has (id: string): boolean { + return this.cache[id] !== undefined; + } + + get (id: string): T | null { + return this.cache[id] ?? null; + } + + load (id: string): T | Promise { + if (this.cache[id]) { + return this.cache[id]; + } + + if (this.loading[id] !== undefined) { + // Already loading + return this.loading[id]; + } + + let loadingComponent = import(this.path + id + '.js') + .then(m => { + let component: T = m.default ?? m; + this.add(component, id); + return component; + }) + .catch(error => { + console.error(error); + return null; + }); + + this.loading[id] = loadingComponent as Promise; + this.loadingList.push(loadingComponent as Promise); + return loadingComponent; + } + + loadAll (ids: string[]): (T | Promise)[] { + if (!Array.isArray(ids)) { + ids = [ids]; + } + + return ids.map(id => this.load(id)); + } +} diff --git a/src/core/classes/language-registry.ts b/src/core/classes/language-registry.ts new file mode 100644 index 0000000000..e63f7b3063 --- /dev/null +++ b/src/core/classes/language-registry.ts @@ -0,0 +1,115 @@ +import ComponentRegistry from './component-registry'; +import Language from './language'; +import type { LanguageProto, Languages } from './language'; + +export { type ComponentProtoBase } from './component-registry'; + +export default class LanguageRegistry extends ComponentRegistry { + static type: string = 'language'; + aliases: Record = {}; + instances: Languages = {}; + defs = new WeakMap(); + + /** + * Add a language definition to the registry. + * This does not necessarily resolve the language. + */ + add (def: LanguageProto): boolean { + let added = super.add(def); + + if (added && def.alias) { + let id = def.id; + + if (typeof def.alias === 'string') { + this.aliases[def.alias] = id; + } + else if (Array.isArray(def.alias)) { + for (let alias of def.alias) { + this.aliases[alias] = id; + } + } + } + + return added; + } + + resolveRef (ref: string | LanguageProto | Language): { + id: string; + def: LanguageProto; + language?: Language; + } { + if (ref instanceof Language) { + return { id: ref.id, def: ref.def, language: ref }; + } + + let id: string; + let def: LanguageProto; + + if (typeof ref === 'object') { + def = ref; + id = def.id; + } + else if (typeof ref === 'string') { + id = ref; + } + else { + // eslint-disable-next-line @typescript-eslint/restrict-plus-operands + throw new Error('Invalid argument type: ' + ref); + } + + id = this.aliases[id] ?? id; + def ??= this.cache[id]; + let language = this.instances[id]; + + return { id, def, language }; + } + + /** + * Get resolved language, language definition or null if it doesn't exist. + * If definition is loaded but not yet resolved, it will NOT be resolved. Use {@link getLanguage} for that. + * + * @param ref Language id or definition + */ + peek (ref: string | LanguageProto | Language): Language | null { + let { id, def, language } = this.resolveRef(ref); + + if (language) { + return language; + } + + if (this.defs.has(def)) { + return this.defs.get(def) ?? null; + } + + if (this.instances[id]) { + return this.instances[id]; + } + + return null; + } + + /** + * Get resolved language or null if it doesn't exist + * If definition is loaded but not yet resolved, it will be resolved and returned. + */ + getLanguage (ref: string | LanguageProto | Language): Language | null { + let languageOrDef = this.peek(ref); + + if (languageOrDef instanceof Language) { + return languageOrDef; + } + + let { id, def } = this.resolveRef(ref); + + if (!this.cache[id]) { + return null; + } + + // NOTE this will overwrite any existing language with the same id + // We can add an option to prevent this in the future + let language = new Language(def, this); + this.defs.set(def, language); + this.instances[def.id] = language; + return language; + } +} diff --git a/src/core/classes/language.ts b/src/core/classes/language.ts new file mode 100644 index 0000000000..007e853af4 --- /dev/null +++ b/src/core/classes/language.ts @@ -0,0 +1,176 @@ +import { extend } from '../../util/extend'; +import { grammarPatch } from '../../util/grammar-patch'; +import { deepClone, defineLazyProperty } from '../../util/objects'; +import List from './list'; +import type { Grammar, GrammarOptions } from '../../types'; +import type LanguageRegistry from './language-registry'; +import type { ComponentProtoBase } from './language-registry'; + +export default class Language extends EventTarget { + def: LanguageProto; + registry: LanguageRegistry; + require: List = new List(); + optional: List = new List(); + languages: LanguageGrammars = {}; + readyState = 0; + + constructor (def: LanguageProto, registry: LanguageRegistry) { + super(); + this.def = def; + this.registry = registry; + + if (this.def.base) { + this.require.add(this.def.base); + } + if (this.def.require) { + this.require.addAll(this.def.require as LanguageProto | readonly LanguageProto[]); + } + + if (this.def.optional) { + this.optional.addAll(this.def.optional); + + if (this.optional.size > 0) { + for (let optionalLanguageId of this.optional) { + if (!this.registry.has(optionalLanguageId as string)) { + this.registry.whenDefined(optionalLanguageId as string).then(() => { + // TODO + }); + } + } + } + } + if (this.def.extends) { + this.optional.addAll(this.def.extends); + } + + for (let def of this.require) { + defineLazyProperty(this.languages, def.id, () => { + let language = this.registry.peek(def as LanguageProto); + if (language) { + // Already resolved + return language.resolvedGrammar; + } + else { + this.registry.add(def as LanguageProto); + return this.registry.getLanguage(def.id)!.resolvedGrammar; + } + }); + } + + for (let id of this.optional as List) { + // TODO we need to update the grammar + defineLazyProperty( + this.languages, + id, + () => { + return this.registry.getLanguage(id)!.resolvedGrammar; + }, + this.registry.peek(id) ?? this.registry.whenDefined(id) + ); + } + } + + resolve () {} + + get id () { + return this.def.id; + } + + get alias () { + if (!this.def.alias) { + return []; + } + + return Array.isArray(this.def.alias) ? this.def.alias : [this.def.alias]; + } + + get base (): Language | null { + if (!this.def.base) { + return null; + } + + let base = this.def.base; + let language = this.registry.peek(base); + if (language) { + // Already resolved + return language; + } + else { + this.registry.add(base as LanguageProto); + return this.registry.getLanguage(base.id); + } + } + + get extends () { + if (!this.def.extends) { + return []; + } + return Array.isArray(this.def.extends) ? this.def.extends : [this.def.extends]; + } + + get grammar (): Grammar { + // Lazily evaluate grammar + const def = this.def; + + let { grammar } = def; + let base = this.base; + + if (typeof grammar === 'function') { + grammar = grammar.call(this, { + get base () { + return base?.resolvedGrammar; + }, + languages: this.languages, + getLanguage: (id: string) => { + let language = this.languages[id] ?? this.registry.getLanguage(id); + return language?.resolvedGrammar as Grammar ?? language; + }, + whenDefined: (id: string) => { + return this.registry.whenDefined(id) as unknown as Promise; + }, + }); + } + + if (base) { + grammar = extend(base.grammar, grammar); + } + + if (def.grammar === grammar) { + // We need these to be separate so that any code modifying them doesn't affect other instances + grammar = deepClone(grammar); + } + + // This will replace the getter with a writable property + return (this.grammar = grammar as Grammar); + } + + set grammar (grammar: Grammar) { + this.readyState = 2; + Object.defineProperty(this, 'grammar', { value: grammar, writable: true }); + } + + get resolvedGrammar () { + let ret = grammarPatch(this.grammar); + return (this.resolvedGrammar = ret); + } + + set resolvedGrammar (grammar: Grammar) { + this.readyState = 3; + Object.defineProperty(this, 'resolvedGrammar', { value: grammar, writable: true }); + } +} + +export interface LanguageProto extends ComponentProtoBase { + media?: string | readonly string[]; + extensions?: string | readonly string[]; + alias?: string | readonly string[]; + grammar: Grammar | ((options: GrammarOptions) => Grammar); + plugin?: undefined; + base?: LanguageLike; + extends?: string | LanguageLike | readonly (string | LanguageLike)[]; +} + +export type { Language }; +export type Languages = Record; +export type LanguageGrammars = Record; +export type LanguageLike = Language | LanguageProto; diff --git a/src/core/classes/list.ts b/src/core/classes/list.ts new file mode 100644 index 0000000000..24b67f5bb9 --- /dev/null +++ b/src/core/classes/list.ts @@ -0,0 +1,25 @@ +import { toIterable } from '../../util/iterables'; + +/** + * Set with some conveniences + */ +export default class List extends Set { + /** + * Alias of `size` so these objects can be handled like arrays + */ + get length () { + return this.size; + } + + addAll (arg: Iterable | T) { + if (!arg) { + return this; + } + + for (const item of toIterable(arg)) { + this.add(item); + } + + return this; + } +} diff --git a/src/core/classes/plugin-registry.ts b/src/core/classes/plugin-registry.ts new file mode 100644 index 0000000000..e15a4d69f2 --- /dev/null +++ b/src/core/classes/plugin-registry.ts @@ -0,0 +1,13 @@ +import ComponentRegistry from './component-registry'; +import type { KebabToCamelCase, Prism } from '../../types'; +import type { ComponentProtoBase } from './component-registry'; + +export default class PluginRegistry extends ComponentRegistry { + static type: string = 'plugin'; +} + +export interface PluginProto extends ComponentProtoBase { + grammar?: undefined; + plugin?: (Prism: Prism & { plugins: Record, undefined> }) => {}; + effect?: (Prism: Prism & { plugins: Record, {}> }) => () => void; +} diff --git a/src/core/classes/prism.ts b/src/core/classes/prism.ts index c83ffb4bc5..3b21b4afa5 100644 --- a/src/core/classes/prism.ts +++ b/src/core/classes/prism.ts @@ -1,14 +1,19 @@ import globalDefaults from '../../config'; +import { allSettled, documentReady, nextTick } from '../../util/async'; import { highlight } from '../highlight'; import { highlightAll } from '../highlight-all'; import { highlightElement } from '../highlight-element'; import { Registry } from '../registry'; import { tokenize } from '../tokenize/tokenize'; import { Hooks } from './hooks'; -import type { Grammar } from '../../types'; +import LanguageRegistry from './language-registry'; +import PluginRegistry from './plugin-registry'; +import type { PrismConfig } from '../../config'; +import type { Grammar, PluginProto } from '../../types'; import type { HighlightOptions } from '../highlight'; import type { HighlightAllOptions } from '../highlight-all'; import type { HighlightElementOptions } from '../highlight-element'; +import type { LanguageLike } from './language'; import type { TokenStream } from './token'; /** @@ -17,9 +22,96 @@ import type { TokenStream } from './token'; */ export default class Prism { hooks = new Hooks(); + + // TODO remove this and make sure the functionality is covered by language and plugin registries components = new Registry(this); - plugins: Record = {}; - config = globalDefaults; + languageRegistry: LanguageRegistry; + pluginRegistry: PluginRegistry; + + config: PrismConfig; + + languagesReady: Promise; + waitFor: Promise[] = [nextTick()]; + ready: Promise = allSettled(this.waitFor); + + constructor (config: PrismConfig = {}) { + this.config = Object.assign({}, globalDefaults, config); + this.config.errorHandler ??= ( + this.config.silent ? () => undefined : console.error + ) as PrismConfig['errorHandler']; + + const reportError: PrismConfig['errorHandler'] = this.config.errorHandler; + + this.languageRegistry = new LanguageRegistry({ + path: this.config.languagePath as string, + preload: this.config.languages, + }); + this.pluginRegistry = new PluginRegistry({ + path: this.config.pluginPath as string, + preload: this.config.plugins, + }); + + // Preload languages + const languages = this.config.languages; + if (languages && languages.length > 0) { + this.languageRegistry.loadAll(languages); + } + + this.languagesReady = this.languageRegistry.ready; + this.waitFor.push(this.languagesReady); + + const plugins = this.config.plugins; + if (plugins && plugins.length > 0) { + let pluginsReady = this.languagesReady + .then(() => { + return this.pluginRegistry.loadAll(plugins); + }) + .catch(reportError); + this.waitFor.push(pluginsReady); + } + + if (!this.config.manual) { + this.waitFor.push(documentReady()); + + this.ready + .then(() => { + this.highlightAll(); + }) + .catch(reportError); + } + } + + get languages () { + return this.languageRegistry.cache; + } + + get plugins () { + return this.pluginRegistry.cache; + } + + /** + * Load a language by its ID. + */ + async loadLanguage (id: string): Promise { + let language = await this.languageRegistry.load(id); + + return language; + } + + /** + * Load a plugin by its ID. + */ + async loadPlugin (id: string): Promise { + await this.languagesReady; // first, wait for any pending languages to load + let plugin = await this.pluginRegistry.load(id); + + if (plugin?.effect) { + // Call the effect function of the plugin + plugin.effect(this); + } + + return plugin; + } /** * See {@link highlightAll}. diff --git a/src/core/highlight-all.ts b/src/core/highlight-all.ts index ebf7153fc9..9bbd1ca086 100644 --- a/src/core/highlight-all.ts +++ b/src/core/highlight-all.ts @@ -1,7 +1,5 @@ -import singleton from './prism'; -import type { HookEnv } from './classes/hooks'; -import type { AsyncHighlighter } from './highlight-element'; -import type { Prism } from './prism'; +import singleton, { type Prism } from './prism'; +import type { HighlightElementOptions } from './highlight-element'; /** * This is the most high-level function in Prism’s API. @@ -17,7 +15,7 @@ export function highlightAll (this: Prism, options: HighlightAllOptions = {}) { const prism = this ?? singleton; const { root, async, callback } = options; - const env: HookEnv = { + const env: Record = { callback, root: root ?? document, selector: @@ -35,16 +33,9 @@ export function highlightAll (this: Prism, options: HighlightAllOptions = {}) { } } -export interface HighlightAllOptions { +export interface HighlightAllOptions extends HighlightElementOptions { /** * The root element, whose descendants that have a `.language-xxxx` class will be highlighted. */ root?: ParentNode; - async?: AsyncHighlighter; - /** - * An optional callback to be invoked on each element after its highlighting is done. - * - * @see HighlightElementOptions#callback - */ - callback?: (element: Element) => void; } diff --git a/src/core/highlight-element.ts b/src/core/highlight-element.ts index 2901832910..db83bc8d25 100644 --- a/src/core/highlight-element.ts +++ b/src/core/highlight-element.ts @@ -1,8 +1,7 @@ import { getLanguage, setLanguage } from '../shared/dom-util'; import { htmlEncode } from '../shared/util'; import singleton from './prism'; -import type { Grammar } from '../types'; -import type { HookEnv } from './classes/hooks'; +import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types'; import type { Prism } from './prism'; /** @@ -32,8 +31,8 @@ export function highlightElement ( // Find language const language = getLanguage(element); - const languageId = this.components.resolveAlias(language); - const grammar = this.components.getLanguage(languageId); + const languageId = prism.components.resolveAlias(language); + const grammar = prism.languageRegistry.getLanguage(languageId); // Set language on the element, if not present setLanguage(element, language); @@ -46,7 +45,7 @@ export function highlightElement ( const code = element.textContent as string; - const env: HookEnv = { + const env: Record = { element, language, grammar, @@ -113,5 +112,4 @@ export interface AsyncHighlightingData { code: string; grammar: Grammar; } - export type AsyncHighlighter = (data: AsyncHighlightingData) => Promise; diff --git a/src/core/highlight.ts b/src/core/highlight.ts index 5a39d7c3ec..a65bc50fa9 100644 --- a/src/core/highlight.ts +++ b/src/core/highlight.ts @@ -1,11 +1,26 @@ import singleton from './prism'; -import stringify from './stringify'; -import type { Grammar } from '../types'; +import { stringify } from './stringify'; +import { tokenize } from './tokenize/tokenize'; import type { HookEnv } from './classes/hooks'; +import type { LanguageLike, LanguageProto } from './classes/language'; +import type { Token, TokenStream } from './classes/token'; import type { Prism } from './prism'; +declare module './classes/hooks' { + interface HookEnv { + 'before-tokenize': { + code: string; + languageId?: string; + languageDef?: LanguageProto; + language?: any; + languageReady?: Promise; + }; + 'after-tokenize': HookEnv['before-tokenize'] & { tokens?: TokenStream }; + } +} + /** - * Low-level function, only use if you know what you’re doing. It accepts a string of text as input + * Low-level function, only use if you know what you're doing. It accepts a string of text as input * and the language definitions to use, and returns a string with the HTML produced. * * The following hooks will be run: @@ -25,30 +40,34 @@ import type { Prism } from './prism'; export function highlight ( this: Prism, text: string, - language: string, - options?: HighlightOptions + languageRef: string | LanguageLike, + options: HighlightOptions = {} ): string { const prism = this ?? singleton; - const languageId = this.components.resolveAlias(language); - const grammar = options?.grammar ?? this.components.getLanguage(languageId); + const { id, def, language } = prism.languageRegistry.resolveRef(languageRef); - const env: HookEnv = { + let env: HookEnv['after-tokenize'] = { code: text, - grammar, + languageId: id, + languageDef: def, language, }; + + if (env.languageDef && !env.language) { + env.language = prism.languageRegistry.getLanguage(env.languageDef); + } + prism.hooks.run('before-tokenize', env); - if (!env.grammar) { - throw new Error('The language "' + env.language + '" has no grammar.'); + + if (!env.language) { + throw new Error(`No language definition found for ${env.languageId}.`); } - env.tokens = prism.tokenize(env.code, env.grammar); + env.tokens = tokenize.call(prism, env.code, env.language!.resolvedGrammar); prism.hooks.run('after-tokenize', env); return stringify(env.tokens, env.language, prism.hooks); } -export interface HighlightOptions { - grammar?: Grammar; -} +export interface HighlightOptions {} diff --git a/src/core/registry.ts b/src/core/registry.ts index a66c315a55..f7de25d849 100644 --- a/src/core/registry.ts +++ b/src/core/registry.ts @@ -1,8 +1,9 @@ +// TODO make sure the functionality is covered elsewhere and remove this file import { kebabToCamelCase } from '../shared/util'; +import { extend } from '../util/extend'; import { forEach } from '../util/iterables'; -import { extend } from '../util/language-util'; import type { ComponentProto, Grammar } from '../types'; -import type { Prism } from './prism'; +import type Prism from './classes/prism'; interface Entry { proto: ComponentProto; @@ -167,8 +168,8 @@ export class Registry { return (entry.evaluatedGrammar = grammar({ getLanguage: required, - getOptionalLanguage: id => this.getLanguage(id), - extend: (id, ref) => extend(required(id), id, ref), + getOptionalLanguage: (id: string) => this.getLanguage(id), + extend: (id: string, ref: Grammar) => extend(required(id), ref), })); } } diff --git a/src/core/stringify.ts b/src/core/stringify.ts index 670b01ac16..7e610d99a5 100644 --- a/src/core/stringify.ts +++ b/src/core/stringify.ts @@ -1,6 +1,19 @@ import { htmlEncode } from '../shared/util'; import type { HookEnv, Hooks } from './classes/hooks'; -import type { Token, TokenStream } from './classes/token'; +import type { Token, TokenName, TokenStream } from './classes/token'; + +declare module './classes/hooks' { + interface HookEnv { + 'wrap': { + type: TokenName; + languageId: string; + content: string; + tag: string; + classes: string[]; + attributes: Record; + }; + } +} /** * Converts the given token or token stream to an HTML representation. @@ -9,28 +22,29 @@ import type { Token, TokenStream } from './classes/token'; * 1. `wrap`: On each {@link Token}. * * @param o The token or token stream to be converted. - * @param language The name of current language. + * @param languageId The name of current language. * @returns The HTML representation of the token or token stream. */ -function stringify (o: string | Token | TokenStream, language: string, hooks: Hooks): string { +function stringify (o: string | Token | TokenStream, languageId: string, hooks: Hooks): string { if (typeof o === 'string') { return htmlEncode(o); } + if (Array.isArray(o)) { let s = ''; o.forEach(e => { - s += stringify(e, language, hooks); + s += stringify(e, languageId, hooks); }); return s; } - const env: HookEnv = { + const env: HookEnv['wrap'] = { type: o.type, - content: stringify(o.content, language, hooks), + content: stringify(o.content, languageId, hooks), tag: 'span', classes: ['token', o.type], attributes: {}, - language, + languageId, }; const aliases = o.alias; @@ -45,25 +59,12 @@ function stringify (o: string | Token | TokenStream, language: string, hooks: Ho hooks.run('wrap', env); - let attributes = ''; - for (const name in env.attributes) { - attributes += - ' ' + name + '="' + (env.attributes[name] || '').replace(/"/g, '"') + '"'; - } + const attributes = + Object.entries(env.attributes) + .map(([name, value]) => ` ${name}=${(value ?? '').replace(/"/g, '"')}"`) + .join('') || ''; - return ( - '<' + - env.tag + - ' class="' + - env.classes.join(' ') + - '"' + - attributes + - '>' + - env.content + - '' - ); + return `<${env.tag} class="${env.classes.join(' ')}"${attributes}>${env.content}`; } export { stringify }; diff --git a/src/core/tokenize/match.ts b/src/core/tokenize/match.ts index e7a0ee368a..aca6c06d29 100644 --- a/src/core/tokenize/match.ts +++ b/src/core/tokenize/match.ts @@ -1,8 +1,8 @@ import { Token } from '../classes/token'; import singleton from '../prism'; import { tokenize } from './tokenize'; -import { resolve } from './util'; -import type { GrammarToken, GrammarTokens, RegExpLike } from '../../types'; +import { resolve, tokenizeByNamedGroups } from './util'; +import type { Grammar, GrammarToken, GrammarTokens, RegExpLike, TokenStream } from '../../types'; import type { LinkedList, LinkedListHeadNode, @@ -22,13 +22,18 @@ export function _matchGrammar ( ): void { const prism = this ?? singleton; + grammar = resolve.call(prism, grammar) as Grammar; + for (const token in grammar) { const tokenValue = grammar[token]; - if (!grammar.hasOwnProperty(token) || token.startsWith('$') || !tokenValue) { + if (!grammar.hasOwnProperty(token) || !tokenValue) { continue; } - const patterns = Array.isArray(tokenValue) ? tokenValue : [tokenValue]; + const patterns = (Array.isArray(tokenValue) ? tokenValue : [tokenValue]) as ( + | RegExpLike + | GrammarToken + )[]; for (let j = 0; j < patterns.length; ++j) { if (rematch && rematch.cause === `${token},${j}`) { @@ -37,11 +42,21 @@ export function _matchGrammar ( const patternObj = toGrammarToken(patterns[j]); let { pattern, lookbehind = false, greedy = false, alias, inside } = patternObj; - const insideGrammar = resolve(prism.components, inside); + const insideGrammar = resolve.call(prism, inside); + + let flagsToAdd = ''; if (greedy && !pattern.global) { // Without the global flag, lastIndex won't work - patternObj.pattern = pattern = RegExp(pattern.source, pattern.flags + 'g'); + flagsToAdd += 'g'; + } + if (pattern.source?.includes('(?<') && pattern.hasIndices === false) { + // Has named groups, we need to be able to capture their indices + flagsToAdd += 'd'; + } + + if (flagsToAdd) { + patternObj.pattern = pattern = RegExp(pattern.source, pattern.flags + flagsToAdd); } for ( @@ -66,7 +81,7 @@ export function _matchGrammar ( } let removeCount = 1; // this is the to parameter of removeBetween - let match; + let match: RegExpExecArray | null = null; if (greedy) { match = matchPattern(pattern, pos, text, lookbehind); @@ -119,9 +134,9 @@ export function _matchGrammar ( } } - // eslint-disable-next-line no-redeclare const from = match.index; const matchStr = match[0]; + let content: TokenStream | string = matchStr; const before = str.slice(0, from); const after = str.slice(from + matchStr.length); @@ -138,13 +153,37 @@ export function _matchGrammar ( } tokenList.removeRange(removeFrom, removeCount); + let byGroups = match.groups ? tokenizeByNamedGroups(match) : null; + + if (byGroups && byGroups.length > 1) { + content = byGroups.map(arg => { + let content = typeof arg === 'string' ? arg : arg.content; + let type = typeof arg === 'string' ? undefined : arg.type; + + if (insideGrammar) { + let localInsideGrammar = insideGrammar[type] ?? insideGrammar; - const wrapped = new Token( - token, - insideGrammar ? tokenize.call(prism, matchStr, insideGrammar) : matchStr, - alias, - matchStr - ); + if (typeof localInsideGrammar === 'function') { + // Late resolving + localInsideGrammar = resolve.call( + prism, + localInsideGrammar(match.groups) + ); + } + + content = tokenize.call(prism, content, localInsideGrammar); + } + + return typeof arg === 'object' && arg.type + ? new Token(arg.type, content) + : content; + }); + } + else if (insideGrammar) { + content = tokenize.call(prism, content, insideGrammar as Grammar); + } + + const wrapped = new Token(token, content, alias, matchStr); currentNode = tokenList.addAfter(removeFrom, wrapped); if (after) { @@ -159,8 +198,9 @@ export function _matchGrammar ( cause: `${token},${j}`, reach, }; + _matchGrammar.call( - prism, + this, text, tokenList, grammar, diff --git a/src/core/tokenize/tokenize.ts b/src/core/tokenize/tokenize.ts index 3a470a98dd..c3f5acf3c1 100644 --- a/src/core/tokenize/tokenize.ts +++ b/src/core/tokenize/tokenize.ts @@ -1,7 +1,6 @@ import { LinkedList } from '../linked-list'; import singleton from '../prism'; import { _matchGrammar } from './match'; -import { resolve } from './util'; import type { Grammar } from '../../types'; import type { Token, TokenStream } from '../classes/token'; import type { Prism } from '../prism'; @@ -35,12 +34,6 @@ export function tokenize (this: Prism, text: string, grammar: Grammar): TokenStr return customTokenize(text, grammar, prism); } - let restGrammar = resolve(prism.components, grammar.$rest); - while (restGrammar) { - grammar = { ...grammar, ...restGrammar }; - restGrammar = resolve(prism.components, restGrammar.$rest); - } - const tokenList = new LinkedList(); tokenList.addAfter(tokenList.head, text); diff --git a/src/core/tokenize/util.ts b/src/core/tokenize/util.ts index 495b7f34d9..3736638353 100644 --- a/src/core/tokenize/util.ts +++ b/src/core/tokenize/util.ts @@ -1,15 +1,62 @@ -import type { Grammar } from '../../types'; -import type { Registry } from '../registry'; +import { camelToKebabCase } from '../../shared/util'; +import singleton from '../prism'; +import type { Grammar, Prism } from '../../types'; export function resolve ( - components: Registry, - reference: Grammar | string | null | undefined -): Grammar | undefined { - if (reference) { - if (typeof reference === 'string') { - return components.getLanguage(reference); + this: Prism, + reference: Grammar | string | null | undefined | Function +): Grammar | undefined | Function { + let prism = this ?? singleton; + let ret = reference ?? undefined; + + if (typeof ret === 'string') { + ret = prism.languageRegistry.getLanguage(ret)?.resolvedGrammar; + } + + if (typeof ret === 'function' && ret.length === 0) { + // Function with no arguments, resolve eagerly + ret = ret.call(prism); + } + + if (typeof ret === 'object' && ret.$rest) { + let restGrammar = resolve.call(prism, ret.$rest) ?? {}; + delete ret.$rest; + + if (typeof restGrammar === 'object') { + ret = { ...ret, ...restGrammar }; } - return reference; } - return undefined; + + return ret as Grammar | undefined | Function; +} + +export function tokenizeByNamedGroups (match: RegExpExecArray): ({ type; content } | string)[] { + let str = match[0]; + let result: ({ type; content } | string)[] = []; + let i = 0; + + let entries = Object.entries(match.indices?.groups || {}) + .map(([type, [start, end]]) => ({ + type, + start: start - match.index, + end: end - match.index, + })) + .sort((a, b) => a.start - b.start); + + for (let { type, start, end } of entries) { + if (start > i) { + result.push(str.slice(i, start)); + } + + let content = str.slice(start, end); + type = camelToKebabCase(type); + result.push({ type, content }); + i = end; + } + + if (i < str.length) { + result.push(str.slice(i)); + } + + return result; } diff --git a/src/index.ts b/src/index.ts index 4ac1c50765..f907f176d0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,5 @@ // Auto Start runs on Global Prism and the re-exports it. -import globalPrism from './auto-start'; +import globalPrism from './global'; export * from './core'; export { loadLanguages } from './load-languages'; diff --git a/src/languages/actionscript.ts b/src/languages/actionscript.ts index bf0ce545ba..fca05530b9 100644 --- a/src/languages/actionscript.ts +++ b/src/languages/actionscript.ts @@ -1,35 +1,29 @@ -import { insertBefore } from '../util/language-util'; import javascript from './javascript'; -import type { GrammarToken, LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'actionscript', - require: javascript, - grammar ({ extend }) { - const actionscript = extend('javascript', { + base: javascript, + grammar (): Grammar { + return { 'keyword': /\b(?:as|break|case|catch|class|const|default|delete|do|dynamic|each|else|extends|final|finally|for|function|get|if|implements|import|in|include|instanceof|interface|internal|is|namespace|native|new|null|override|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|use|var|void|while|with)\b/, 'operator': /\+\+|--|(?:[+\-*\/%^]|&&?|\|\|?|<>?>?|[!=]=?)=?|[~?@]/, - }); - - const className = actionscript['class-name'] as GrammarToken; - className.alias = 'function'; - - delete actionscript['doc-comment']; - - // doesn't work with AS because AS is too complex - delete actionscript['parameter']; - delete actionscript['literal-property']; - - insertBefore(actionscript, 'string', { - 'xml': { - pattern: - /(^|[^.])<\/?\w+(?:\s+[^\s>\/=]+=("|')(?:\\[\s\S]|(?!\2)[^\\])*\2)*\s*\/?>/, - lookbehind: true, - inside: 'markup', + // doesn't work with AS because AS is too complex + $delete: ['doc-comment', 'parameter', 'literal-property'], + $insertBefore: { + 'string': { + 'xml': { + pattern: + /(^|[^.])<\/?\w+(?:\s+[^\s>\/=]+=("|')(?:\\[\s\S]|(?!\2)[^\\])*\2)*\s*\/?>/, + lookbehind: true, + inside: 'markup', + }, + }, }, - }); - - return actionscript; + $merge: { + 'class-name': { alias: 'function' }, + }, + }; }, } as LanguageProto<'actionscript'>; diff --git a/src/languages/apex.ts b/src/languages/apex.ts index 00655c4a22..ddec2a09b8 100644 --- a/src/languages/apex.ts +++ b/src/languages/apex.ts @@ -1,11 +1,11 @@ import clike from './clike'; import sql from './sql'; -import type { LanguageProto } from '../types'; +import type { GrammarOptions, LanguageProto } from '../types'; export default { id: 'apex', require: [clike, sql], - grammar ({ getLanguage }) { + grammar ({ languages }: GrammarOptions) { const keywords = /\b(?:abstract|activate|(?:after|before)(?=\s+[a-z])|and|any|array|as|asc|autonomous|begin|bigdecimal|blob|boolean|break|bulk|by|byte|case|cast|catch|char|class|collect|commit|const|continue|currency|date|datetime|decimal|default|delete|desc|do|double|else|end|enum|exception|exit|export|extends|final|finally|float|for|from|get(?=\s*[{};])|global|goto|group|having|hint|if|implements|import|in|inner|insert|instanceof|int|integer|interface|into|join|like|limit|list|long|loop|map|merge|new|not|null|nulls|number|object|of|on|or|outer|override|package|parallel|pragma|private|protected|public|retrieve|return|rollback|select|set|short|sObject|sort|static|string|super|switch|synchronized|system|testmethod|then|this|throw|time|transaction|transient|trigger|try|undelete|update|upsert|using|virtual|void|webservice|when|where|while|(?:inherited|with|without)\s+sharing)\b/i; @@ -26,17 +26,14 @@ export default { 'punctuation': /[()\[\]{};,:.<>]/, }; - const clike = getLanguage('clike'); - return { - 'comment': clike.comment, - 'string': clike.string, + 'comment': languages.clike.comment, + 'string': languages.clike.string, 'sql': { pattern: /((?:[=,({:]|\breturn)\s*)\[[^\[\]]*\]/i, lookbehind: true, greedy: true, - alias: 'language-sql', - inside: 'sql', + $inside: 'sql', }, 'annotation': { diff --git a/src/languages/arduino.ts b/src/languages/arduino.ts index 5282cb6b54..bebc6e0495 100644 --- a/src/languages/arduino.ts +++ b/src/languages/arduino.ts @@ -3,16 +3,16 @@ import type { LanguageProto } from '../types'; export default { id: 'arduino', - require: cpp, + base: cpp, alias: 'ino', - grammar ({ extend }) { - return extend('cpp', { + grammar () { + return { 'keyword': /\b(?:String|array|bool|boolean|break|byte|case|catch|continue|default|do|double|else|finally|for|function|goto|if|in|instanceof|int|integer|long|loop|new|null|return|setup|string|switch|throw|try|void|while|word)\b/, 'constant': /\b(?:ANALOG_MESSAGE|DEFAULT|DIGITAL_MESSAGE|EXTERNAL|FIRMATA_STRING|HIGH|INPUT|INPUT_PULLUP|INTERNAL|INTERNAL1V1|INTERNAL2V56|LED_BUILTIN|LOW|OUTPUT|REPORT_ANALOG|REPORT_DIGITAL|SET_PIN_MODE|SYSEX_START|SYSTEM_RESET)\b/, 'builtin': /\b(?:Audio|BSSID|Bridge|Client|Console|EEPROM|Esplora|EsploraTFT|Ethernet|EthernetClient|EthernetServer|EthernetUDP|File|FileIO|FileSystem|Firmata|GPRS|GSM|GSMBand|GSMClient|GSMModem|GSMPIN|GSMScanner|GSMServer|GSMVoiceCall|GSM_SMS|HttpClient|IPAddress|IRread|Keyboard|KeyboardController|LiquidCrystal|LiquidCrystal_I2C|Mailbox|Mouse|MouseController|PImage|Process|RSSI|RobotControl|RobotMotor|SD|SPI|SSID|Scheduler|Serial|Server|Servo|SoftwareSerial|Stepper|Stream|TFT|Task|USBHost|WiFi|WiFiClient|WiFiServer|WiFiUDP|Wire|YunClient|YunServer|abs|addParameter|analogRead|analogReadResolution|analogReference|analogWrite|analogWriteResolution|answerCall|attach|attachGPRS|attachInterrupt|attached|autoscroll|available|background|beep|begin|beginPacket|beginSD|beginSMS|beginSpeaker|beginTFT|beginTransmission|beginWrite|bit|bitClear|bitRead|bitSet|bitWrite|blink|blinkVersion|buffer|changePIN|checkPIN|checkPUK|checkReg|circle|cityNameRead|cityNameWrite|clear|clearScreen|click|close|compassRead|config|connect|connected|constrain|cos|countryNameRead|countryNameWrite|createChar|cursor|debugPrint|delay|delayMicroseconds|detach|detachInterrupt|digitalRead|digitalWrite|disconnect|display|displayLogos|drawBMP|drawCompass|encryptionType|end|endPacket|endSMS|endTransmission|endWrite|exists|exitValue|fill|find|findUntil|flush|gatewayIP|get|getAsynchronously|getBand|getButton|getCurrentCarrier|getIMEI|getKey|getModifiers|getOemKey|getPINUsed|getResult|getSignalStrength|getSocket|getVoiceCallStatus|getXChange|getYChange|hangCall|height|highByte|home|image|interrupts|isActionDone|isDirectory|isListening|isPIN|isPressed|isValid|keyPressed|keyReleased|keyboardRead|knobRead|leftToRight|line|lineFollowConfig|listen|listenOnLocalhost|loadImage|localIP|lowByte|macAddress|maintain|map|max|messageAvailable|micros|millis|min|mkdir|motorsStop|motorsWrite|mouseDragged|mouseMoved|mousePressed|mouseReleased|move|noAutoscroll|noBlink|noBuffer|noCursor|noDisplay|noFill|noInterrupts|noListenOnLocalhost|noStroke|noTone|onReceive|onRequest|open|openNextFile|overflow|parseCommand|parseFloat|parseInt|parsePacket|pauseMode|peek|pinMode|playFile|playMelody|point|pointTo|position|pow|prepare|press|print|printFirmwareVersion|printVersion|println|process|processInput|pulseIn|put|random|randomSeed|read|readAccelerometer|readBlue|readButton|readBytes|readBytesUntil|readGreen|readJoystickButton|readJoystickSwitch|readJoystickX|readJoystickY|readLightSensor|readMessage|readMicrophone|readNetworks|readRed|readSlider|readString|readStringUntil|readTemperature|ready|rect|release|releaseAll|remoteIP|remoteNumber|remotePort|remove|requestFrom|retrieveCallingNumber|rewindDirectory|rightToLeft|rmdir|robotNameRead|robotNameWrite|run|runAsynchronously|runShellCommand|runShellCommandAsynchronously|running|scanNetworks|scrollDisplayLeft|scrollDisplayRight|seek|sendAnalog|sendDigitalPortPair|sendDigitalPorts|sendString|sendSysex|serialEvent|setBand|setBitOrder|setClockDivider|setCursor|setDNS|setDataMode|setFirmwareVersion|setMode|setPINUsed|setSpeed|setTextSize|setTimeout|shiftIn|shiftOut|shutdown|sin|size|sqrt|startLoop|step|stop|stroke|subnetMask|switchPIN|tan|tempoWrite|text|tone|transfer|tuneWrite|turn|updateIR|userNameRead|userNameWrite|voiceCall|waitContinue|width|write|writeBlue|writeGreen|writeJSON|writeMessage|writeMicroseconds|writeRGB|writeRed|yield)\b/, - }); + }; }, } as LanguageProto<'arduino'>; diff --git a/src/languages/arturo.ts b/src/languages/arturo.ts index 6c48548c03..5a3cc4e9bd 100644 --- a/src/languages/arturo.ts +++ b/src/languages/arturo.ts @@ -1,5 +1,6 @@ import type { LanguageProto } from '../types'; +// TODO use function with groups function createLanguageString (lang: string, pattern?: string) { return { pattern: RegExp(/\{!/.source + '(?:' + (pattern || lang) + ')' + /$[\s\S]*\}/.source, 'm'), @@ -8,8 +9,7 @@ function createLanguageString (lang: string, pattern?: string) { 'embedded': { pattern: /(^\{!\w+\b)[\s\S]+(?=\}$)/, lookbehind: true, - alias: 'language-' + lang, - inside: lang, + $inside: lang, }, 'string': /[\s\S]+/, }, diff --git a/src/languages/asciidoc.ts b/src/languages/asciidoc.ts index 864e4933d7..2f38d390bd 100644 --- a/src/languages/asciidoc.ts +++ b/src/languages/asciidoc.ts @@ -28,7 +28,7 @@ export default { 'variable': /\w+(?==)/, 'punctuation': /^\[|\]$|,/, 'operator': /=/, - // The negative look-ahead prevents blank matches + // The negative lookahead prevents blank matches 'attr-value': /(?!^\s+$).+/, }, }; @@ -146,7 +146,7 @@ export default { }, 'inline': { /* - The initial look-behind prevents the highlighting of escaped quoted text. + The initial lookbehind prevents the highlighting of escaped quoted text. Quoted text can be multi-line but cannot span an empty line. All quoted text can have attributes before [foobar, 'foobar', baz="bar"]. @@ -212,11 +212,11 @@ export default { // Allow some nesting. There is no recursion though, so cloning should not be needed. function copyFromAsciiDoc (...keys: (keyof typeof asciidoc)[]) { - const o: Grammar = {}; + const o: Record = {}; for (const key of keys) { o[key] = asciidoc[key] as GrammarToken; } - return o; + return o as Grammar; } attributes.inside['interpreted'].inside.$rest = copyFromAsciiDoc( @@ -280,12 +280,4 @@ export default { return asciidoc; }, - effect (Prism) { - // Plugin to make entity title show the real entity, idea by Roman Komarov - return Prism.hooks.add('wrap', env => { - if (env.type === 'entity') { - env.attributes['title'] = env.content.replace(/&/, '&'); - } - }); - }, } as LanguageProto<'asciidoc'>; diff --git a/src/languages/aspnet.ts b/src/languages/aspnet.ts index 5ebace869d..9ffa1297aa 100644 --- a/src/languages/aspnet.ts +++ b/src/languages/aspnet.ts @@ -1,70 +1,74 @@ -import { insertBefore } from '../util/language-util'; import csharp from './csharp'; import markup from './markup'; -import type { Grammar, GrammarToken, LanguageProto } from '../types'; +import type { Grammar, GrammarOptions, LanguageProto } from '../types'; export default { id: 'aspnet', - require: [markup, csharp], - grammar ({ extend }) { - const pageDirectiveInside: Grammar = { - 'page-directive': { - pattern: - /<%\s*@\s*(?:Assembly|Control|Implements|Import|Master(?:Type)?|OutputCache|Page|PreviousPageType|Reference|Register)?|%>/i, - alias: 'tag', + require: csharp, + base: markup, + grammar ({ base }: GrammarOptions): Grammar { + const directive = { + pattern: /<%.*%>/, + alias: 'tag', + inside: { + 'directive': { + pattern: /<%\s*?[$=%#:]{0,2}|%>/, + alias: 'tag', + }, + $rest: 'csharp', }, }; - const aspnet = extend('markup', { + return { 'page-directive': { pattern: /<%\s*@.*%>/, alias: 'tag', - inside: pageDirectiveInside, - }, - 'directive': { - pattern: /<%.*%>/, - alias: 'tag', inside: { - 'directive': { - pattern: /<%\s*?[$=%#:]{0,2}|%>/, + 'page-directive': { + pattern: + /<%\s*@\s*(?:Assembly|Control|Implements|Import|Master(?:Type)?|OutputCache|Page|PreviousPageType|Reference|Register)?|%>/i, alias: 'tag', }, - $rest: 'csharp', - } as unknown as Grammar, + $rest: base['tag'].inside, + }, }, - }); - - const tag = aspnet['tag'] as GrammarToken & { - inside: { 'attr-value': { inside: Grammar } }; - }; - pageDirectiveInside.$rest = tag.inside; - - // Regexp copied from markup, with a negative look-ahead added - tag.pattern = - /<(?!%)\/?[^\s>\/]+(?:\s+[^\s>\/=]+(?:=(?:("|')(?:\\[\s\S]|(?!\1)[^\\])*\1|[^\s'">=]+))?)*\s*\/?>/; - - // match directives of attribute value foo="<% Bar %>" - insertBefore(tag.inside['attr-value'].inside, 'punctuation', { - 'directive': aspnet['directive'], - }); - - insertBefore(aspnet, 'comment', { - 'asp-comment': { - pattern: /<%--[\s\S]*?--%>/, - alias: ['asp', 'comment'], + 'directive': directive, + $merge: { + 'tag': { + // Regexp copied from markup, with a negative look-ahead added + pattern: + /<(?!%)\/?[^\s>\/]+(?:\s+[^\s>\/=]+(?:=(?:("|')(?:\\[\s\S]|(?!\1)[^\\])*\1|[^\s'">=]+))?)*\s*\/?>/, + inside: { + 'attr-value': { + inside: { + $insertBefore: { + 'punctuation': { + // match directives of attribute value foo="<% Bar %>" + 'directive': directive, + }, + }, + }, + }, + }, + }, }, - }); - - // script runat="server" contains csharp, not javascript - insertBefore(aspnet, 'script' in aspnet ? 'script' : 'tag', { - 'asp-script': { - pattern: /(]*>)[\s\S]*?(?=<\/script>)/i, - lookbehind: true, - alias: ['asp', 'script'], - inside: 'csharp', + $insertBefore: { + 'comment': { + 'asp-comment': { + pattern: /<%--[\s\S]*?--%>/, + alias: ['asp', 'comment'], + }, + }, + // script runat="server" contains csharp, not javascript + ['script' in base ? 'script' : 'tag']: { + 'asp-script': { + pattern: /(]*>)[\s\S]*?(?=<\/script>)/i, + lookbehind: true, + alias: ['asp', 'script'], + inside: 'csharp', + }, + }, }, - }); - - return aspnet; + }; }, } as LanguageProto<'aspnet'>; diff --git a/src/languages/birb.ts b/src/languages/birb.ts index 27ab9e2351..e9deab812c 100644 --- a/src/languages/birb.ts +++ b/src/languages/birb.ts @@ -1,12 +1,11 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; import type { LanguageProto } from '../types'; export default { id: 'birb', - require: clike, - grammar ({ extend }) { - const birb = extend('clike', { + base: clike, + grammar () { + return { 'string': { pattern: /r?("|')(?:\\.|(?!\1)[^\\])*\1/, greedy: true, @@ -21,16 +20,15 @@ export default { /\b(?:assert|break|case|class|const|default|else|enum|final|follows|for|grab|if|nest|new|next|noSeeb|return|static|switch|throw|var|void|while)\b/, 'operator': /\+\+|--|&&|\|\||<<=?|>>=?|~(?:\/=?)?|[+\-*\/%&^|=!<>]=?|\?|:/, 'variable': /\b[a-z_]\w*\b/, - }); - - insertBefore(birb, 'function', { - 'metadata': { - pattern: /<\w+>/, - greedy: true, - alias: 'symbol', + $insertBefore: { + 'function': { + 'metadata': { + pattern: /<\w+>/, + greedy: true, + alias: 'symbol', + }, + }, }, - }); - - return birb; + }; }, } as LanguageProto<'birb'>; diff --git a/src/languages/bison.ts b/src/languages/bison.ts index 69db78ece5..482204b177 100644 --- a/src/languages/bison.ts +++ b/src/languages/bison.ts @@ -1,52 +1,50 @@ -import { insertBefore } from '../util/language-util'; import c from './c'; -import type { Grammar, LanguageProto } from '../types'; +import type { Grammar, GrammarOptions, LanguageProto } from '../types'; export default { id: 'bison', - require: c, - grammar ({ extend, getLanguage }) { - const c = getLanguage('c'); - const bison = extend('c', {}); - - insertBefore(bison, 'comment', { - 'bison': { - // This should match all the beginning of the file - // including the prologue(s), the bison declarations and - // the grammar rules. - pattern: /^(?:[^%]|%(?!%))*%%[\s\S]*?%%/, - inside: { - 'c': { - // Allow for one level of nested braces - pattern: /%\{[\s\S]*?%\}|\{(?:\{[^}]*\}|[^{}])*\}/, + base: c, + grammar ({ base }: GrammarOptions): Grammar { + return { + $insertBefore: { + 'comment': { + 'bison': { + // This should match all the beginning of the file + // including the prologue(s), the bison declarations and + // the grammar rules. + pattern: /^(?:[^%]|%(?!%))*%%[\s\S]*?%%/, inside: { - 'delimiter': { - pattern: /^%?\{|%?\}$/, - alias: 'punctuation', - }, - 'bison-variable': { - pattern: /[$@](?:<[^\s>]+>)?[\w$]+/, - alias: 'variable', + 'c': { + // Allow for one level of nested braces + pattern: /%\{[\s\S]*?%\}|\{(?:\{[^}]*\}|[^{}])*\}/, inside: { - 'punctuation': /<|>/, + 'delimiter': { + pattern: /^%?\{|%?\}$/, + alias: 'punctuation', + }, + 'bison-variable': { + pattern: /[$@](?:<[^\s>]+>)?[\w$]+/, + alias: 'variable', + inside: { + 'punctuation': /<|>/, + }, + }, + $rest: base, }, }, - $rest: c, - } as unknown as Grammar, - }, - 'comment': c.comment, - 'string': c.string, - 'property': /\S+(?=:)/, - 'keyword': /%\w+/, - 'number': { - pattern: /(^|[^@])\b(?:0x[\da-f]+|\d+)/i, - lookbehind: true, + 'comment': base.comment, + 'string': base.string, + 'property': /\S+(?=:)/, + 'keyword': /%\w+/, + 'number': { + pattern: /(^|[^@])\b(?:0x[\da-f]+|\d+)/i, + lookbehind: true, + }, + 'punctuation': /%[%?]|[|:;\[\]<>]/, + }, }, - 'punctuation': /%[%?]|[|:;\[\]<>]/, }, }, - }); - - return bison; + }; }, } as LanguageProto<'bison'>; diff --git a/src/languages/c.ts b/src/languages/c.ts index dce414988a..0ed898af6c 100644 --- a/src/languages/c.ts +++ b/src/languages/c.ts @@ -1,22 +1,28 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; -import type { GrammarToken, LanguageProto } from '../types'; +import type { Grammar, GrammarToken, LanguageProto } from '../types'; export default { id: 'c', - require: clike, - optional: 'opencl-extensions', - grammar ({ extend, getOptionalLanguage }) { - const c = extend('clike', { - 'comment': { - pattern: /\/\/(?:[^\r\n\\]|\\(?:\r\n?|\n|(?![\r\n])))*|\/\*[\s\S]*?(?:\*\/|$)/, - greedy: true, - }, - 'string': { - // https://en.cppreference.com/w/c/language/string_literal - pattern: /"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"/, - greedy: true, - }, + base: clike, + grammar (): Grammar { + const string: GrammarToken = { + // https://en.cppreference.com/w/c/language/string_literal + pattern: /"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"/, + greedy: true, + }; + const comment: GrammarToken = { + pattern: /\/\/(?:[^\r\n\\]|\\(?:\r\n?|\n|(?![\r\n])))*|\/\*[\s\S]*?(?:\*\/|$)/, + greedy: true, + }; + const char: GrammarToken = { + // https://en.cppreference.com/w/c/language/character_constant + pattern: /'(?:\\(?:\r\n|[\s\S])|[^'\\\r\n]){0,32}'/, + greedy: true, + }; + + return { + comment, + string, 'class-name': { pattern: /(\b(?:enum|struct)\s+(?:__attribute__\s*\(\([\s\S]*?\)\)\s*)?)\w+|\b[a-z]\w*_t\b/, @@ -28,78 +34,61 @@ export default { 'number': /(?:\b0x(?:[\da-f]+(?:\.[\da-f]*)?|\.[\da-f]+)(?:p[+-]?\d+)?|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?)[ful]{0,4}/i, 'operator': />>=?|<<=?|->|([-+&|:])\1|[?:~]|[-+*/%&|^!=<>]=?/, - }); - - insertBefore(c, 'string', { - 'char': { - // https://en.cppreference.com/w/c/language/character_constant - pattern: /'(?:\\(?:\r\n|[\s\S])|[^'\\\r\n]){0,32}'/, - greedy: true, - }, - }); - - insertBefore(c, 'string', { - 'macro': { - // allow for multiline macro definitions - // spaces after the # character compile fine with gcc - pattern: - /(^[\t ]*)#\s*[a-z](?:[^\r\n\\/]|\/(?!\*)|\/\*(?:[^*]|\*(?!\/))*\*\/|\\(?:\r\n|[\s\S]))*/im, - lookbehind: true, - greedy: true, - alias: 'property', - inside: { - 'string': [ - { - // highlight the path of the include statement as a string - pattern: /^(#\s*include\s*)<[^>]+>/, - lookbehind: true, - }, - c['string'] as GrammarToken, - ], - 'char': c['char'], - 'comment': c['comment'], - 'macro-name': [ - { - pattern: /(^#\s*define\s+)\w+\b(?!\()/i, - lookbehind: true, - }, - { - pattern: /(^#\s*define\s+)\w+\b(?=\()/i, - lookbehind: true, - alias: 'function', - }, - ], - // highlight macro directives as keywords - 'directive': { - pattern: /^(#\s*)[a-z]+/, + $insertBefore: { + 'string': { + char, + 'macro': { + // allow for multiline macro definitions + // spaces after the # character compile fine with gcc + pattern: + /(^[\t ]*)#\s*[a-z](?:[^\r\n\\/]|\/(?!\*)|\/\*(?:[^*]|\*(?!\/))*\*\/|\\(?:\r\n|[\s\S]))*/im, lookbehind: true, - alias: 'keyword', - }, - 'directive-hash': /^#/, - 'punctuation': /##|\\(?=[\r\n])/, - 'expression': { - pattern: /\S[\s\S]*/, - inside: c, + greedy: true, + alias: 'property', + inside: { + 'string': [ + { + // highlight the path of the include statement as a string + pattern: /^(#\s*include\s*)<[^>]+>/, + lookbehind: true, + }, + string, + ], + 'char': char, + 'comment': comment, + 'macro-name': [ + { + pattern: /(^#\s*define\s+)\w+\b(?!\()/i, + lookbehind: true, + }, + { + pattern: /(^#\s*define\s+)\w+\b(?=\()/i, + lookbehind: true, + alias: 'function', + }, + ], + // highlight macro directives as keywords + 'directive': { + pattern: /^(#\s*)[a-z]+/, + lookbehind: true, + alias: 'keyword', + }, + 'directive-hash': /^#/, + 'punctuation': /##|\\(?=[\r\n])/, + 'expression': { + pattern: /\S[\s\S]*/, + inside: { $rest: 'c' }, + }, + }, }, }, + 'function': { + // highlight predefined macros as constants + 'constant': + /\b(?:EOF|NULL|SEEK_CUR|SEEK_END|SEEK_SET|__DATE__|__FILE__|__LINE__|__TIMESTAMP__|__TIME__|__func__|stderr|stdin|stdout)\b/, + }, }, - }); - - insertBefore(c, 'function', { - // highlight predefined macros as constants - 'constant': - /\b(?:EOF|NULL|SEEK_CUR|SEEK_END|SEEK_SET|__DATE__|__FILE__|__LINE__|__TIMESTAMP__|__TIME__|__func__|stderr|stdin|stdout)\b/, - }); - - delete c['boolean']; - - /* OpenCL host API */ - const extensions = getOptionalLanguage('opencl-extensions'); - if (extensions) { - insertBefore(c, 'keyword', extensions); - delete c['type-opencl-host-cpp']; - } - - return c; + $delete: ['boolean'], + } as unknown as Grammar; }, } as LanguageProto<'c'>; diff --git a/src/languages/cfscript.ts b/src/languages/cfscript.ts index cbc7f7c4af..8e0adf5e09 100644 --- a/src/languages/cfscript.ts +++ b/src/languages/cfscript.ts @@ -1,14 +1,13 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'cfscript', - require: clike, + base: clike, alias: 'cfc', - grammar ({ extend }) { + grammar (): Grammar { // https://cfdocs.org/script - const cfscript = extend('clike', { + return { 'comment': [ { pattern: /(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/, @@ -42,19 +41,17 @@ export default { /\b(?:any|array|binary|boolean|date|guid|numeric|query|string|struct|uuid|void|xml)\b/, alias: 'builtin', }, - }); - - insertBefore(cfscript, 'keyword', { - // This must be declared before keyword because we use "function" inside the lookahead - 'function-variable': { - pattern: - /[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*[=:]\s*(?:\bfunction\b|(?:\((?:[^()]|\([^()]*\))*\)|(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)\s*=>))/, - alias: 'function', + $insertBefore: { + 'keyword': { + // This must be declared before keyword because we use "function" inside the lookahead + 'function-variable': { + pattern: + /[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*[=:]\s*(?:\bfunction\b|(?:\((?:[^()]|\([^()]*\))*\)|(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)\s*=>))/, + alias: 'function', + }, + }, }, - }); - - delete cfscript['class-name']; - - return cfscript; + $delete: ['class-name'], + }; }, } as LanguageProto<'cfscript'>; diff --git a/src/languages/chaiscript.ts b/src/languages/chaiscript.ts index 448cd6ce9e..e78ab79571 100644 --- a/src/languages/chaiscript.ts +++ b/src/languages/chaiscript.ts @@ -1,16 +1,14 @@ import { toArray } from '../util/iterables'; -import { insertBefore } from '../util/language-util'; import clike from './clike'; import cpp from './cpp'; -import type { LanguageProto } from '../types'; +import type { Grammar, GrammarOptions, LanguageProto } from '../types'; export default { id: 'chaiscript', - require: [clike, cpp], - grammar ({ extend, getLanguage }) { - const cpp = getLanguage('cpp'); - - const chaiscript = extend('clike', { + require: cpp, + base: clike, + grammar ({ languages }: GrammarOptions): Grammar { + return { 'string': { pattern: /(^|[^\\])'(?:[^'\\]|\\[\s\S])*'/, lookbehind: true, @@ -30,47 +28,45 @@ export default { ], 'keyword': /\b(?:attr|auto|break|case|catch|class|continue|def|default|else|finally|for|fun|global|if|return|switch|this|try|var|while)\b/, - 'number': [...toArray(cpp.number), /\b(?:Infinity|NaN)\b/], + 'number': [...toArray(languages.cpp.number), /\b(?:Infinity|NaN)\b/], 'operator': />>=?|<<=?|\|\||&&|:[:=]?|--|\+\+|[=!<>+\-*/%|&^]=?|[?~]|`[^`\r\n]{1,4}`/, - }); - - insertBefore(chaiscript, 'operator', { - 'parameter-type': { - // e.g. def foo(int x, Vector y) {...} - pattern: /([,(]\s*)\w+(?=\s+\w)/, - lookbehind: true, - alias: 'class-name', - }, - }); - - insertBefore(chaiscript, 'string', { - 'string-interpolation': { - pattern: - /(^|[^\\])"(?:[^"$\\]|\\[\s\S]|\$(?!\{)|\$\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\})*"/, - lookbehind: true, - greedy: true, - inside: { - 'interpolation': { + $insertBefore: { + 'operator': { + 'parameter-type': { + // e.g. def foo(int x, Vector y) {...} + pattern: /([,(]\s*)\w+(?=\s+\w)/, + lookbehind: true, + alias: 'class-name', + }, + }, + 'string': { + 'string-interpolation': { pattern: - /((?:^|[^\\])(?:\\{2})*)\$\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\}/, + /(^|[^\\])"(?:[^"$\\]|\\[\s\S]|\$(?!\{)|\$\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\})*"/, lookbehind: true, + greedy: true, inside: { - 'interpolation-expression': { - pattern: /(^\$\{)[\s\S]+(?=\}$)/, + 'interpolation': { + pattern: + /((?:^|[^\\])(?:\\{2})*)\$\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\}/, lookbehind: true, - inside: 'chaiscript', - }, - 'interpolation-punctuation': { - pattern: /^\$\{|\}$/, - alias: 'punctuation', + inside: { + 'interpolation-expression': { + pattern: /(^\$\{)[\s\S]+(?=\}$)/, + lookbehind: true, + inside: 'chaiscript', + }, + 'interpolation-punctuation': { + pattern: /^\$\{|\}$/, + alias: 'punctuation', + }, + }, }, + 'string': /[\s\S]+/, }, }, - 'string': /[\s\S]+/, }, }, - }); - - return chaiscript; + }; }, } as LanguageProto<'chaiscript'>; diff --git a/src/languages/cilkc.ts b/src/languages/cilkc.ts index ad98416570..5399706b51 100644 --- a/src/languages/cilkc.ts +++ b/src/languages/cilkc.ts @@ -1,19 +1,21 @@ -import { insertBefore } from '../util/language-util'; import c from './c'; import type { LanguageProto } from '../types'; export default { id: 'cilkc', require: c, + base: c, alias: 'cilk-c', - grammar ({ extend }) { - const cilkc = extend('c', {}); - insertBefore(cilkc, 'function', { - 'parallel-keyword': { - pattern: /\bcilk_(?:for|reducer|s(?:cope|pawn|ync))\b/, - alias: 'keyword', + grammar () { + return { + $insertBefore: { + 'function': { + 'parallel-keyword': { + pattern: /\bcilk_(?:for|reducer|s(?:cope|pawn|ync))\b/, + alias: 'keyword', + }, + }, }, - }); - return cilkc; + }; }, } as LanguageProto<'cilkc'>; diff --git a/src/languages/cilkcpp.ts b/src/languages/cilkcpp.ts index 9683605604..a5170572b7 100644 --- a/src/languages/cilkcpp.ts +++ b/src/languages/cilkcpp.ts @@ -1,19 +1,20 @@ -import { insertBefore } from '../util/language-util'; import cpp from './cpp'; import type { LanguageProto } from '../types'; export default { id: 'cilkcpp', - require: cpp, + base: cpp, alias: ['cilk-cpp', 'cilk'], - grammar ({ extend }) { - const cilkcpp = extend('cpp', {}); - insertBefore(cilkcpp, 'function', { - 'parallel-keyword': { - pattern: /\bcilk_(?:for|reducer|s(?:cope|pawn|ync))\b/, - alias: 'keyword', + grammar () { + return { + $insertBefore: { + 'function': { + 'parallel-keyword': { + pattern: /\bcilk_(?:for|reducer|s(?:cope|pawn|ync))\b/, + alias: 'keyword', + }, + }, }, - }); - return cilkcpp; + }; }, } as LanguageProto<'cilkcpp'>; diff --git a/src/languages/coffeescript.ts b/src/languages/coffeescript.ts index 2742313105..027a01ab0d 100644 --- a/src/languages/coffeescript.ts +++ b/src/languages/coffeescript.ts @@ -1,12 +1,11 @@ -import { insertBefore } from '../util/language-util'; import javascript from './javascript'; -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'coffeescript', - require: javascript, + base: javascript, alias: 'coffee', - grammar ({ extend }) { + grammar (): Grammar { // Ignore comments starting with { to privilege string interpolation highlighting const comment = /#(?!\{).+/; const interpolation = { @@ -14,7 +13,7 @@ export default { alias: 'variable', }; - const coffeescript = extend('javascript', { + return { 'comment': comment, 'string': [ // Strings are multiline @@ -38,67 +37,58 @@ export default { pattern: /@(?!\d)\w+/, alias: 'variable', }, - }); - - insertBefore(coffeescript, 'comment', { - 'multiline-comment': { - pattern: /###[\s\S]+?###/, - alias: 'comment', - }, - - // Block regexp can contain comments and interpolation - 'block-regex': { - pattern: /\/{3}[\s\S]*?\/{3}/, - alias: 'regex', - inside: { - 'comment': comment, - 'interpolation': interpolation, - }, - }, - }); - - insertBefore(coffeescript, 'string', { - 'inline-javascript': { - pattern: /`(?:\\[\s\S]|[^\\`])*`/, - inside: { - 'delimiter': { - pattern: /^`|`$/, - alias: 'punctuation', + $insertBefore: { + 'comment': { + 'multiline-comment': { + pattern: /###[\s\S]+?###/, + alias: 'comment', }, - 'script': { - pattern: /[\s\S]+/, - alias: 'language-javascript', - inside: 'javascript', + + // Block regexp can contain comments and interpolation + 'block-regex': { + pattern: /\/{3}[\s\S]*?\/{3}/, + alias: 'regex', + inside: { + 'comment': comment, + 'interpolation': interpolation, + }, }, }, - }, + 'string': { + 'inline-javascript': { + pattern: /`(?:\\[\s\S]|[^\\`])*`/, + inside: { + 'delimiter': { + pattern: /^`|`$/, + alias: 'punctuation', + }, + 'script': 'javascript', + }, + }, - // Block strings - 'multiline-string': [ - { - pattern: /'''[\s\S]*?'''/, - greedy: true, - alias: 'string', + // Block strings + 'multiline-string': [ + { + pattern: /'''[\s\S]*?'''/, + greedy: true, + alias: 'string', + }, + { + pattern: /"""[\s\S]*?"""/, + greedy: true, + alias: 'string', + inside: { + 'interpolation': interpolation, + }, + }, + ], }, - { - pattern: /"""[\s\S]*?"""/, - greedy: true, - alias: 'string', - inside: { - 'interpolation': interpolation, - }, + 'keyword': { + // Object property + 'property': /(?!\d)\w+(?=\s*:(?!:))/, }, - ], - }); - - insertBefore(coffeescript, 'keyword', { - // Object property - 'property': /(?!\d)\w+(?=\s*:(?!:))/, - }); - - delete coffeescript['doc-comment']; - delete coffeescript['template-string']; - - return coffeescript; + }, + $delete: ['doc-comment', 'template-string'], + }; }, } as LanguageProto<'coffeescript'>; diff --git a/src/languages/cpp.ts b/src/languages/cpp.ts index 0540ae783c..73ba9ed88e 100644 --- a/src/languages/cpp.ts +++ b/src/languages/cpp.ts @@ -1,12 +1,11 @@ -import { insertBefore } from '../util/language-util'; import c from './c'; -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'cpp', - require: c, - optional: 'opencl-extensions', - grammar ({ extend, getOptionalLanguage }) { + base: c, + alias: 'c++', + grammar (): Grammar { const keyword = /\b(?:alignas|alignof|asm|auto|bool|break|case|catch|char|char16_t|char32_t|char8_t|class|co_await|co_return|co_yield|compl|concept|const|const_cast|consteval|constexpr|constinit|continue|decltype|default|delete|do|double|dynamic_cast|else|enum|explicit|export|extern|final|float|for|friend|goto|if|import|inline|int|int16_t|int32_t|int64_t|int8_t|long|module|mutable|namespace|new|noexcept|nullptr|operator|override|private|protected|public|register|reinterpret_cast|requires|return|short|signed|sizeof|static|static_assert|static_cast|struct|switch|template|this|thread_local|throw|try|typedef|typeid|typename|uint16_t|uint32_t|uint64_t|uint8_t|union|unsigned|using|virtual|void|volatile|wchar_t|while)\b/; const modName = /\b(?!)\w+(?:\s*\.\s*\w+)*\b/.source.replace( @@ -14,7 +13,7 @@ export default { () => keyword.source ); - const cpp = extend('c', { + return { 'class-name': [ { pattern: RegExp( @@ -46,84 +45,72 @@ export default { 'operator': />>=?|<<=?|->|--|\+\+|&&|\|\||[?:~]|<=>|[-+*/%&|^!=<>]=?|\b(?:and|and_eq|bitand|bitor|not|not_eq|or|or_eq|xor|xor_eq)\b/, 'boolean': /\b(?:false|true)\b/, - }); - - insertBefore(cpp, 'string', { - 'module': { - // https://en.cppreference.com/w/cpp/language/modules - pattern: RegExp( - /(\b(?:import|module)\s+)/.source + - '(?:' + - // header-name - /"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"|<[^<>\r\n]*>/.source + - '|' + - // module name or partition or both - /(?:\s*:\s*)?|:\s*/.source.replace( - //g, - () => modName - ) + - ')' - ), - lookbehind: true, - greedy: true, - inside: { - 'string': /^[<"][\s\S]+/, - 'operator': /:/, - 'punctuation': /\./, + $insertBefore: { + 'string': { + 'module': { + // https://en.cppreference.com/w/cpp/language/modules + pattern: RegExp( + /(\b(?:import|module)\s+)/.source + + '(?:' + + // header-name + /"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"|<[^<>\r\n]*>/.source + + '|' + + // module name or partition or both + /(?:\s*:\s*)?|:\s*/.source.replace( + //g, + () => modName + ) + + ')' + ), + lookbehind: true, + greedy: true, + inside: { + 'string': /^[<"][\s\S]+/, + 'operator': /:/, + 'punctuation': /\./, + }, + }, + 'raw-string': { + pattern: /R"([^()\\ ]{0,16})\([\s\S]*?\)\1"/, + alias: 'string', + greedy: true, + }, }, - }, - 'raw-string': { - pattern: /R"([^()\\ ]{0,16})\([\s\S]*?\)\1"/, - alias: 'string', - greedy: true, - }, - }); - - insertBefore(cpp, 'keyword', { - 'generic-function': { - pattern: /\b(?!operator\b)[a-z_]\w*\s*<(?:[^<>]|<[^<>]*>)*>(?=\s*\()/i, - inside: { - 'function': /^\w+/, - 'generic': { - pattern: /<[\s\S]+/, - alias: 'class-name', - inside: 'cpp', + 'keyword': { + 'generic-function': { + pattern: /\b(?!operator\b)[a-z_]\w*\s*<(?:[^<>]|<[^<>]*>)*>(?=\s*\()/i, + inside: { + 'function': /^\w+/, + 'generic': { + pattern: /<[\s\S]+/, + alias: 'class-name', + inside: 'cpp', + }, + }, }, }, - }, - }); - - insertBefore(cpp, 'operator', { - 'double-colon': { - pattern: /::/, - alias: 'punctuation', - }, - }); - - /* OpenCL host API */ - const extensions = getOptionalLanguage('opencl-extensions'); - if (extensions) { - insertBefore(cpp, 'keyword', extensions); - } - - const baseInside = { ...cpp }; - insertBefore(baseInside, 'double-colon', { - // All untokenized words that are not namespaces should be class names - 'class-name': /\b[a-z_]\w*\b(?!\s*::)/i, - }); - - insertBefore(cpp, 'class-name', { - // the base clause is an optional list of parent classes - // https://en.cppreference.com/w/cpp/language/class - 'base-clause': { - pattern: - /(\b(?:class|struct)\s+\w+\s*:\s*)[^;{}"'\s]+(?:\s+[^;{}"'\s]+)*(?=\s*[;{])/, - lookbehind: true, - greedy: true, - inside: baseInside, - }, - }); - - return cpp; + 'operator': { + 'double-colon': { + pattern: /::/, + alias: 'punctuation', + }, + }, + 'double-colon': { + // All untokenized words that are not namespaces should be class names + 'class-name': /\b[a-z_]\w*\b(?!\s*::)/i, + }, + 'class-name': { + // the base clause is an optional list of parent classes + // https://en.cppreference.com/w/cpp/language/class + 'base-clause': { + pattern: + /(\b(?:class|struct)\s+\w+\s*:\s*)[^;{}"'\s]+(?:\s+[^;{}"'\s]+)*(?=\s*[;{])/, + lookbehind: true, + greedy: true, + inside: '$self', + }, + }, + }, // end $insertBefore + }; }, } as LanguageProto<'cpp'>; diff --git a/src/languages/crystal.ts b/src/languages/crystal.ts index 582fe95b61..4c7d9ecec4 100644 --- a/src/languages/crystal.ts +++ b/src/languages/crystal.ts @@ -1,14 +1,12 @@ import { toArray } from '../util/iterables'; -import { insertBefore } from '../util/language-util'; import ruby from './ruby'; import type { LanguageProto } from '../types'; export default { id: 'crystal', - require: ruby, - grammar ({ extend, getLanguage }) { - const ruby = getLanguage('ruby'); - const crystal = extend('ruby', { + base: ruby, + grammar () { + return { 'keyword': [ /\b(?:__DIR__|__END_LINE__|__FILE__|__LINE__|abstract|alias|annotation|as|asm|begin|break|case|class|def|do|else|elsif|end|ensure|enum|extend|for|fun|if|ifdef|include|instance_sizeof|lib|macro|module|next|of|out|pointerof|private|protected|ptr|require|rescue|return|select|self|sizeof|struct|super|then|type|typeof|undef|uninitialized|union|unless|until|when|while|with|yield)\b/, { @@ -20,47 +18,47 @@ export default { /\b(?:0b[01_]*[01]|0o[0-7_]*[0-7]|0x[\da-fA-F_]*[\da-fA-F]|(?:\d(?:[\d_]*\d)?)(?:\.[\d_]*\d)?(?:[eE][+-]?[\d_]*\d)?)(?:_(?:[uif](?:8|16|32|64))?)?\b/, 'operator': [/->/, ...toArray(ruby.operator)], 'punctuation': /[(){}[\].,;\\]/, - }); - - insertBefore(crystal, 'string-literal', { - 'attribute': { - pattern: /@\[.*?\]/, - inside: { - 'delimiter': { - pattern: /^@\[|\]$/, - alias: 'punctuation', - }, + $insertBefore: { + 'string-literal': { 'attribute': { - pattern: /^(\s*)\w+/, - lookbehind: true, - alias: 'class-name', + pattern: /@\[.*?\]/, + inside: { + 'delimiter': { + pattern: /^@\[|\]$/, + alias: 'punctuation', + }, + 'attribute': { + pattern: /^(\s*)\w+/, + lookbehind: true, + alias: 'class-name', + }, + 'args': { + pattern: /\S(?:[\s\S]*\S)?/, + inside: 'crystal', + }, + }, }, - 'args': { - pattern: /\S(?:[\s\S]*\S)?/, - inside: 'crystal', + 'expansion': { + pattern: /\{(?:\{.*?\}|%.*?%)\}/, + inside: { + 'content': { + pattern: /^(\{.)[\s\S]+(?=.\}$)/, + lookbehind: true, + inside: 'crystal', + }, + 'delimiter': { + pattern: /^\{[\{%]|[\}%]\}$/, + alias: 'operator', + }, + }, }, - }, - }, - 'expansion': { - pattern: /\{(?:\{.*?\}|%.*?%)\}/, - inside: { - 'content': { - pattern: /^(\{.)[\s\S]+(?=.\}$)/, - lookbehind: true, - inside: 'crystal', - }, - 'delimiter': { - pattern: /^\{[\{%]|[\}%]\}$/, - alias: 'operator', + 'char': { + pattern: + /'(?:[^\\\r\n]{1,2}|\\(?:.|u(?:[A-Fa-f0-9]{1,4}|\{[A-Fa-f0-9]{1,6}\})))'/, + greedy: true, }, }, }, - 'char': { - pattern: /'(?:[^\\\r\n]{1,2}|\\(?:.|u(?:[A-Fa-f0-9]{1,4}|\{[A-Fa-f0-9]{1,6}\})))'/, - greedy: true, - }, - }); - - return crystal; + }; }, } as LanguageProto<'crystal'>; diff --git a/src/languages/csharp.ts b/src/languages/csharp.ts index 90488e7fcf..e8234abd3d 100644 --- a/src/languages/csharp.ts +++ b/src/languages/csharp.ts @@ -1,4 +1,3 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; import type { LanguageProto } from '../types'; @@ -33,10 +32,9 @@ function nested (pattern: string, depthLog2: number) { export default { id: 'csharp', - require: clike, - optional: 'xml-doc', - alias: ['cs', 'dotnet'], - grammar ({ extend, getOptionalLanguage }) { + base: clike, + alias: ['c#', 'cs', 'dotnet'], + grammar ({ languages }) { // https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/keywords/ const keywordKinds = { // keywords which represent a return or variable type @@ -112,7 +110,46 @@ export default { const regularString = /"(?:\\.|[^\\"\r\n])*"/.source; const verbatimString = /@"(?:""|\\[\s\S]|[^\\"])*"(?!")/.source; - const csharp = extend('clike', { + // attributes + const regularStringOrCharacter = regularString + '|' + character; + const regularStringCharacterOrComment = replace( + /\/(?![*/])|\/\/[^\r\n]*[\r\n]|\/\*(?:[^*]|\*(?!\/))*\*\/|<<0>>/.source, + [regularStringOrCharacter] + ); + const roundExpression = nested( + replace(/[^"'/()]|<<0>>|\(<>*\)/.source, [regularStringCharacterOrComment]), + 2 + ); + + // https://docs.microsoft.com/en-us/dotnet/csharp/programming-guide/concepts/attributes/#attribute-targets + const attrTarget = /\b(?:assembly|event|field|method|module|param|property|return|type)\b/ + .source; + const attr = replace(/<<0>>(?:\s*\(<<1>>*\))?/.source, [identifier, roundExpression]); + + // string interpolation + const formatString = /:[^}\r\n]+/.source; + // multi line + const mInterpolationRound = nested( + replace(/[^"'/()]|<<0>>|\(<>*\)/.source, [regularStringCharacterOrComment]), + 2 + ); + const mInterpolation = replace(/\{(?!\{)(?:(?![}:])<<0>>)*<<1>>?\}/.source, [ + mInterpolationRound, + formatString, + ]); + // single line + const sInterpolationRound = nested( + replace(/[^"'/()]|\/(?!\*)|\/\*(?:[^*]|\*(?!\/))*\*\/|<<0>>|\(<>*\)/.source, [ + regularStringOrCharacter, + ]), + 2 + ); + const sInterpolation = replace(/\{(?!\{)(?:(?![}:])<<0>>)*<<1>>?\}/.source, [ + sInterpolationRound, + formatString, + ]); + + const csharp = { 'string': [ { pattern: re(/(^|[^$\\])<<0>>/.source, [verbatimString]), @@ -198,204 +235,205 @@ export default { /(?:\b0(?:x[\da-f_]*[\da-f]|b[01_]*[01])|(?:\B\.\d+(?:_+\d+)*|\b\d+(?:_+\d+)*(?:\.\d+(?:_+\d+)*)?)(?:e[-+]?\d+(?:_+\d+)*)?)(?:[dflmu]|lu|ul)?\b/i, 'operator': />>=?|<<=?|[-=]>|([-+&|])\1|~|\?\?=?|[-+*/%&|^!=<>]=?/, 'punctuation': /\?\.?|::|[{}[\];(),.:]/, - }); - - insertBefore(csharp, 'number', { - 'range': { - pattern: /\.\./, - alias: 'operator', - }, - }); - - insertBefore(csharp, 'punctuation', { - 'named-parameter': { - pattern: re(/([(,]\s*)<<0>>(?=\s*:)/.source, [name]), - lookbehind: true, - alias: 'punctuation', - }, - }); - - insertBefore(csharp, 'class-name', { - 'namespace': { - // namespace Foo.Bar {} - // using Foo.Bar; - pattern: re(/(\b(?:namespace|using)\s+)<<0>>(?:\s*\.\s*<<0>>)*(?=\s*[;{])/.source, [ - name, - ]), - lookbehind: true, - inside: { - 'punctuation': /\./, + $insertBefore: { + 'string': { + 'interpolation-string': [ + { + pattern: re( + /(^|[^\\])(?:\$@|@\$)"(?:""|\\[\s\S]|\{\{|<<0>>|[^\\{"])*"/.source, + [mInterpolation] + ), + lookbehind: true, + greedy: true, + inside: createInterpolationInside(mInterpolation, mInterpolationRound), + }, + { + pattern: re(/(^|[^@\\])\$"(?:\\.|\{\{|<<0>>|[^\\"{])*"/.source, [ + sInterpolation, + ]), + lookbehind: true, + greedy: true, + inside: createInterpolationInside(sInterpolation, sInterpolationRound), + }, + ], + 'char': { + pattern: RegExp(character), + greedy: true, + }, + }, + 'comment': { + 'doc-comment': { + pattern: /\/\/\/.*/, + greedy: true, + alias: 'comment', + inside: { + // TODO use helper + get 'tag'() { + if (languages.markup) { + delete this['tag']; + return (this['tag'] = languages.markup['tag']); + } + return undefined; + }, + }, + }, + }, + 'number': { + 'range': { + pattern: /\.\./, + alias: 'operator', + }, + }, + 'punctuation': { + 'named-parameter': { + pattern: re(/([(,]\s*)<<0>>(?=\s*:)/.source, [name]), + lookbehind: true, + alias: 'punctuation', + }, }, - }, - 'type-expression': { - // default(Foo), typeof(Foo), sizeof(int) - pattern: re( - /(\b(?:default|sizeof|typeof)\s*\(\s*(?!\s))(?:[^()\s]|\s(?!\s)|<<0>>)*(?=\s*\))/ - .source, - [nestedRound] - ), - lookbehind: true, - alias: 'class-name', - inside: typeInside, - }, - 'return-type': { - // Foo ForBar(); Foo IFoo.Bar() => 0 - // int this[int index] => 0; T IReadOnlyList.this[int index] => this[index]; - // int Foo => 0; int Foo { get; set } = 0; - pattern: re(/<<0>>(?=\s+(?:<<1>>\s*(?:=>|[({]|\.\s*this\s*\[)|this\s*\[))/.source, [ - typeExpression, - identifier, - ]), - inside: typeInside, - alias: 'class-name', - }, - 'constructor-invocation': { - // new List> { } - pattern: re(/(\bnew\s+)<<0>>(?=\s*[[({])/.source, [typeExpression]), - lookbehind: true, - inside: typeInside, - alias: 'class-name', - }, - /*'explicit-implementation': { + 'class-name': { + 'namespace': { + // namespace Foo.Bar {} + // using Foo.Bar; + pattern: re( + /(\b(?:namespace|using)\s+)<<0>>(?:\s*\.\s*<<0>>)*(?=\s*[;{])/.source, + [name] + ), + lookbehind: true, + inside: { + 'punctuation': /\./, + }, + }, + 'type-expression': { + // default(Foo), typeof(Foo), sizeof(int) + pattern: re( + /(\b(?:default|sizeof|typeof)\s*\(\s*(?!\s))(?:[^()\s]|\s(?!\s)|<<0>>)*(?=\s*\))/ + .source, + [nestedRound] + ), + lookbehind: true, + alias: 'class-name', + inside: typeInside, + }, + 'return-type': { + // Foo ForBar(); Foo IFoo.Bar() => 0 + // int this[int index] => 0; T IReadOnlyList.this[int index] => this[index]; + // int Foo => 0; int Foo { get; set } = 0; + pattern: re( + /<<0>>(?=\s+(?:<<1>>\s*(?:=>|[({]|\.\s*this\s*\[)|this\s*\[))/.source, + [typeExpression, identifier] + ), + inside: typeInside, + alias: 'class-name', + }, + 'constructor-invocation': { + // new List> { } + pattern: re(/(\bnew\s+)<<0>>(?=\s*[[({])/.source, [typeExpression]), + lookbehind: true, + inside: typeInside, + alias: 'class-name', + }, + /*'explicit-implementation': { // int IFoo.Bar => 0; void IFoo>.Foo(); pattern: replace(/\b<<0>>(?=\.<<1>>)/, className, methodOrPropertyDeclaration), inside: classNameInside, alias: 'class-name' },*/ - 'generic-method': { - // foo() - pattern: re(/<<0>>\s*<<1>>(?=\s*\()/.source, [name, generic]), - inside: { - 'function': re(/^<<0>>/.source, [name]), - 'generic': { - pattern: RegExp(generic), - alias: 'class-name', - inside: typeInside, + 'generic-method': { + // foo() + pattern: re(/<<0>>\s*<<1>>(?=\s*\()/.source, [name, generic]), + inside: { + 'function': re(/^<<0>>/.source, [name]), + 'generic': { + pattern: RegExp(generic), + alias: 'class-name', + inside: typeInside, + }, + }, }, - }, - }, - 'type-list': { - // The list of types inherited or of generic constraints - // class Foo : Bar, IList - // where F : Bar, IList - pattern: re( - /\b((?:<<0>>\s+<<1>>|record\s+<<1>>\s*<<5>>|where\s+<<2>>)\s*:\s*)(?:<<3>>|<<4>>|<<1>>\s*<<5>>|<<6>>)(?:\s*,\s*(?:<<3>>|<<4>>|<<6>>))*(?=\s*(?:where|[{;]|=>|$))/ - .source, - [ - typeDeclarationKeywords, - genericName, - name, - typeExpression, - keywords.source, - nestedRound, - /\bnew\s*\(\s*\)/.source, - ] - ), - lookbehind: true, - inside: { - 'record-arguments': { - pattern: re(/(^(?!new\s*\()<<0>>\s*)<<1>>/.source, [ - genericName, - nestedRound, - ]), + 'type-list': { + // The list of types inherited or of generic constraints + // class Foo : Bar, IList + // where F : Bar, IList + pattern: re( + /\b((?:<<0>>\s+<<1>>|record\s+<<1>>\s*<<5>>|where\s+<<2>>)\s*:\s*)(?:<<3>>|<<4>>|<<1>>\s*<<5>>|<<6>>)(?:\s*,\s*(?:<<3>>|<<4>>|<<6>>))*(?=\s*(?:where|[{;]|=>|$))/ + .source, + [ + typeDeclarationKeywords, + genericName, + name, + typeExpression, + keywords.source, + nestedRound, + /\bnew\s*\(\s*\)/.source, + ] + ), lookbehind: true, - greedy: true, - inside: 'csharp', - }, - 'keyword': keywords, - 'class-name': { - pattern: RegExp(typeExpression), - greedy: true, - inside: typeInside, + inside: { + 'record-arguments': { + pattern: re(/(^(?!new\s*\()<<0>>\s*)<<1>>/.source, [ + genericName, + nestedRound, + ]), + lookbehind: true, + greedy: true, + inside: 'csharp', + }, + 'keyword': keywords, + 'class-name': { + pattern: RegExp(typeExpression), + greedy: true, + inside: typeInside, + }, + 'punctuation': /[,()]/, + }, }, - 'punctuation': /[,()]/, - }, - }, - 'preprocessor': { - pattern: /(^[\t ]*)#.*/m, - lookbehind: true, - alias: 'property', - inside: { - // highlight preprocessor directives as keywords - 'directive': { - pattern: - /(#)\b(?:define|elif|else|endif|endregion|error|if|line|nullable|pragma|region|undef|warning)\b/, + 'preprocessor': { + pattern: /(^[\t ]*)#.*/m, lookbehind: true, - alias: 'keyword', + alias: 'property', + inside: { + // highlight preprocessor directives as keywords + 'directive': { + pattern: + /(#)\b(?:define|elif|else|endif|endregion|error|if|line|nullable|pragma|region|undef|warning)\b/, + lookbehind: true, + alias: 'keyword', + }, + }, }, - }, - }, - }); - - // attributes - const regularStringOrCharacter = regularString + '|' + character; - const regularStringCharacterOrComment = replace( - /\/(?![*/])|\/\/[^\r\n]*[\r\n]|\/\*(?:[^*]|\*(?!\/))*\*\/|<<0>>/.source, - [regularStringOrCharacter] - ); - const roundExpression = nested( - replace(/[^"'/()]|<<0>>|\(<>*\)/.source, [regularStringCharacterOrComment]), - 2 - ); - - // https://docs.microsoft.com/en-us/dotnet/csharp/programming-guide/concepts/attributes/#attribute-targets - const attrTarget = /\b(?:assembly|event|field|method|module|param|property|return|type)\b/ - .source; - const attr = replace(/<<0>>(?:\s*\(<<1>>*\))?/.source, [identifier, roundExpression]); - insertBefore(csharp, 'class-name', { - 'attribute': { - // Attributes - // [Foo], [Foo(1), Bar(2, Prop = "foo")], [return: Foo(1), Bar(2)], [assembly: Foo(Bar)] - pattern: re( - /((?:^|[^\s\w>)?])\s*\[\s*)(?:<<0>>\s*:\s*)?<<1>>(?:\s*,\s*<<1>>)*(?=\s*\])/ - .source, - [attrTarget, attr] - ), - lookbehind: true, - greedy: true, - inside: { - 'target': { - pattern: re(/^<<0>>(?=\s*:)/.source, [attrTarget]), - alias: 'keyword', - }, - 'attribute-arguments': { - pattern: re(/\(<<0>>*\)/.source, [roundExpression]), - inside: 'csharp', - }, - 'class-name': { - pattern: RegExp(identifier), + 'attribute': { + // Attributes + // [Foo], [Foo(1), Bar(2, Prop = "foo")], [return: Foo(1), Bar(2)], [assembly: Foo(Bar)] + pattern: re( + /((?:^|[^\s\w>)?])\s*\[\s*)(?:<<0>>\s*:\s*)?<<1>>(?:\s*,\s*<<1>>)*(?=\s*\])/ + .source, + [attrTarget, attr] + ), + lookbehind: true, + greedy: true, inside: { - 'punctuation': /\./, + 'target': { + pattern: re(/^<<0>>(?=\s*:)/.source, [attrTarget]), + alias: 'keyword', + }, + 'attribute-arguments': { + pattern: re(/\(<<0>>*\)/.source, [roundExpression]), + inside: 'csharp', + }, + 'class-name': { + pattern: RegExp(identifier), + inside: { + 'punctuation': /\./, + }, + }, + 'punctuation': /[:,]/, }, }, - 'punctuation': /[:,]/, }, - }, - }); - - // string interpolation - const formatString = /:[^}\r\n]+/.source; - // multi line - const mInterpolationRound = nested( - replace(/[^"'/()]|<<0>>|\(<>*\)/.source, [regularStringCharacterOrComment]), - 2 - ); - const mInterpolation = replace(/\{(?!\{)(?:(?![}:])<<0>>)*<<1>>?\}/.source, [ - mInterpolationRound, - formatString, - ]); - // single line - const sInterpolationRound = nested( - replace(/[^"'/()]|\/(?!\*)|\/\*(?:[^*]|\*(?!\/))*\*\/|<<0>>|\(<>*\)/.source, [ - regularStringOrCharacter, - ]), - 2 - ); - const sInterpolation = replace(/\{(?!\{)(?:(?![}:])<<0>>)*<<1>>?\}/.source, [ - sInterpolationRound, - formatString, - ]); + }, // end of $insertBefore + }; function createInterpolationInside (interpolation: string, interpolationRound: string) { return { @@ -425,36 +463,6 @@ export default { }; } - insertBefore(csharp, 'string', { - 'interpolation-string': [ - { - pattern: re( - /(^|[^\\])(?:\$@|@\$)"(?:""|\\[\s\S]|\{\{|<<0>>|[^\\{"])*"/.source, - [mInterpolation] - ), - lookbehind: true, - greedy: true, - inside: createInterpolationInside(mInterpolation, mInterpolationRound), - }, - { - pattern: re(/(^|[^@\\])\$"(?:\\.|\{\{|<<0>>|[^\\"{])*"/.source, [ - sInterpolation, - ]), - lookbehind: true, - greedy: true, - inside: createInterpolationInside(sInterpolation, sInterpolationRound), - }, - ], - 'char': { - pattern: RegExp(character), - greedy: true, - }, - }); - - insertBefore(csharp, 'comment', { - 'doc-comment': getOptionalLanguage('xml-doc')?.slash, - }); - return csharp; }, -} as LanguageProto<'csharp'>; +} satisfies LanguageProto<'csharp'>; diff --git a/src/languages/cshtml.ts b/src/languages/cshtml.ts index 004cc79c10..f1400b8b01 100644 --- a/src/languages/cshtml.ts +++ b/src/languages/cshtml.ts @@ -1,4 +1,4 @@ -import { insertBefore } from '../util/language-util'; +import { insertBefore } from '../util/insert'; import csharp from './csharp'; import markup from './markup'; import type { Grammar, GrammarToken, LanguageProto } from '../types'; diff --git a/src/languages/css-extras.ts b/src/languages/css-extras.ts index 65afb8f761..8bc1142b42 100644 --- a/src/languages/css-extras.ts +++ b/src/languages/css-extras.ts @@ -1,9 +1,11 @@ +import css from './css'; import cssSelector from './css-selector'; import type { LanguageProto } from '../types'; export default { id: 'css-extras', require: cssSelector, + extends: css, grammar () { const unit = { pattern: /(\b\d+)(?:%|[a-z]+(?![\w-]))/, @@ -33,6 +35,7 @@ export default { lookbehind: true, }, { + // TODO update this for newer color functions pattern: /\b(?:hsl|rgb)\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*\)\B|\b(?:hsl|rgb)a\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*,\s*(?:0|0?\.\d+|1)\s*\)\B/i, inside: { diff --git a/src/languages/css.ts b/src/languages/css.ts index 995188919c..9c7184ffba 100644 --- a/src/languages/css.ts +++ b/src/languages/css.ts @@ -1,14 +1,14 @@ -import { insertBefore } from '../util/language-util'; import type { Grammar, LanguageProto } from '../types'; export default { id: 'css', - optional: 'css-extras', - grammar ({ getOptionalLanguage }) { + media: 'text/css', + extensions: ['css', 'postcss'], + grammar (): Grammar { const string = /(?:"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"|'(?:\\(?:\r\n|[\s\S])|[^'\\\r\n])*')/; - const css = { + return { 'comment': /\/\*[\s\S]*?\*\//, 'atrule': { pattern: RegExp( @@ -85,12 +85,5 @@ export default { }, 'punctuation': /[(){};:,]/, }; - - const extras = getOptionalLanguage('css-extras'); - if (extras) { - insertBefore(css, 'function', extras); - } - - return css; }, } as LanguageProto<'css'>; diff --git a/src/languages/d.ts b/src/languages/d.ts index 1589daad42..d3d1bcca56 100644 --- a/src/languages/d.ts +++ b/src/languages/d.ts @@ -1,12 +1,11 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'd', - require: clike, - grammar ({ extend }) { - const d = extend('clike', { + base: clike, + grammar (): Grammar { + return { 'comment': [ { // Shebang @@ -82,27 +81,24 @@ export default { 'operator': /\|[|=]?|&[&=]?|\+[+=]?|-[-=]?|\.?\.\.|=[>=]?|!(?:i[ns]\b|<>?=?|>=?|=)?|\bi[ns]\b|(?:<[<>]?|>>?>?|\^\^|[*\/%^~])=?/, - }); - - insertBefore(d, 'string', { - // Characters - // 'a', '\\', '\n', '\xFF', '\377', '\uFFFF', '\U0010FFFF', '\quot' - 'char': /'(?:\\(?:\W|\w+)|[^\\])'/, - }); - - insertBefore(d, 'keyword', { - 'property': /\B@\w*/, - }); - - insertBefore(d, 'function', { - 'register': { - // Iasm registers - pattern: - /\b(?:[ABCD][LHX]|E?(?:BP|DI|SI|SP)|[BS]PL|CR[0234]|[ECSDGF]S|[DS]IL|DR[012367]|E[ABCD]X|X?MM[0-7]|R(?:1[0-5]|[89])[BWD]?|R[ABCD]X|R[BS]P|R[DS]I|TR[3-7]|XMM(?:1[0-5]|[89])|YMM(?:1[0-5]|\d))\b|\bST(?:\([0-7]\)|\b)/, - alias: 'variable', + $insertBefore: { + 'string': { + // Characters + // 'a', '\\', '\n', '\xFF', '\377', '\uFFFF', '\U0010FFFF', '\quot' + 'char': /'(?:\\(?:\W|\w+)|[^\\])'/, + }, + 'keyword': { + 'property': /\B@\w*/, + }, + 'function': { + 'register': { + // Iasm registers + pattern: + /\b(?:[ABCD][LHX]|E?(?:BP|DI|SI|SP)|[BS]PL|CR[0234]|[ECSDGF]S|[DS]IL|DR[012367]|E[ABCD]X|X?MM[0-7]|R(?:1[0-5]|[89])[BWD]?|R[ABCD]X|R[BS]P|R[DS]I|TR[3-7]|XMM(?:1[0-5]|[89])|YMM(?:1[0-5]|\d))\b|\bST(?:\([0-7]\)|\b)/, + alias: 'variable', + }, + }, }, - }); - - return d; + }; }, } as LanguageProto<'d'>; diff --git a/src/languages/dart.ts b/src/languages/dart.ts index 055b65d549..0d1255706c 100644 --- a/src/languages/dart.ts +++ b/src/languages/dart.ts @@ -1,11 +1,10 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'dart', - require: clike, - grammar ({ extend }) { + base: clike, + grammar (): Grammar { const keywords = [ /\b(?:async|sync|yield)\*/, /\b(?:abstract|assert|async|await|break|case|catch|class|const|continue|covariant|default|deferred|do|dynamic|else|enum|export|extends|extension|external|factory|final|finally|for|get|hide|if|implements|import|in|interface|library|mixin|new|null|on|operator|part|rethrow|return|set|show|static|super|switch|sync|this|throw|try|typedef|var|void|while|with|yield)\b/, @@ -28,7 +27,7 @@ export default { }, }; - const dart = extend('clike', { + return { 'class-name': [ className, { @@ -42,49 +41,46 @@ export default { 'keyword': keywords, 'operator': /\bis!|\b(?:as|is)\b|\+\+|--|&&|\|\||<<=?|>>=?|~(?:\/=?)?|[+\-*\/%&^|=!<>]=?|\?/, - }); - - insertBefore(dart, 'string', { - 'string-literal': { - pattern: /r?(?:("""|''')[\s\S]*?\1|(["'])(?:\\.|(?!\2)[^\\\r\n])*\2(?!\2))/, - greedy: true, - inside: { - 'interpolation': { - pattern: /((?:^|[^\\])(?:\\{2})*)\$(?:\w+|\{(?:[^{}]|\{[^{}]*\})*\})/, - lookbehind: true, + $insertBefore: { + 'string': { + 'string-literal': { + pattern: /r?(?:("""|''')[\s\S]*?\1|(["'])(?:\\.|(?!\2)[^\\\r\n])*\2(?!\2))/, + greedy: true, inside: { - 'punctuation': /^\$\{?|\}$/, - 'expression': { - pattern: /[\s\S]+/, - inside: 'dart', + 'interpolation': { + pattern: + /((?:^|[^\\])(?:\\{2})*)\$(?:\w+|\{(?:[^{}]|\{[^{}]*\})*\})/, + lookbehind: true, + inside: { + 'punctuation': /^\$\{?|\}$/, + 'expression': { + pattern: /[\s\S]+/, + inside: 'dart', + }, + }, }, + 'string': /[\s\S]+/, }, }, - 'string': /[\s\S]+/, + 'string': undefined, }, - }, - 'string': undefined, - }); - - insertBefore(dart, 'class-name', { - 'metadata': { - pattern: /@\w+/, - alias: 'function', - }, - }); - - insertBefore(dart, 'class-name', { - 'generics': { - pattern: /<(?:[\w\s,.&?]|<(?:[\w\s,.&?]|<(?:[\w\s,.&?]|<[\w\s,.&?]*>)*>)*>)*>/, - inside: { - 'class-name': className, - 'keyword': keywords, - 'punctuation': /[<>(),.:]/, - 'operator': /[?&|]/, + 'class-name': { + 'metadata': { + pattern: /@\w+/, + alias: 'function', + }, + 'generics': { + pattern: + /<(?:[\w\s,.&?]|<(?:[\w\s,.&?]|<(?:[\w\s,.&?]|<[\w\s,.&?]*>)*>)*>)*>/, + inside: { + 'class-name': className, + 'keyword': keywords, + 'punctuation': /[<>(),.:]/, + 'operator': /[?&|]/, + }, + }, }, }, - }); - - return dart; + }; }, } as LanguageProto<'dart'>; diff --git a/src/languages/elixir.ts b/src/languages/elixir.ts index f8a9624198..b1e765b8f1 100644 --- a/src/languages/elixir.ts +++ b/src/languages/elixir.ts @@ -2,7 +2,7 @@ import type { Grammar, LanguageProto } from '../types'; export default { id: 'elixir', - grammar () { + grammar (): Grammar { const stringInside = { 'interpolation': { pattern: /#\{[^}]+\}/, diff --git a/src/languages/firestore-security-rules.ts b/src/languages/firestore-security-rules.ts index 5cbf10480a..f50aae2d90 100644 --- a/src/languages/firestore-security-rules.ts +++ b/src/languages/firestore-security-rules.ts @@ -1,48 +1,45 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'firestore-security-rules', - require: clike, - grammar ({ extend }) { - const fsr = extend('clike', { + base: clike, + grammar (): Grammar { + return { 'comment': /\/\/.*/, 'keyword': /\b(?:allow|function|if|match|null|return|rules_version|service)\b/, 'operator': /&&|\|\||[<>!=]=?|[-+*/%]|\b(?:in|is)\b/, - }); - - delete fsr['class-name']; - - insertBefore(fsr, 'keyword', { - 'path': { - pattern: - /(^|[\s(),])(?:\/(?:[\w\xA0-\uFFFF]+|\{[\w\xA0-\uFFFF]+(?:=\*\*)?\}|\$\([\w\xA0-\uFFFF.]+\)))+/, - lookbehind: true, - greedy: true, - inside: { - 'variable': { - pattern: /\{[\w\xA0-\uFFFF]+(?:=\*\*)?\}|\$\([\w\xA0-\uFFFF.]+\)/, + $insertBefore: { + 'keyword': { + 'path': { + pattern: + /(^|[\s(),])(?:\/(?:[\w\xA0-\uFFFF]+|\{[\w\xA0-\uFFFF]+(?:=\*\*)?\}|\$\([\w\xA0-\uFFFF.]+\)))+/, + lookbehind: true, + greedy: true, inside: { - 'operator': /=/, - 'keyword': /\*\*/, - 'punctuation': /[.$(){}]/, + 'variable': { + pattern: /\{[\w\xA0-\uFFFF]+(?:=\*\*)?\}|\$\([\w\xA0-\uFFFF.]+\)/, + inside: { + 'operator': /=/, + 'keyword': /\*\*/, + 'punctuation': /[.$(){}]/, + }, + }, + 'punctuation': /\//, + }, + }, + 'method': { + // to make the pattern shorter, the actual method names are omitted + pattern: /(\ballow\s+)[a-z]+(?:\s*,\s*[a-z]+)*(?=\s*[:;])/, + lookbehind: true, + alias: 'builtin', + inside: { + 'punctuation': /,/, }, }, - 'punctuation': /\//, - }, - }, - 'method': { - // to make the pattern shorter, the actual method names are omitted - pattern: /(\ballow\s+)[a-z]+(?:\s*,\s*[a-z]+)*(?=\s*[:;])/, - lookbehind: true, - alias: 'builtin', - inside: { - 'punctuation': /,/, }, }, - }); - - return fsr; + $delete: ['class-name'], + }; }, } as LanguageProto<'firestore-security-rules'>; diff --git a/src/languages/flow.ts b/src/languages/flow.ts index 3857301d5f..7576a07e82 100644 --- a/src/languages/flow.ts +++ b/src/languages/flow.ts @@ -1,14 +1,12 @@ import { toArray } from '../util/iterables'; -import { insertBefore } from '../util/language-util'; import javascript from './javascript'; -import type { GrammarToken, LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'flow', - require: javascript, - grammar ({ extend, getLanguage }) { - const javascript = getLanguage('javascript'); - const flow = extend('javascript', { + base: javascript, + grammar ({ base }) { + return { 'keyword': [ { pattern: /(^|[^$]\b)(?:Class|declare|opaque|type)\b(?!\$)/, @@ -19,31 +17,30 @@ export default { /(^|[^$]\B)\$(?:Diff|Enum|Exact|Keys|ObjMap|PropertyType|Record|Shape|Subtype|Supertype|await)\b(?!\$)/, lookbehind: true, }, - ...toArray(javascript['keyword']), + ...toArray(base!['keyword']), ], - }); - - insertBefore(flow, 'keyword', { - 'type': { - pattern: - /\b(?:[Bb]oolean|Function|[Nn]umber|[Ss]tring|[Ss]ymbol|any|mixed|null|void)\b/, - alias: 'class-name', + $insertBefore: { + 'operator': { + 'flow-punctuation': { + pattern: /\{\||\|\}/, + alias: 'punctuation', + }, + }, + 'keyword': { + 'type': { + pattern: + /\b(?:[Bb]oolean|Function|[Nn]umber|[Ss]tring|[Ss]ymbol|any|mixed|null|void)\b/, + alias: 'class-name', + }, + }, }, - }); - - insertBefore(flow, 'operator', { - 'flow-punctuation': { - pattern: /\{\||\|\}/, - alias: 'punctuation', + $merge: { + 'function-variable': { + pattern: + /(?!\s)[_$a-z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*=\s*(?:function\b|(?:\([^()]*\)(?:\s*:\s*\w+)?|(?!\s)[_$a-z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)\s*=>))/i, + }, }, - }); - - const fnVariable = flow['function-variable'] as GrammarToken; - fnVariable.pattern = - /(?!\s)[_$a-z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*=\s*(?:function\b|(?:\([^()]*\)(?:\s*:\s*\w+)?|(?!\s)[_$a-z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)\s*=>))/i; - - delete flow['parameter']; - - return flow; + $delete: ['parameter'], + } as unknown as Grammar; }, } as LanguageProto<'flow'>; diff --git a/src/languages/fsharp.ts b/src/languages/fsharp.ts index 228a086f17..4cec3341cd 100644 --- a/src/languages/fsharp.ts +++ b/src/languages/fsharp.ts @@ -1,13 +1,23 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; import type { LanguageProto } from '../types'; export default { id: 'fsharp', - require: clike, - optional: 'xml-doc', - grammar ({ extend, getOptionalLanguage }) { - const fsharp = extend('clike', { + alias: 'f#', + base: clike, + optional: 'markup', + grammar ({ languages }) { + return { + 'doc-comment': { + pattern: /\/\/\/.*/, + greedy: true, + alias: 'comment', + inside: { + get 'tag'() { + return languages.markup?.tag; + }, + }, + }, 'comment': [ { pattern: /(^|[^\\])\(\*(?!\))[\s\S]*?\*\)/, @@ -43,54 +53,50 @@ export default { ], 'operator': /([<>~&^])\1\1|([*.:<>&])\2|<-|->|[!=:]=|?|\??(?:<=|>=|<>|[-+*/%=<>])\??|[!?^&]|~[+~-]|:>|:\?>?/, - }); - insertBefore(fsharp, 'keyword', { - 'preprocessor': { - pattern: /(^[\t ]*)#.*/m, - lookbehind: true, - alias: 'property', - inside: { - 'directive': { - pattern: /(^#)\b(?:else|endif|if|light|line|nowarn)\b/, - lookbehind: true, - alias: 'keyword', + $insert: { + 'preprocessor': { + $before: 'keyword', + pattern: /(^[\t ]*)#.*/m, + lookbehind: true, + alias: 'property', + inside: { + 'directive': { + pattern: /(^#)\b(?:else|endif|if|light|line|nowarn)\b/, + lookbehind: true, + alias: 'keyword', + }, }, }, - }, - }); - insertBefore(fsharp, 'punctuation', { - 'computation-expression': { - pattern: /\b[_a-z]\w*(?=\s*\{)/i, - alias: 'keyword', - }, - }); - insertBefore(fsharp, 'string', { - 'annotation': { - pattern: /\[<.+?>\]/, - greedy: true, - inside: { - 'punctuation': /^\[<|>\]$/, - 'class-name': { - pattern: /^\w+$|(^|;\s*)[A-Z]\w*(?=\()/, - lookbehind: true, - }, - 'annotation-content': { - pattern: /[\s\S]+/, - inside: 'fsharp', + + 'computation-expression': { + $before: 'punctuation', + pattern: /\b[_a-z]\w*(?=\s*\{)/i, + alias: 'keyword', + }, + + 'annotation': { + $before: 'string', + pattern: /\[<.+?>\]/, + greedy: true, + inside: { + 'punctuation': /^\[<|>\]$/, + 'class-name': { + pattern: /^\w+$|(^|;\s*)[A-Z]\w*(?=\()/, + lookbehind: true, + }, + 'annotation-content': { + pattern: /[\s\S]+/, + inside: 'fsharp', + }, }, }, + 'char': { + $before: 'string', + pattern: + /'(?:[^\\']|\\(?:.|\d{3}|x[a-fA-F\d]{2}|u[a-fA-F\d]{4}|U[a-fA-F\d]{8}))'B?/, + greedy: true, + }, }, - 'char': { - pattern: - /'(?:[^\\']|\\(?:.|\d{3}|x[a-fA-F\d]{2}|u[a-fA-F\d]{4}|U[a-fA-F\d]{8}))'B?/, - greedy: true, - }, - }); - - insertBefore(fsharp, 'comment', { - 'doc-comment': getOptionalLanguage('xml-doc')?.slash, - }); - - return fsharp; + }; }, } as LanguageProto<'fsharp'>; diff --git a/src/languages/ftl.ts b/src/languages/ftl.ts index aa7472c620..e7b61a5299 100644 --- a/src/languages/ftl.ts +++ b/src/languages/ftl.ts @@ -5,7 +5,7 @@ import type { Grammar, LanguageProto } from '../types'; export default { id: 'ftl', require: markup, - grammar () { + grammar (): Grammar { // https://freemarker.apache.org/docs/dgui_template_exp.html // FTL expression with 4 levels of nesting supported @@ -96,7 +96,7 @@ export default { alias: 'ftl', inside: ftl, }, - }, + } as Grammar, }, 'ftl-interpolation': { pattern: RegExp( @@ -112,9 +112,9 @@ export default { alias: 'ftl', inside: ftl, }, - }, + } as Grammar, }, $tokenize: embeddedIn('markup') as Grammar['$tokenize'], - }; + } as Grammar; }, } as LanguageProto<'ftl'>; diff --git a/src/languages/glsl.ts b/src/languages/glsl.ts index af4756f19c..6e3d9c134e 100644 --- a/src/languages/glsl.ts +++ b/src/languages/glsl.ts @@ -3,11 +3,11 @@ import type { LanguageProto } from '../types'; export default { id: 'glsl', - require: c, - grammar ({ extend }) { - return extend('c', { + base: c, + grammar () { + return { 'keyword': /\b(?:active|asm|atomic_uint|attribute|bool|break|buffer|[ibdu]?vec[234]|case|cast|centroid|class|coherent|common|const|continue|default|discard|d?mat[234](?:x[234])?|do|double|else|enum|extern|external|false|filter|fixed|flat|float|for|fvec[234]|goto|half|highp|hvec[234]|if|[iu]?image[123]D|[iu]?image[12]DArray|[iu]?image2DMS(?:Array)?|[iu]?image2DRect|[iu]?imageBuffer|[iu]?imageCube|[iu]?imageCubeArray|in|inline|inout|input|int|interface|invariant|[iu]?sampler[123]D|[iu]?sampler[12]DArray|[iu]?sampler2DMS(?:Array)?|[iu]?sampler2DRect|[iu]?samplerBuffer|[iu]?samplerCube|[iu]?samplerCubeArray|layout|long|lowp|mediump|namespace|noinline|noperspective|out|output|partition|patch|precise|precision|public|readonly|resource|restrict|return|sample|sampler[12]DArrayShadow|sampler[12]DShadow|sampler2DRectShadow|sampler3DRect|samplerCubeArrayShadow|samplerCubeShadow|shared|short|sizeof|smooth|static|struct|subroutine|superp|switch|template|this|true|typedef|uint|uniform|union|unsigned|using|varying|void|volatile|while|writeonly)\b/, - }); + }; }, } as LanguageProto<'glsl'>; diff --git a/src/languages/gml.ts b/src/languages/gml.ts index d963353b76..2bd9d9bbd6 100644 --- a/src/languages/gml.ts +++ b/src/languages/gml.ts @@ -3,10 +3,10 @@ import type { LanguageProto } from '../types'; export default { id: 'gml', - require: clike, + base: clike, alias: 'gamemakerlanguage', - grammar ({ extend }) { - return extend('clike', { + grammar () { + return { 'keyword': /\b(?:break|case|continue|default|do|else|enum|exit|for|globalvar|if|repeat|return|switch|until|var|while)\b/, 'number': /(?:\b0x[\da-f]+|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?)[ulf]{0,4}/i, @@ -16,6 +16,6 @@ export default { /\b(?:GM_build_date|GM_version|action_(?:continue|restart|reverse|stop)|all|gamespeed_(?:fps|microseconds)|global|local|noone|other|pi|pointer_(?:invalid|null)|self|timezone_(?:local|utc)|undefined|ev_(?:alarm|animation_end|boundary|collision|create|destroy|draw|draw_(?:begin|end|post|pre)|end_of_path|game_end|game_start|global_(?:left|middle|right)_button|global_(?:left|middle|right)_press|global_(?:left|middle|right)_release|joystick(?:1|2)_(?:button1|button2|button3|button4|button5|button6|button7|button8|down|left|right|up)|keyboard|keypress|keyrelease|(?:left|middle|no|right)_button|(?:left|middle|right)_press|(?:left|middle|right)_release|mouse|mouse_(?:enter|leave|wheel_down|wheel_up)|no_more_health|no_more_lives|other|outside|room_end|room_start|step|trigger|user\d|gui|gui_begin|gui_end|step_(?:begin|end|normal))|vk_(?:alt|anykey|backspace|control|delete|down|end|enter|escape|home|insert|left|nokey|pagedown|pageup|pause|printscreen|return|right|shift|space|tab|up|f\d|numpad\d|add|decimal|divide|lalt|lcontrol|lshift|multiply|ralt|rcontrol|rshift|subtract)|achievement_(?:filter_(?:all_players|favorites_only|friends_only)|friends_info|info|leaderboard_info|our_info|pic_loaded|show_(?:achievement|bank|friend_picker|leaderboard|profile|purchase_prompt|ui)|type_challenge|type_score_challenge)|asset_(?:font|object|path|room|script|shader|sound|sprite|tiles|timeline|unknown)|audio_(?:3d|falloff_(?:exponent_distance|exponent_distance_clamped|inverse_distance|inverse_distance_clamped|linear_distance|linear_distance_clamped|none)|mono|new_system|old_system|stereo)|bm_(?:add|complex|dest_alpha|dest_color|dest_colour|inv_dest_alpha|inv_dest_color|inv_dest_colour|inv_src_alpha|inv_src_color|inv_src_colour|max|normal|one|src_alpha|src_alpha_sat|src_color|src_colour|subtract|zero)|browser_(?:chrome|firefox|ie|ie_mobile|not_a_browser|opera|safari|safari_mobile|tizen|unknown|windows_store)|buffer_(?:bool|f16|f32|f64|fast|fixed|generalerror|grow|invalidtype|network|outofbounds|outofspace|s16|s32|s8|seek_end|seek_relative|seek_start|string|text|u16|u32|u64|u8|vbuffer|wrap)|c_(?:aqua|black|blue|dkgray|fuchsia|gray|green|lime|ltgray|maroon|navy|olive|orange|purple|red|silver|teal|white|yellow)|cmpfunc_(?:always|equal|greater|greaterequal|less|lessequal|never|notequal)|cr_(?:appstart|arrow|beam|cross|default|drag|handpoint|hourglass|none|size_all|size_nesw|size_ns|size_nwse|size_we|uparrow)|cull_(?:clockwise|counterclockwise|noculling)|device_(?:emulator|tablet)|device_ios_(?:ipad|ipad_retina|iphone|iphone5|iphone6|iphone6plus|iphone_retina|unknown)|display_(?:landscape|landscape_flipped|portrait|portrait_flipped)|dll_(?:cdecl|cdel|stdcall)|ds_type_(?:grid|list|map|priority|queue|stack)|ef_(?:cloud|ellipse|explosion|firework|flare|rain|ring|smoke|smokeup|snow|spark|star)|fa_(?:archive|bottom|center|directory|hidden|left|middle|readonly|right|sysfile|top|volumeid)|fb_login_(?:default|fallback_to_webview|forcing_safari|forcing_webview|no_fallback_to_webview|use_system_account)|iap_(?:available|canceled|ev_consume|ev_product|ev_purchase|ev_restore|ev_storeload|failed|purchased|refunded|status_available|status_loading|status_processing|status_restoring|status_unavailable|status_uninitialised|storeload_failed|storeload_ok|unavailable)|leaderboard_type_(?:number|time_mins_secs)|lighttype_(?:dir|point)|matrix_(?:projection|view|world)|mb_(?:any|left|middle|none|right)|network_(?:config_(?:connect_timeout|disable_reliable_udp|enable_reliable_udp|use_non_blocking_socket)|socket_(?:bluetooth|tcp|udp)|type_(?:connect|data|disconnect|non_blocking_connect))|of_challenge_(?:lose|tie|win)|os_(?:android|ios|linux|macosx|ps3|ps4|psvita|unknown|uwp|win32|win8native|windows|winphone|xboxone)|phy_debug_render_(?:aabb|collision_pairs|coms|core_shapes|joints|obb|shapes)|phy_joint_(?:anchor_1_x|anchor_1_y|anchor_2_x|anchor_2_y|angle|angle_limits|damping_ratio|frequency|length_1|length_2|lower_angle_limit|max_force|max_length|max_motor_force|max_motor_torque|max_torque|motor_force|motor_speed|motor_torque|reaction_force_x|reaction_force_y|reaction_torque|speed|translation|upper_angle_limit)|phy_particle_data_flag_(?:category|color|colour|position|typeflags|velocity)|phy_particle_flag_(?:colormixing|colourmixing|elastic|powder|spring|tensile|viscous|wall|water|zombie)|phy_particle_group_flag_(?:rigid|solid)|pr_(?:linelist|linestrip|pointlist|trianglefan|trianglelist|trianglestrip)|ps_(?:distr|shape)_(?:diamond|ellipse|gaussian|invgaussian|line|linear|rectangle)|pt_shape_(?:circle|cloud|disk|explosion|flare|line|pixel|ring|smoke|snow|spark|sphere|square|star)|ty_(?:real|string)|gp_(?:face\d|axislh|axislv|axisrh|axisrv|padd|padl|padr|padu|select|shoulderl|shoulderlb|shoulderr|shoulderrb|start|stickl|stickr)|lb_disp_(?:none|numeric|time_ms|time_sec)|lb_sort_(?:ascending|descending|none)|ov_(?:achievements|community|friends|gamegroup|players|settings)|ugc_(?:filetype_(?:community|microtrans)|list_(?:Favorited|Followed|Published|Subscribed|UsedOrPlayed|VotedDown|VotedOn|VotedUp|WillVoteLater)|match_(?:AllGuides|Artwork|Collections|ControllerBindings|IntegratedGuides|Items|Items_Mtx|Items_ReadyToUse|Screenshots|UsableInGame|Videos|WebGuides)|query_(?:AcceptedForGameRankedByAcceptanceDate|CreatedByFriendsRankedByPublicationDate|FavoritedByFriendsRankedByPublicationDate|NotYetRated)|query_RankedBy(?:NumTimesReported|PublicationDate|TextSearch|TotalVotesAsc|Trend|Vote|VotesUp)|result_success|sortorder_CreationOrder(?:Asc|Desc)|sortorder_(?:ForModeration|LastUpdatedDesc|SubscriptionDateDesc|TitleAsc|VoteScoreDesc)|visibility_(?:friends_only|private|public))|vertex_usage_(?:binormal|blendindices|blendweight|color|colour|depth|fog|normal|position|psize|sample|tangent|texcoord|textcoord)|vertex_type_(?:float\d|color|colour|ubyte4)|input_type|layerelementtype_(?:background|instance|oldtilemap|particlesystem|sprite|tile|tilemap|undefined)|se_(?:chorus|compressor|echo|equalizer|flanger|gargle|none|reverb)|text_type|tile_(?:flip|index_mask|mirror|rotate)|(?:obj|rm|scr|spr)\w+)\b/, 'variable': /\b(?:alarm|application_surface|async_load|background_(?:alpha|blend|color|colour|foreground|height|hspeed|htiled|index|showcolor|showcolour|visible|vspeed|vtiled|width|x|xscale|y|yscale)|bbox_(?:bottom|left|right|top)|browser_(?:height|width)|caption_(?:health|lives|score)|current_(?:day|hour|minute|month|second|time|weekday|year)|cursor_sprite|debug_mode|delta_time|direction|display_aa|error_(?:last|occurred)|event_(?:action|number|object|type)|fps|fps_real|friction|game_(?:display|project|save)_(?:id|name)|gamemaker_(?:pro|registered|version)|gravity|gravity_direction|health|(?:h|v)speed|iap_data|id|image_(?:alpha|angle|blend|depth|index|number|speed|xscale|yscale)|instance_(?:count|id)|keyboard_(?:key|lastchar|lastkey|string)|layer|lives|mask_index|mouse_(?:button|lastbutton|x|y)|object_index|os_(?:browser|device|type|version)|path_(?:endaction|index|orientation|position|positionprevious|scale|speed)|persistent|phy_(?:active|angular_(?:damping|velocity)|bullet|(?:col_normal|collision|com|linear_velocity|position|speed)_(?:x|y)|collision_points|dynamic|fixed_rotation|inertia|kinematic|linear_damping|mass|position_(?:x|y)previous|rotation|sleeping|speed)|pointer_(?:invalid|null)|room|room_(?:caption|first|height|last|persistent|speed|width)|score|secure_mode|show_(?:health|lives|score)|solid|speed|sprite_(?:height|index|width|xoffset|yoffset)|temp_directory|timeline_(?:index|loop|position|running|speed)|transition_(?:color|kind|steps)|undefined|view_(?:angle|current|enabled|(?:h|v)(?:border|speed)|(?:h|w|x|y)port|(?:h|w|x|y)view|object|surface_id|visible)|visible|webgl_enabled|working_directory|x|(?:x|y)(?:previous|start)|y|argument(?:_relitive|_count|\d)|argument|global|local|other|self)\b/, - }); + }; }, } as LanguageProto<'gml'>; diff --git a/src/languages/go.ts b/src/languages/go.ts index c0674c39a1..f4421aa10e 100644 --- a/src/languages/go.ts +++ b/src/languages/go.ts @@ -1,12 +1,11 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'go', - require: clike, - grammar ({ extend }) { - const go = extend('clike', { + base: clike, + grammar () { + return { 'string': { pattern: /(^|[^\\])"(?:\\.|[^"\\\r\n])*"|`[^`]*`/, lookbehind: true, @@ -27,17 +26,14 @@ export default { /[*\/%^!=]=?|\+[=+]?|-[=-]?|\|[=|]?|&(?:=|&|\^=?)?|>(?:>=?|=)?|<(?:<=?|=|-)?|:=|\.\.\./, 'builtin': /\b(?:append|bool|byte|cap|close|complex|complex(?:64|128)|copy|delete|error|float(?:32|64)|imag|u?int(?:8|16|32|64)?|len|make|new|panic|print(?:ln)?|real|recover|rune|string|uintptr)\b/, - }); - - insertBefore(go, 'string', { - 'char': { - pattern: /'(?:\\.|[^'\\\r\n]){0,10}'/, - greedy: true, + $insert: { + 'char': { + $before: 'string', + pattern: /'(?:\\.|[^'\\\r\n]){0,10}'/, + greedy: true, + }, }, - }); - - delete go['class-name']; - - return go; + $delete: ['class-name'], + } as unknown as Grammar; }, } as LanguageProto<'go'>; diff --git a/src/languages/gradle.ts b/src/languages/gradle.ts index ff70c9c469..b40f739366 100644 --- a/src/languages/gradle.ts +++ b/src/languages/gradle.ts @@ -1,11 +1,10 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; import type { LanguageProto } from '../types'; export default { id: 'gradle', - require: clike, - grammar ({ extend }) { + base: clike, + grammar () { const interpolation = { pattern: /((?:^|[^\\$])(?:\\{2})*)\$(?:\w+|\{[^{}]*\})/, lookbehind: true, @@ -21,7 +20,7 @@ export default { }, }; - const gradle = extend('clike', { + return { 'string': { pattern: /'''(?:[^\\]|\\[\s\S])*?'''|'(?:\\.|[^\\'\r\n])*'/, greedy: true, @@ -36,37 +35,34 @@ export default { lookbehind: true, }, 'punctuation': /\.+|[{}[\];(),:$]/, - }); - - insertBefore(gradle, 'string', { - 'shebang': { - pattern: /#!.+/, - alias: 'comment', - greedy: true, - }, - 'interpolation-string': { - pattern: - /"""(?:[^\\]|\\[\s\S])*?"""|(["/])(?:\\.|(?!\1)[^\\\r\n])*\1|\$\/(?:[^/$]|\$(?:[/$]|(?![/$]))|\/(?!\$))*\/\$/, - greedy: true, - inside: { - 'interpolation': interpolation, - 'string': /[\s\S]+/, + $insertBefore: { + 'string': { + 'shebang': { + pattern: /#!.+/, + alias: 'comment', + greedy: true, + }, + 'interpolation-string': { + pattern: + /"""(?:[^\\]|\\[\s\S])*?"""|(["/])(?:\\.|(?!\1)[^\\\r\n])*\1|\$\/(?:[^/$]|\$(?:[/$]|(?![/$]))|\/(?!\$))*\/\$/, + greedy: true, + inside: { + 'interpolation': interpolation, + 'string': /[\s\S]+/, + }, + }, + }, + 'punctuation': { + 'spock-block': /\b(?:and|cleanup|expect|given|setup|then|when|where):/, + }, + 'function': { + 'annotation': { + pattern: /(^|[^.])@\w+/, + lookbehind: true, + alias: 'punctuation', + }, }, }, - }); - - insertBefore(gradle, 'punctuation', { - 'spock-block': /\b(?:and|cleanup|expect|given|setup|then|when|where):/, - }); - - insertBefore(gradle, 'function', { - 'annotation': { - pattern: /(^|[^.])@\w+/, - lookbehind: true, - alias: 'punctuation', - }, - }); - - return gradle; + }; }, } as LanguageProto<'gradle'>; diff --git a/src/languages/graphql.ts b/src/languages/graphql.ts index fbccc3f819..6bd6a66aa6 100644 --- a/src/languages/graphql.ts +++ b/src/languages/graphql.ts @@ -1,202 +1,206 @@ -import { withoutTokenize } from '../util/language-util'; +import { withoutTokenize } from '../util/without-tokenize'; import type { Token } from '../core'; -import type { Prism } from '../core/prism'; import type { Grammar, LanguageProto } from '../types'; export default { id: 'graphql', - grammar: { - 'comment': /#.*/, - 'description': { - pattern: /(?:"""(?:[^"]|(?!""")")*"""|"(?:\\.|[^\\"\r\n])*")(?=\s*[a-z_])/i, - greedy: true, - alias: 'string', - inside: { - 'language-markdown': { - pattern: /(^"(?:"")?)(?!\1)[\s\S]+(?=\1$)/, - lookbehind: true, - inside: 'markdown', + alias: 'gql', + optional: 'markdown', + grammar (): Grammar { + return { + 'comment': /#.*/, + 'description': { + pattern: /(?:"""(?:[^"]|(?!""")")*"""|"(?:\\.|[^\\"\r\n])*")(?=\s*[a-z_])/i, + greedy: true, + alias: 'string', + inside: { + // TODO we should be able to do this in one step + 'language-markdown': { + pattern: /(^"(?:"")?)(?!\1)[\s\S]+(?=\1$)/, + lookbehind: true, + inside: 'markdown', + }, }, }, - }, - 'string': { - pattern: /"""(?:[^"]|(?!""")")*"""|"(?:\\.|[^\\"\r\n])*"/, - greedy: true, - }, - 'number': /(?:\B-|\b)\d+(?:\.\d+)?(?:e[+-]?\d+)?\b/i, - 'boolean': /\b(?:false|true)\b/, - 'variable': /\$[a-z_]\w*/i, - 'directive': { - pattern: /@[a-z_]\w*/i, - alias: 'function', - }, - 'attr-name': { - pattern: /\b[a-z_]\w*(?=\s*(?:\((?:[^()"]|"(?:\\.|[^\\"\r\n])*")*\))?:)/i, - greedy: true, - }, - 'atom-input': { - pattern: /\b[A-Z]\w*Input\b/, - alias: 'class-name', - }, - 'scalar': /\b(?:Boolean|Float|ID|Int|String)\b/, - 'constant': /\b[A-Z][A-Z_\d]*\b/, - 'class-name': { - pattern: - /(\b(?:enum|implements|interface|on|scalar|type|union)\s+|&\s*|:\s*|\[)[A-Z_]\w*/, - lookbehind: true, - }, - 'fragment': { - pattern: /(\bfragment\s+|\.{3}\s*(?!on\b))[a-zA-Z_]\w*/, - lookbehind: true, - alias: 'function', - }, - 'definition-mutation': { - pattern: /(\bmutation\s+)[a-zA-Z_]\w*/, - lookbehind: true, - alias: 'function', - }, - 'definition-query': { - pattern: /(\bquery\s+)[a-zA-Z_]\w*/, - lookbehind: true, - alias: 'function', - }, - 'keyword': - /\b(?:directive|enum|extend|fragment|implements|input|interface|mutation|on|query|repeatable|scalar|schema|subscription|type|union)\b/, - 'operator': /[!=|&]|\.{3}/, - 'property-query': /\w+(?=\s*\()/, - 'object': /\w+(?=\s*\{)/, - 'punctuation': /[!(){}\[\]:=,]/, - 'property': /\w+/, - $tokenize (code: string, grammar: Grammar, Prism: Prism) { - const tokens = Prism.tokenize(code, withoutTokenize(grammar)); - - function isToken (token: Token | string): token is Token { - return typeof token !== 'string'; - } - - /** - * get the graphql token stream that we want to customize - */ - const validTokens = tokens - .filter(isToken) - .filter(token => token.type !== 'comment' && token.type !== 'scalar'); - - let currentIndex = 0; - - /** - * Returns whether the token relative to the current index has the given type. - */ - function getToken (offset = 0): Token { - return validTokens[currentIndex + offset]; - } - - /** - * Returns whether the token relative to the current index has the given type. - */ - function isTokenType (types: readonly string[], offset = 0): boolean { - for (let i = 0; i < types.length; i++) { - const token = getToken(i + offset); - if (!token || token.type !== types[i]) { - return false; - } + 'string': { + pattern: /"""(?:[^"]|(?!""")")*"""|"(?:\\.|[^\\"\r\n])*"/, + greedy: true, + }, + 'number': /(?:\B-|\b)\d+(?:\.\d+)?(?:e[+-]?\d+)?\b/i, + 'boolean': /\b(?:false|true)\b/, + 'variable': /\$[a-z_]\w*/i, + 'directive': { + pattern: /@[a-z_]\w*/i, + alias: 'function', + }, + 'attr-name': { + pattern: /\b[a-z_]\w*(?=\s*(?:\((?:[^()"]|"(?:\\.|[^\\"\r\n])*")*\))?:)/i, + greedy: true, + }, + 'atom-input': { + pattern: /\b[A-Z]\w*Input\b/, + alias: 'class-name', + }, + 'scalar': /\b(?:Boolean|Float|ID|Int|String)\b/, + 'constant': /\b[A-Z][A-Z_\d]*\b/, + 'class-name': { + pattern: + /(\b(?:enum|implements|interface|on|scalar|type|union)\s+|&\s*|:\s*|\[)[A-Z_]\w*/, + lookbehind: true, + }, + 'fragment': { + pattern: /(\bfragment\s+|\.{3}\s*(?!on\b))[a-zA-Z_]\w*/, + lookbehind: true, + alias: 'function', + }, + 'definition-mutation': { + pattern: /(\bmutation\s+)[a-zA-Z_]\w*/, + lookbehind: true, + alias: 'function', + }, + 'definition-query': { + pattern: /(\bquery\s+)[a-zA-Z_]\w*/, + lookbehind: true, + alias: 'function', + }, + 'keyword': + /\b(?:directive|enum|extend|fragment|implements|input|interface|mutation|on|query|repeatable|scalar|schema|subscription|type|union)\b/, + 'operator': /[!=|&]|\.{3}/, + 'property-query': /\w+(?=\s*\()/, + 'object': /\w+(?=\s*\{)/, + 'punctuation': /[!(){}\[\]:=,]/, + 'property': /\w+/, + $tokenize (code, grammar, Prism) { + const tokens = Prism.tokenize(code, withoutTokenize(grammar)); + + function isToken (token: Token | string): token is Token { + return typeof token !== 'string'; + } + + /** + * get the graphql token stream that we want to customize + */ + const validTokens = tokens + .filter(isToken) + .filter(token => token.type !== 'comment' && token.type !== 'scalar'); + + let currentIndex = 0; + + /** + * Returns whether the token relative to the current index has the given type. + */ + function getToken (offset = 0): Token { + return validTokens[currentIndex + offset]; } - return true; - } - - /** - * Returns the index of the closing bracket to an opening bracket. - * - * It is assumed that `token[currentIndex - 1]` is an opening bracket. - * - * If no closing bracket could be found, `-1` will be returned. - */ - function findClosingBracket (open: RegExp, close: RegExp): number { - let stackHeight = 1; - - for (let i = currentIndex; i < validTokens.length; i++) { - const token = validTokens[i]; - const content = token.content; - - if (token.type === 'punctuation' && typeof content === 'string') { - if (open.test(content)) { - stackHeight++; + + /** + * Returns whether the token relative to the current index has the given type. + */ + function isTokenType (types: readonly string[], offset = 0): boolean { + for (let i = 0; i < types.length; i++) { + const token = getToken(i + offset); + if (!token || token.type !== types[i]) { + return false; } - else if (close.test(content)) { - stackHeight--; + } + return true; + } - if (stackHeight === 0) { - return i; + /** + * Returns the index of the closing bracket to an opening bracket. + * + * It is assumed that `token[currentIndex - 1]` is an opening bracket. + * + * If no closing bracket could be found, `-1` will be returned. + */ + function findClosingBracket (open: RegExp, close: RegExp): number { + let stackHeight = 1; + + for (let i = currentIndex; i < validTokens.length; i++) { + const token = validTokens[i]; + const content = token.content; + + if (token.type === 'punctuation' && typeof content === 'string') { + if (open.test(content)) { + stackHeight++; + } + else if (close.test(content)) { + stackHeight--; + + if (stackHeight === 0) { + return i; + } } } } - } - return -1; - } + return -1; + } - for (; currentIndex < validTokens.length; ) { - const startToken = validTokens[currentIndex++]; + for (; currentIndex < validTokens.length; ) { + const startToken = validTokens[currentIndex++]; - // add special aliases for mutation tokens - if (startToken.type === 'keyword' && startToken.content === 'mutation') { - // any array of the names of all input variables (if any) - const inputVariables = []; + // add special aliases for mutation tokens + if (startToken.type === 'keyword' && startToken.content === 'mutation') { + // any array of the names of all input variables (if any) + const inputVariables = []; - if ( - isTokenType(['definition-mutation', 'punctuation']) && - getToken(1).content === '(' - ) { - // definition + if ( + isTokenType(['definition-mutation', 'punctuation']) && + getToken(1).content === '(' + ) { + // definition - currentIndex += 2; // skip 'definition-mutation' and 'punctuation' + currentIndex += 2; // skip 'definition-mutation' and 'punctuation' - const definitionEnd = findClosingBracket(/^\($/, /^\)$/); - if (definitionEnd === -1) { - continue; - } + const definitionEnd = findClosingBracket(/^\($/, /^\)$/); + if (definitionEnd === -1) { + continue; + } - // find all input variables - for (; currentIndex < definitionEnd; currentIndex++) { - const t = getToken(0); - if (t.type === 'variable') { - t.addAlias('variable-input'); - inputVariables.push(t.content); + // find all input variables + for (; currentIndex < definitionEnd; currentIndex++) { + const t = getToken(0); + if (t.type === 'variable') { + t.addAlias('variable-input'); + inputVariables.push(t.content); + } } - } - currentIndex = definitionEnd + 1; - } + currentIndex = definitionEnd + 1; + } - if ( - isTokenType(['punctuation', 'property-query']) && - getToken(0).content === '{' - ) { - currentIndex++; // skip opening bracket + if ( + isTokenType(['punctuation', 'property-query']) && + getToken(0).content === '{' + ) { + currentIndex++; // skip opening bracket - getToken(0).addAlias('property-mutation'); + getToken(0).addAlias('property-mutation'); - if (inputVariables.length > 0) { - const mutationEnd = findClosingBracket(/^\{$/, /^\}$/); - if (mutationEnd === -1) { - continue; - } + if (inputVariables.length > 0) { + const mutationEnd = findClosingBracket(/^\{$/, /^\}$/); + if (mutationEnd === -1) { + continue; + } - // give references to input variables a special alias - for (let i = currentIndex; i < mutationEnd; i++) { - const varToken = validTokens[i]; - if ( - varToken.type === 'variable' && - inputVariables.includes(varToken.content) - ) { - varToken.addAlias('variable-input'); + // give references to input variables a special alias + for (let i = currentIndex; i < mutationEnd; i++) { + const varToken = validTokens[i]; + if ( + varToken.type === 'variable' && + inputVariables.includes(varToken.content) + ) { + varToken.addAlias('variable-input'); + } } } } } } - } - return tokens; - }, - } as unknown as Grammar, -} as LanguageProto<'graphql'>; + return tokens; + }, + }; + }, +} satisfies LanguageProto<'graphql'>; diff --git a/src/languages/groovy.ts b/src/languages/groovy.ts index 4c2453e2ba..9442ef009e 100644 --- a/src/languages/groovy.ts +++ b/src/languages/groovy.ts @@ -1,11 +1,10 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; import type { LanguageProto } from '../types'; export default { id: 'groovy', - require: clike, - grammar ({ extend }) { + base: clike, + grammar () { const interpolation = { pattern: /((?:^|[^\\$])(?:\\{2})*)\$(?:\w+|\{[^{}]*\})/, lookbehind: true, @@ -21,7 +20,7 @@ export default { }, }; - const groovy = extend('clike', { + return { 'string': { // https://groovy-lang.org/syntax.html#_dollar_slashy_string pattern: /'''(?:[^\\]|\\[\s\S])*?'''|'(?:\\.|[^\\'\r\n])*'/, @@ -37,39 +36,36 @@ export default { lookbehind: true, }, 'punctuation': /\.+|[{}[\];(),:$]/, - }); - - insertBefore(groovy, 'string', { - 'shebang': { - pattern: /#!.+/, - alias: 'comment', - greedy: true, - }, - 'interpolation-string': { - // TODO: Slash strings (e.g. /foo/) can contain line breaks but this will cause a lot of trouble with - // simple division (see JS regex), so find a fix maybe? - pattern: - /"""(?:[^\\]|\\[\s\S])*?"""|(["/])(?:\\.|(?!\1)[^\\\r\n])*\1|\$\/(?:[^/$]|\$(?:[/$]|(?![/$]))|\/(?!\$))*\/\$/, - greedy: true, - inside: { - 'interpolation': interpolation, - 'string': /[\s\S]+/, + $insertBefore: { + 'string': { + 'shebang': { + pattern: /#!.+/, + alias: 'comment', + greedy: true, + }, + 'interpolation-string': { + // TODO: Slash strings (e.g. /foo/) can contain line breaks but this will cause a lot of trouble with + // simple division (see JS regex), so find a fix maybe? + pattern: + /"""(?:[^\\]|\\[\s\S])*?"""|(["/])(?:\\.|(?!\1)[^\\\r\n])*\1|\$\/(?:[^/$]|\$(?:[/$]|(?![/$]))|\/(?!\$))*\/\$/, + greedy: true, + inside: { + 'interpolation': interpolation, + 'string': /[\s\S]+/, + }, + }, + }, + 'punctuation': { + 'spock-block': /\b(?:and|cleanup|expect|given|setup|then|when|where):/, + }, + 'function': { + 'annotation': { + pattern: /(^|[^.])@\w+/, + lookbehind: true, + alias: 'punctuation', + }, }, }, - }); - - insertBefore(groovy, 'punctuation', { - 'spock-block': /\b(?:and|cleanup|expect|given|setup|then|when|where):/, - }); - - insertBefore(groovy, 'function', { - 'annotation': { - pattern: /(^|[^.])@\w+/, - lookbehind: true, - alias: 'punctuation', - }, - }); - - return groovy; + }; }, } as LanguageProto<'groovy'>; diff --git a/src/languages/haml.ts b/src/languages/haml.ts index bfd10df75a..1b17047b7a 100644 --- a/src/languages/haml.ts +++ b/src/languages/haml.ts @@ -1,4 +1,3 @@ -import { insertBefore } from '../util/language-util'; import ruby from './ruby'; import type { Grammar, LanguageProto } from '../types'; @@ -13,7 +12,42 @@ export default { code | */ - const haml = { + const filter_pattern = + '((?:^|\\r?\\n|\\r)([\\t ]*)):{{filter_name}}(?:(?:\\r?\\n|\\r)(?:\\2[\\t ].+|\\s*?(?=\\r?\\n|\\r)))+'; + + // Non exhaustive list of available filters and associated languages + const filters = [ + 'css', + { filter: 'coffee', language: 'coffeescript' }, + 'erb', + 'javascript', + 'less', + 'markdown', + 'ruby', + 'scss', + 'textile', + ]; + const all_filters: Grammar = {}; + for (const f of filters) { + const { filter, language } = typeof f === 'string' ? { filter: f, language: f } : f; + all_filters['filter-' + filter] = { + pattern: RegExp(filter_pattern.replace('{{filter_name}}', () => filter)), + lookbehind: true, + inside: { + 'filter-name': { + pattern: /^:[\w-]+/, + alias: 'symbol', + }, + 'text': { + pattern: /[\s\S]+/, + alias: [language, 'language-' + language], + inside: language, + }, + }, + }; + } + + return { // Multiline stuff should appear before the rest 'multiline-comment': { @@ -115,45 +149,9 @@ export default { pattern: /((?:^|\r?\n|\r)[\t ]*)[~=\-&!]+/, lookbehind: true, }, + $insertBefore: { + 'filter': all_filters, + }, }; - - const filter_pattern = - '((?:^|\\r?\\n|\\r)([\\t ]*)):{{filter_name}}(?:(?:\\r?\\n|\\r)(?:\\2[\\t ].+|\\s*?(?=\\r?\\n|\\r)))+'; - - // Non exhaustive list of available filters and associated languages - const filters = [ - 'css', - { filter: 'coffee', language: 'coffeescript' }, - 'erb', - 'javascript', - 'less', - 'markdown', - 'ruby', - 'scss', - 'textile', - ]; - const all_filters: Grammar = {}; - for (const f of filters) { - const { filter, language } = typeof f === 'string' ? { filter: f, language: f } : f; - all_filters['filter-' + filter] = { - pattern: RegExp(filter_pattern.replace('{{filter_name}}', () => filter)), - lookbehind: true, - inside: { - 'filter-name': { - pattern: /^:[\w-]+/, - alias: 'symbol', - }, - 'text': { - pattern: /[\s\S]+/, - alias: [language, 'language-' + language], - inside: language, - }, - }, - }; - } - - insertBefore(haml, 'filter', all_filters); - - return haml; }, } as LanguageProto<'haml'>; diff --git a/src/languages/haxe.ts b/src/languages/haxe.ts index bd70db3ede..f806b08d90 100644 --- a/src/languages/haxe.ts +++ b/src/languages/haxe.ts @@ -1,12 +1,11 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; import type { LanguageProto } from '../types'; export default { id: 'haxe', - require: clike, - grammar ({ extend }) { - const haxe = extend('clike', { + base: clike, + grammar () { + return { 'string': { // Strings can be multi-line pattern: /"(?:[^"\\]|\\[\s\S])*"/, @@ -29,64 +28,61 @@ export default { greedy: true, }, 'operator': /\.{3}|\+\+|--|&&|\|\||->|=>|(?:<{1,3}|[-+*/%!=&|^])=?|[?:~]/, - }); - - insertBefore(haxe, 'string', { - 'string-interpolation': { - pattern: /'(?:[^'\\]|\\[\s\S])*'/, - greedy: true, - inside: { - 'interpolation': { - pattern: /(^|[^\\])\$(?:\w+|\{[^{}]+\})/, - lookbehind: true, + $insertBefore: { + 'string': { + 'string-interpolation': { + pattern: /'(?:[^'\\]|\\[\s\S])*'/, + greedy: true, inside: { - 'interpolation-punctuation': { - pattern: /^\$\{?|\}$/, - alias: 'punctuation', + 'interpolation': { + pattern: /(^|[^\\])\$(?:\w+|\{[^{}]+\})/, + lookbehind: true, + inside: { + 'interpolation-punctuation': { + pattern: /^\$\{?|\}$/, + alias: 'punctuation', + }, + 'expression': { + pattern: /[\s\S]+/, + inside: 'haxe', + }, + }, }, - 'expression': { - pattern: /[\s\S]+/, - inside: 'haxe', + 'string': /[\s\S]+/, + }, + }, + }, + 'class-name': { + 'regex': { + pattern: /~\/(?:[^\/\\\r\n]|\\.)+\/[a-z]*/, + greedy: true, + inside: { + 'regex-flags': /\b[a-z]+$/, + 'regex-source': { + pattern: /^(~\/)[\s\S]+(?=\/$)/, + lookbehind: true, + alias: 'language-regex', + inside: 'regex', }, + 'regex-delimiter': /^~\/|\/$/, }, }, - 'string': /[\s\S]+/, }, - }, - }); - - insertBefore(haxe, 'class-name', { - 'regex': { - pattern: /~\/(?:[^\/\\\r\n]|\\.)+\/[a-z]*/, - greedy: true, - inside: { - 'regex-flags': /\b[a-z]+$/, - 'regex-source': { - pattern: /^(~\/)[\s\S]+(?=\/$)/, - lookbehind: true, - alias: 'language-regex', - inside: 'regex', + 'keyword': { + 'preprocessor': { + pattern: /#(?:else|elseif|end|if)\b.*/, + alias: 'property', + }, + 'metadata': { + pattern: /@:?[\w.]+/, + alias: 'symbol', + }, + 'reification': { + pattern: /\$(?:\w+|(?=\{))/, + alias: 'important', }, - 'regex-delimiter': /^~\/|\/$/, }, }, - }); - - insertBefore(haxe, 'keyword', { - 'preprocessor': { - pattern: /#(?:else|elseif|end|if)\b.*/, - alias: 'property', - }, - 'metadata': { - pattern: /@:?[\w.]+/, - alias: 'symbol', - }, - 'reification': { - pattern: /\$(?:\w+|(?=\{))/, - alias: 'important', - }, - }); - - return haxe; + }; }, } as LanguageProto<'haxe'>; diff --git a/src/languages/hlsl.ts b/src/languages/hlsl.ts index 924e9f0f17..865c963c71 100644 --- a/src/languages/hlsl.ts +++ b/src/languages/hlsl.ts @@ -4,16 +4,15 @@ import type { LanguageProto } from '../types'; export default { id: 'hlsl', - require: c, - grammar ({ extend, getLanguage }) { - const c = getLanguage('c'); - return extend('c', { + base: c, + grammar ({ base }) { + return { // Regarding keywords and class names: // The list of all keywords was split into 'keyword' and 'class-name' tokens based on whether they are capitalized. // https://docs.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl-appendix-keywords // https://docs.microsoft.com/en-us/windows/win32/direct3dhlsl/dx-graphics-hlsl-appendix-reserved-words 'class-name': [ - ...toArray(c['class-name']), + ...toArray(base!['class-name']), /\b(?:AppendStructuredBuffer|BlendState|Buffer|ByteAddressBuffer|CompileShader|ComputeShader|ConsumeStructuredBuffer|DepthStencilState|DepthStencilView|DomainShader|GeometryShader|Hullshader|InputPatch|LineStream|OutputPatch|PixelShader|PointStream|RWBuffer|RWByteAddressBuffer|RWStructuredBuffer|RWTexture(?:1D|1DArray|2D|2DArray|3D)|RasterizerState|RenderTargetView|SamplerComparisonState|SamplerState|StructuredBuffer|Texture(?:1D|1DArray|2D|2DArray|2DMS|2DMSArray|3D|Cube|CubeArray)|TriangleStream|VertexShader)\b/, ], 'keyword': [ @@ -26,6 +25,6 @@ export default { 'number': /(?:(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:[eE][+-]?\d+)?|\b0x[\da-fA-F]+)[fFhHlLuU]?\b/, 'boolean': /\b(?:false|true)\b/, - }); + }; }, } as LanguageProto<'hlsl'>; diff --git a/src/languages/http.ts b/src/languages/http.ts index 0feaef8f94..a3adae82ba 100644 --- a/src/languages/http.ts +++ b/src/languages/http.ts @@ -1,15 +1,68 @@ -import { insertBefore } from '../util/language-util'; import type { Grammar, LanguageProto } from '../types'; export default { id: 'http', optional: 'json', - grammar ({ getOptionalLanguage }) { + grammar ({ languages }) { function headerValueOf (name: string) { return RegExp('(^(?:' + name + '):[ \t]*(?![ \t]))[^]+', 'i'); } - const http = { + // Create a mapping of Content-Type headers to language definitions + // TODO use actual language metadata instead of hardcoding this + const httpLanguages = { + 'application/javascript': 'javascript', + 'application/json': languages.json || 'javascript', + 'application/xml': 'xml', + 'text/xml': 'xml', + 'text/html': 'html', + 'text/css': 'css', + 'text/plain': 'plain', + }; + + // Declare which types can also be suffixes + const suffixTypes: Partial> = { + 'application/json': true, + 'application/xml': true, + }; + + /** + * Returns a pattern for the given content type which matches it and any type which has it as a suffix. + */ + function getSuffixPattern (contentType: string) { + const suffix = contentType.replace(/^[a-z]+\//, ''); + const suffixPattern = '\\w+/(?:[\\w.-]+\\+)+' + suffix + '(?![+\\w.-])'; + return '(?:' + contentType + '|' + suffixPattern + ')'; + } + + // Insert each content type parser that has its associated language + // currently loaded. + const options: Grammar = {}; + for (const key in httpLanguages) { + const contentType = key as keyof typeof httpLanguages; + + const pattern = suffixTypes[contentType] ? getSuffixPattern(contentType) : contentType; + options[contentType.replace(/\//g, '-')] = { + pattern: RegExp( + '(' + + /content-type:\s*/.source + + pattern + + /(?:(?:\r\n?|\n)[\w-].*)*(?:\r(?:\n|(?!\n))|\n)/.source + + ')' + + // This is a little interesting: + // The HTTP format spec required 1 empty line before the body to make everything unambiguous. + // However, when writing code by hand (e.g. to display on a website) people can forget about this, + // so we want to be liberal here. We will allow the empty line to be omitted if the first line of + // the body does not start with a [\w-] character (as headers do). + /[^ \t\w-][\s\S]*/.source, + 'i' + ), + lookbehind: true, + inside: httpLanguages[contentType], + }; + } + + return { 'request-line': { pattern: /^(?:CONNECT|DELETE|GET|HEAD|OPTIONS|PATCH|POST|PRI|PUT|SEARCH|TRACE)\s(?:https?:\/\/|\/)\S*\sHTTP\/[\d.]+/m, @@ -90,62 +143,9 @@ export default { 'punctuation': /^:/, }, }, + $insertBefore: { + 'header': options, + }, }; - - // Create a mapping of Content-Type headers to language definitions - const httpLanguages = { - 'application/javascript': 'javascript', - 'application/json': getOptionalLanguage('json') || 'javascript', - 'application/xml': 'xml', - 'text/xml': 'xml', - 'text/html': 'html', - 'text/css': 'css', - 'text/plain': 'plain', - }; - - // Declare which types can also be suffixes - const suffixTypes: Partial> = { - 'application/json': true, - 'application/xml': true, - }; - - /** - * Returns a pattern for the given content type which matches it and any type which has it as a suffix. - */ - function getSuffixPattern (contentType: string) { - const suffix = contentType.replace(/^[a-z]+\//, ''); - const suffixPattern = '\\w+/(?:[\\w.-]+\\+)+' + suffix + '(?![+\\w.-])'; - return '(?:' + contentType + '|' + suffixPattern + ')'; - } - - // Insert each content type parser that has its associated language - // currently loaded. - const options: Grammar = {}; - for (const key in httpLanguages) { - const contentType = key as keyof typeof httpLanguages; - - const pattern = suffixTypes[contentType] ? getSuffixPattern(contentType) : contentType; - options[contentType.replace(/\//g, '-')] = { - pattern: RegExp( - '(' + - /content-type:\s*/.source + - pattern + - /(?:(?:\r\n?|\n)[\w-].*)*(?:\r(?:\n|(?!\n))|\n)/.source + - ')' + - // This is a little interesting: - // The HTTP format spec required 1 empty line before the body to make everything unambiguous. - // However, when writing code by hand (e.g. to display on a website) people can forget about this, - // so we want to be liberal here. We will allow the empty line to be omitted if the first line of - // the body does not start with a [\w-] character (as headers do). - /[^ \t\w-][\s\S]*/.source, - 'i' - ), - lookbehind: true, - inside: httpLanguages[contentType], - }; - } - insertBefore(http, 'header', options); - - return http; }, -} as LanguageProto<'http'>; +} satisfies LanguageProto<'http'>; diff --git a/src/languages/idris.ts b/src/languages/idris.ts index fd205a0436..287c2279f3 100644 --- a/src/languages/idris.ts +++ b/src/languages/idris.ts @@ -1,31 +1,28 @@ -import { insertBefore } from '../util/language-util'; import haskell from './haskell'; import type { LanguageProto } from '../types'; export default { id: 'idris', - require: haskell, + base: haskell, alias: 'idr', - grammar ({ extend }) { - const idris = extend('haskell', { + grammar () { + return { 'comment': { pattern: /(?:(?:--|\|\|\|).*$|\{-[\s\S]*?-\})/m, }, 'keyword': /\b(?:Type|case|class|codata|constructor|corecord|data|do|dsl|else|export|if|implementation|implicit|import|impossible|in|infix|infixl|infixr|instance|interface|let|module|mutual|namespace|of|parameters|partial|postulate|private|proof|public|quoteGoal|record|rewrite|syntax|then|total|using|where|with)\b/, 'builtin': undefined, - }); - - insertBefore(idris, 'keyword', { - 'import-statement': { - pattern: /(^\s*import\s+)(?:[A-Z][\w']*)(?:\.[A-Z][\w']*)*/m, - lookbehind: true, - inside: { - 'punctuation': /\./, + $insert: { + 'import-statement': { + $before: 'keyword', + pattern: /(^\s*import\s+)(?:[A-Z][\w']*)(?:\.[A-Z][\w']*)*/m, + lookbehind: true, + inside: { + 'punctuation': /\./, + }, }, }, - }); - - return idris; + }; }, } as LanguageProto<'idris'>; diff --git a/src/languages/java.ts b/src/languages/java.ts index 46098701c4..403cc47c87 100644 --- a/src/languages/java.ts +++ b/src/languages/java.ts @@ -1,12 +1,11 @@ import { toArray } from '../util/iterables'; -import { insertBefore } from '../util/language-util'; import clike from './clike'; -import type { LanguageProto } from '../types'; +import type { Grammar, GrammarOptions, LanguageProto } from '../types'; export default { id: 'java', - require: clike, - grammar ({ extend, getLanguage }) { + base: clike, + grammar ({ languages }: GrammarOptions): Grammar { const keywords = /\b(?:abstract|assert|boolean|break|byte|case|catch|char|class|const|continue|default|do|double|else|enum|exports|extends|final|finally|float|for|goto|if|implements|import|instanceof|int|interface|long|module|native|new|non-sealed|null|open|opens|package|permits|private|protected|provides|public|record(?!\s*[(){}[\]<>=%~.:,;?+\-*/&|^])|requires|return|sealed|short|static|strictfp|super|switch|synchronized|this|throw|throws|to|transient|transitive|try|uses|var|void|volatile|while|with|yield)\b/; @@ -30,9 +29,9 @@ export default { }, }; - const clike = getLanguage('clike'); + const {clike} = languages; - const java = extend('clike', { + return { 'string': { pattern: /(^|[^\\])"(?:\\.|[^"\\\r\n])*"/, lookbehind: true, @@ -79,90 +78,87 @@ export default { lookbehind: true, }, 'constant': /\b[A-Z][A-Z_\d]+\b/, - }); - - insertBefore(java, 'comment', { - 'doc-comment': { - pattern: /\/\*\*(?!\/)[\s\S]*?(?:\*\/|$)/, - greedy: true, - alias: 'comment', - inside: 'javadoc', - }, - }); - - insertBefore(java, 'string', { - 'triple-quoted-string': { - // http://openjdk.java.net/jeps/355#Description - pattern: /"""[ \t]*[\r\n](?:(?:"|"")?(?:\\.|[^"\\]))*"""/, - greedy: true, - alias: 'string', - }, - 'char': { - pattern: /'(?:\\.|[^'\\\r\n]){1,6}'/, - greedy: true, - }, - }); - - insertBefore(java, 'class-name', { - 'annotation': { - pattern: /(^|[^.])@\w+(?:\s*\.\s*\w+)*/, - lookbehind: true, - alias: 'punctuation', - }, - 'generics': { - pattern: - /<(?:[\w\s,.?]|&(?!&)|<(?:[\w\s,.?]|&(?!&)|<(?:[\w\s,.?]|&(?!&)|<(?:[\w\s,.?]|&(?!&))*>)*>)*>)*>/, - inside: { - 'class-name': className, - 'keyword': keywords, - 'punctuation': /[<>(),.:]/, - 'operator': /[?&|]/, - }, - }, - 'import': [ - { - pattern: RegExp( - /(\bimport\s+)/.source + classNamePrefix + /(?:[A-Z]\w*|\*)(?=\s*;)/.source - ), - lookbehind: true, - inside: { - 'namespace': className.inside.namespace, - 'punctuation': /\./, - 'operator': /\*/, - 'class-name': /\w+/, + $insertBefore: { + 'comment': { + 'doc-comment': { + pattern: /\/\*\*(?!\/)[\s\S]*?(?:\*\/|$)/, + greedy: true, + alias: 'comment', + inside: 'javadoc', }, }, - { - pattern: RegExp( - /(\bimport\s+static\s+)/.source + - classNamePrefix + - /(?:\w+|\*)(?=\s*;)/.source - ), - lookbehind: true, - alias: 'static', - inside: { - 'namespace': className.inside.namespace, - 'static': /\b\w+$/, - 'punctuation': /\./, - 'operator': /\*/, - 'class-name': /\w+/, + 'string': { + 'triple-quoted-string': { + // http://openjdk.java.net/jeps/355#Description + pattern: /"""[ \t]*[\r\n](?:(?:"|"")?(?:\\.|[^"\\]))*"""/, + greedy: true, + alias: 'string', + }, + 'char': { + pattern: /'(?:\\.|[^'\\\r\n]){1,6}'/, + greedy: true, }, }, - ], - 'namespace': { - pattern: RegExp( - /(\b(?:exports|import(?:\s+static)?|module|open|opens|package|provides|requires|to|transitive|uses|with)\s+)(?!)[a-z]\w*(?:\.[a-z]\w*)*\.?/.source.replace( - //g, - () => keywords.source - ) - ), - lookbehind: true, - inside: { - 'punctuation': /\./, - }, - }, - }); - - return java; + 'class-name': { + 'annotation': { + pattern: /(^|[^.])@\w+(?:\s*\.\s*\w+)*/, + lookbehind: true, + alias: 'punctuation', + }, + 'generics': { + pattern: + /<(?:[\w\s,.?]|&(?!&)|<(?:[\w\s,.?]|&(?!&)|<(?:[\w\s,.?]|&(?!&)|<(?:[\w\s,.?]|&(?!&))*>)*>)*>)*>/, + inside: { + 'class-name': className, + 'keyword': keywords, + 'punctuation': /[<>(),.:]/, + 'operator': /[?&|]/, + }, + }, + 'import': [ + { + pattern: RegExp( + /(\bimport\s+)/.source + classNamePrefix + /(?:[A-Z]\w*|\*)(?=\s*;)/.source + ), + lookbehind: true, + inside: { + 'namespace': className.inside.namespace, + 'punctuation': /\./, + 'operator': /\*/, + 'class-name': /\w+/, + }, + }, + { + pattern: RegExp( + /(\bimport\s+static\s+)/.source + + classNamePrefix + + /(?:\w+|\*)(?=\s*;)/.source + ), + lookbehind: true, + alias: 'static', + inside: { + 'namespace': className.inside.namespace, + 'static': /\b\w+$/, + 'punctuation': /\./, + 'operator': /\*/, + 'class-name': /\w+/, + }, + }, + ], + 'namespace': { + pattern: RegExp( + /(\b(?:exports|import(?:\s+static)?|module|open|opens|package|provides|requires|to|transitive|uses|with)\s+)(?!)[a-z]\w*(?:\.[a-z]\w*)*\.?/.source.replace( + //g, + () => keywords.source + ) + ), + lookbehind: true, + inside: { + 'punctuation': /\./, + }, + }, + } + } + }; }, } as LanguageProto<'java'>; diff --git a/src/languages/javadoc.ts b/src/languages/javadoc.ts index cf52461d84..0e4fc94ceb 100644 --- a/src/languages/javadoc.ts +++ b/src/languages/javadoc.ts @@ -1,4 +1,3 @@ -import { insertBefore } from '../util/language-util'; import java from './java'; import javadoclike from './javadoclike'; import markup from './markup'; @@ -6,10 +5,11 @@ import type { LanguageProto } from '../types'; export default { id: 'javadoc', - require: [markup, java, javadoclike], - grammar ({ extend, getLanguage }) { - const java = getLanguage('java'); - const { tag, entity } = getLanguage('markup'); + base: javadoclike, + require: [markup, java], + grammar ({ languages }) { + const java = languages.java; + const { tag, entity } = languages.markup; const codeLinePattern = /(^(?:[\t ]*(?:\*\s*)*))[^*\s].*$/m; @@ -19,85 +19,86 @@ export default { () => memberReference ); - const javadoc = extend('javadoclike', {}); - insertBefore(javadoc, 'keyword', { - 'reference': { - pattern: RegExp( - /(@(?:exception|link|linkplain|see|throws|value)\s+(?:\*\s*)?)/.source + - '(?:' + - reference + - ')' - ), - lookbehind: true, - inside: { - 'function': { - pattern: /(#\s*)\w+(?=\s*\()/, + return { + $insertBefore: { + 'keyword': { + 'reference': { + pattern: RegExp( + /(@(?:exception|link|linkplain|see|throws|value)\s+(?:\*\s*)?)/.source + + '(?:' + + reference + + ')' + ), lookbehind: true, - }, - 'field': { - pattern: /(#\s*)\w+/, - lookbehind: true, - }, - 'namespace': { - pattern: /\b(?:[a-z]\w*\s*\.\s*)+/, inside: { - 'punctuation': /\./, + 'function': { + pattern: /(#\s*)\w+(?=\s*\()/, + lookbehind: true, + }, + 'field': { + pattern: /(#\s*)\w+/, + lookbehind: true, + }, + 'namespace': { + pattern: /\b(?:[a-z]\w*\s*\.\s*)+/, + inside: { + 'punctuation': /\./, + }, + }, + 'class-name': /\b[A-Z]\w*/, + 'keyword': java.keyword, + 'punctuation': /[#()[\],.]/, }, }, - 'class-name': /\b[A-Z]\w*/, - 'keyword': java.keyword, - 'punctuation': /[#()[\],.]/, - }, - }, - 'class-name': { - // @param the first generic type parameter - pattern: /(@param\s+)<[A-Z]\w*>/, - lookbehind: true, - inside: { - 'punctuation': /[.<>]/, - }, - }, - 'code-section': [ - { - pattern: - /(\{@code\s+(?!\s))(?:[^\s{}]|\s+(?![\s}])|\{(?:[^{}]|\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\})*\})+(?=\s*\})/, - lookbehind: true, - inside: { - 'code': { - // there can't be any HTML inside of {@code} tags - pattern: codeLinePattern, - lookbehind: true, - inside: 'java', - alias: 'language-java', + 'class-name': { + // @param the first generic type parameter + pattern: /(@param\s+)<[A-Z]\w*>/, + lookbehind: true, + inside: { + 'punctuation': /[.<>]/, }, }, - }, - { - pattern: /(<(code|pre|tt)>(?!)\s*)\S(?:\S|\s+\S)*?(?=\s*<\/\2>)/, - lookbehind: true, - inside: { - 'line': { - pattern: codeLinePattern, + 'code-section': [ + { + pattern: + /(\{@code\s+(?!\s))(?:[^\s{}]|\s+(?![\s}])|\{(?:[^{}]|\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})*\})*\})+(?=\s*\})/, lookbehind: true, inside: { - // highlight HTML tags and entities - 'tag': tag, - 'entity': entity, 'code': { - // everything else is Java code - pattern: /.+/, + // there can't be any HTML inside of {@code} tags + pattern: codeLinePattern, + lookbehind: true, inside: 'java', alias: 'language-java', }, }, }, - }, + { + pattern: /(<(code|pre|tt)>(?!)\s*)\S(?:\S|\s+\S)*?(?=\s*<\/\2>)/, + lookbehind: true, + inside: { + 'line': { + pattern: codeLinePattern, + lookbehind: true, + inside: { + // highlight HTML tags and entities + 'tag': tag, + 'entity': entity, + 'code': { + // everything else is Java code + pattern: /.+/, + inside: 'java', + alias: 'language-java', + }, + }, + }, + }, + }, + ], + 'tag': tag, + 'entity': entity, }, - ], - 'tag': tag, - 'entity': entity, - }); - - return javadoc; + }, + }; }, } as LanguageProto<'javadoc'>; diff --git a/src/languages/javascript.ts b/src/languages/javascript.ts index 9557c12946..77a91bd3bb 100644 --- a/src/languages/javascript.ts +++ b/src/languages/javascript.ts @@ -1,16 +1,22 @@ -import { JS_TEMPLATE, JS_TEMPLATE_INTERPOLATION } from '../shared/languages/patterns'; -import { toArray } from '../util/iterables'; -import { insertBefore } from '../util/language-util'; import clike from './clike'; import type { Grammar, LanguageProto } from '../types'; +export const JS_TEMPLATE_INTERPOLATION = /\$\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})+\}/; +export const JS_TEMPLATE = RegExp( + /`(?:\\[\s\S]||[^\\`$]|\$(?!\{))*`/.source.replace( + '', + () => JS_TEMPLATE_INTERPOLATION.source + ) +); + export default { id: 'javascript', - require: clike, - optional: 'js-templates', + base: clike, alias: 'js', - grammar ({ extend, getOptionalLanguage }) { - const javascript = extend('clike', { + media: ['text/javascript', 'application/javascript'], + extensions: ['js', 'mjs', 'cjs'], + grammar (): Grammar { + return { 'class-name': [ { pattern: /(\b(?:class|extends|implements|instanceof|interface|new)\s+)[\w.\\]+/, @@ -81,137 +87,135 @@ export default { }, 'operator': /--|\+\+|\*\*=?|=>|&&=?|\|\|=?|[!=]==|<<=?|>>>?=?|[-+*/%&|^!=<>]=?|\.{3}|\?\?=?|\?\.?|[~:]/, - }); - - insertBefore(javascript, 'comment', { - 'doc-comment': { - pattern: /\/\*\*(?!\/)[\s\S]*?(?:\*\/|$)/, - greedy: true, - inside: 'jsdoc', - }, - }); - insertBefore(javascript, 'keyword', { - 'regex': { - pattern: RegExp( - // lookbehind - // eslint-disable-next-line regexp/no-dupe-characters-character-class - /((?:^|[^$\w\xA0-\uFFFF."'\])\s]|\b(?:return|yield))\s*)/.source + - // Regex pattern: - // There are 2 regex patterns here. The RegExp set notation proposal added support for nested character - // classes if the `v` flag is present. Unfortunately, nested CCs are both context-free and incompatible - // with the only syntax, so we have to define 2 different regex patterns. - /\//.source + - '(?:' + - /(?:\[(?:[^\]\\\r\n]|\\.)*\]|\\.|[^/\\\[\r\n])+\/[dgimyus]{0,7}/.source + - '|' + - // `v` flag syntax. This supports 3 levels of nested character classes. - /(?:\[(?:[^[\]\\\r\n]|\\.|\[(?:[^[\]\\\r\n]|\\.|\[(?:[^[\]\\\r\n]|\\.)*\])*\])*\]|\\.|[^/\\\[\r\n])+\/[dgimyus]{0,7}v[dgimyus]{0,7}/ - .source + - ')' + - // lookahead - /(?=(?:\s|\/\*(?:[^*]|\*(?!\/))*\*\/)*(?:$|[\r\n,.;:})\]]|\/\/))/.source - ), - lookbehind: true, - greedy: true, - inside: { - 'regex-source': { - pattern: /^(\/)[\s\S]+(?=\/[a-z]*$)/, - lookbehind: true, - alias: 'language-regex', - inside: 'regex', + $insertBefore: { + 'comment': { + 'doc-comment': { + pattern: /\/\*\*(?!\/)[\s\S]*?(?:\*\/|$)/, + greedy: true, + inside: 'jsdoc', }, - 'regex-delimiter': /^\/|\/$/, - 'regex-flags': /^[a-z]+$/, - }, - }, - // This must be declared before keyword because we use "function" inside the look-forward - 'function-variable': { - pattern: - /#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*[=:]\s*(?:async\s*)?(?:\bfunction\b|(?:\((?:[^()]|\([^()]*\))*\)|(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)\s*=>))/, - alias: 'function', - }, - 'parameter': [ - { - pattern: - /(function(?:\s+(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)?\s*\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\))/, - lookbehind: true, - inside: 'javascript', }, - { - pattern: - /(^|[^$\w\xA0-\uFFFF])(?!\s)[_$a-z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*=>)/i, - lookbehind: true, - inside: 'javascript', - }, - { - pattern: /(\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\)\s*=>)/, - lookbehind: true, - inside: 'javascript', - }, - { - pattern: - /((?:\b|\s|^)(?!(?:as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)(?![$\w\xA0-\uFFFF]))(?:(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*\s*)\(\s*|\]\s*\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\)\s*\{)/, - lookbehind: true, - inside: 'javascript', - }, - ], - 'constant': /\b[A-Z](?:[A-Z_]|\dx?)*\b/, - }); - - const jsTemplates = getOptionalLanguage('js-templates')?.['template-string']; - insertBefore(javascript, 'string', { - 'hashbang': { - pattern: /^#!.*/, - greedy: true, - alias: 'comment', - }, - 'template-string': [ - ...toArray(jsTemplates), - { - pattern: JS_TEMPLATE, - greedy: true, - inside: { - 'template-punctuation': { - pattern: /^`|`$/, - alias: 'string', + 'keyword': { + 'regex': { + pattern: RegExp( + // lookbehind + // eslint-disable-next-line regexp/no-dupe-characters-character-class + /((?:^|[^$\w\xA0-\uFFFF."'\])\s]|\b(?:return|yield))\s*)/.source + + // Regex pattern: + // There are 2 regex patterns here. The RegExp set notation proposal added support for nested character + // classes if the `v` flag is present. Unfortunately, nested CCs are both context-free and incompatible + // with the only syntax, so we have to define 2 different regex patterns. + /\//.source + + '(?:' + + /(?:\[(?:[^\]\\\r\n]|\\.)*\]|\\.|[^/\\\[\r\n])+\/[dgimyus]{0,7}/ + .source + + '|' + + // `v` flag syntax. This supports 3 levels of nested character classes. + /(?:\[(?:[^[\]\\\r\n]|\\.|\[(?:[^[\]\\\r\n]|\\.|\[(?:[^[\]\\\r\n]|\\.)*\])*\])*\]|\\.|[^/\\\[\r\n])+\/[dgimyus]{0,7}v[dgimyus]{0,7}/ + .source + + ')' + + // lookahead + /(?=(?:\s|\/\*(?:[^*]|\*(?!\/))*\*\/)*(?:$|[\r\n,.;:})\]]|\/\/))/ + .source + ), + lookbehind: true, + greedy: true, + inside: { + 'regex-source': { + pattern: /^(\/)[\s\S]+(?=\/[a-z]*$)/, + lookbehind: true, + alias: 'language-regex', + inside: 'regex', + }, + 'regex-delimiter': /^\/|\/$/, + 'regex-flags': /^[a-z]+$/, + }, + }, + // This must be declared before keyword because we use "function" inside the look-forward + 'function-variable': { + pattern: + /#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*[=:]\s*(?:async\s*)?(?:\bfunction\b|(?:\((?:[^()]|\([^()]*\))*\)|(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)\s*=>))/, + alias: 'function', + }, + 'parameter': [ + { + pattern: + /(function(?:\s+(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)?\s*\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\))/, + lookbehind: true, + inside: 'javascript', + }, + { + pattern: + /(^|[^$\w\xA0-\uFFFF])(?!\s)[_$a-z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*=>)/i, + lookbehind: true, + inside: 'javascript', }, - 'interpolation': { - pattern: RegExp( - /((?:^|[^\\])(?:\\{2})*)/.source + JS_TEMPLATE_INTERPOLATION.source - ), + { + pattern: + /(\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\)\s*=>)/, lookbehind: true, - inside: { - 'interpolation-punctuation': { - pattern: /^\$\{|\}$/, - alias: 'punctuation', + inside: 'javascript', + }, + { + pattern: + /((?:\b|\s|^)(?!(?:as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)(?![$\w\xA0-\uFFFF]))(?:(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*\s*)\(\s*|\]\s*\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\)\s*\{)/, + lookbehind: true, + inside: 'javascript', + }, + ], + 'constant': /\b[A-Z](?:[A-Z_]|\dx?)*\b/, + }, + + 'string': { + 'hashbang': { + pattern: /^#!.*/, + greedy: true, + alias: 'comment', + }, + 'template-string': { + pattern: JS_TEMPLATE, + greedy: true, + inside: { + 'template-punctuation': { + pattern: /^`|`$/, + alias: 'string', + }, + 'interpolation': { + pattern: RegExp( + /((?:^|[^\\])(?:\\{2})*)/.source + + JS_TEMPLATE_INTERPOLATION.source + ), + lookbehind: true, + inside: { + 'interpolation-punctuation': { + pattern: /^\$\{|\}$/, + alias: 'punctuation', + }, + $rest: 'javascript', }, - $rest: 'javascript', }, + 'string': /[\s\S]+/, }, - 'string': /[\s\S]+/, - } as unknown as Grammar, + }, + 'string-property': { + pattern: + /((?:^|[,{])[ \t]*)(["'])(?:\\(?:\r\n|[\s\S])|(?!\2)[^\\\r\n])*\2(?=\s*:)/m, + lookbehind: true, + greedy: true, + alias: 'property', + }, + } as unknown as Grammar, + 'operator': { + 'literal-property': { + pattern: + /((?:^|[,{])[ \t]*)(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*:)/m, + lookbehind: true, + alias: 'property', + }, }, - ], - 'string-property': { - pattern: - /((?:^|[,{])[ \t]*)(["'])(?:\\(?:\r\n|[\s\S])|(?!\2)[^\\\r\n])*\2(?=\s*:)/m, - lookbehind: true, - greedy: true, - alias: 'property', }, - }); - - insertBefore(javascript, 'operator', { - 'literal-property': { - pattern: - /((?:^|[,{])[ \t]*)(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*:)/m, - lookbehind: true, - alias: 'property', - }, - }); - - return javascript; + }; }, -} as LanguageProto<'javascript'>; +} satisfies LanguageProto<'javascript'>; diff --git a/src/languages/jolie.ts b/src/languages/jolie.ts index 38b96e830f..230fbc9707 100644 --- a/src/languages/jolie.ts +++ b/src/languages/jolie.ts @@ -1,12 +1,11 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; import type { LanguageProto } from '../types'; export default { id: 'jolie', - require: clike, - grammar ({ extend }) { - const jolie = extend('clike', { + base: clike, + grammar () { + return { 'string': { pattern: /(^|[^\\])"(?:\\[\s\S]|[^"\\])*"/, lookbehind: true, @@ -25,34 +24,33 @@ export default { 'punctuation': /[()[\]{},;.:]/, 'builtin': /\b(?:Byte|any|bool|char|double|enum|float|int|length|long|ranges|regex|string|undefined|void)\b/, - }); - - insertBefore(jolie, 'keyword', { - 'aggregates': { - pattern: - /(\bAggregates\s*:\s*)(?:\w+(?:\s+with\s+\w+)?\s*,\s*)*\w+(?:\s+with\s+\w+)?/, - lookbehind: true, - inside: { - 'keyword': /\bwith\b/, - 'class-name': /\w+/, - 'punctuation': /,/, + $insertBefore: { + 'keyword': { + 'aggregates': { + pattern: + /(\bAggregates\s*:\s*)(?:\w+(?:\s+with\s+\w+)?\s*,\s*)*\w+(?:\s+with\s+\w+)?/, + lookbehind: true, + inside: { + 'keyword': /\bwith\b/, + 'class-name': /\w+/, + 'punctuation': /,/, + }, + }, + 'redirects': { + pattern: /(\bRedirects\s*:\s*)(?:\w+\s*=>\s*\w+\s*,\s*)*(?:\w+\s*=>\s*\w+)/, + lookbehind: true, + inside: { + 'punctuation': /,/, + 'class-name': /\w+/, + 'operator': /=>/, + }, + }, + 'property': { + pattern: + /\b(?:Aggregates|[Ii]nterfaces|Java|Javascript|Jolie|[Ll]ocation|OneWay|[Pp]rotocol|Redirects|RequestResponse)\b(?=[ \t]*:)/, + }, }, }, - 'redirects': { - pattern: /(\bRedirects\s*:\s*)(?:\w+\s*=>\s*\w+\s*,\s*)*(?:\w+\s*=>\s*\w+)/, - lookbehind: true, - inside: { - 'punctuation': /,/, - 'class-name': /\w+/, - 'operator': /=>/, - }, - }, - 'property': { - pattern: - /\b(?:Aggregates|[Ii]nterfaces|Java|Javascript|Jolie|[Ll]ocation|OneWay|[Pp]rotocol|Redirects|RequestResponse)\b(?=[ \t]*:)/, - }, - }); - - return jolie; + }; }, } as LanguageProto<'jolie'>; diff --git a/src/languages/js-templates.ts b/src/languages/js-templates.ts index 6c8f5a1f1f..d29a5626fb 100644 --- a/src/languages/js-templates.ts +++ b/src/languages/js-templates.ts @@ -1,5 +1,5 @@ -import { JS_TEMPLATE, JS_TEMPLATE_INTERPOLATION } from '../shared/languages/patterns'; import { embeddedIn } from '../shared/languages/templating'; +import javascript, { JS_TEMPLATE, JS_TEMPLATE_INTERPOLATION } from './javascript'; import type { Grammar, GrammarToken, LanguageProto } from '../types'; /** @@ -42,13 +42,16 @@ function createTemplate (language: string, tag: string): GrammarToken { $tokenize: embeddedIn(language), }, }, - } as Grammar, + } as unknown as Grammar, }; } export default { id: 'js-templates', + require: javascript, + extends: 'javascript', grammar () { + // TODO use function with groups return { 'template-string': [ // styled-jsx: diff --git a/src/languages/jsdoc.ts b/src/languages/jsdoc.ts index 4ebbf77f6d..795c2bc026 100644 --- a/src/languages/jsdoc.ts +++ b/src/languages/jsdoc.ts @@ -1,20 +1,17 @@ -import { insertBefore } from '../util/language-util'; import javadoclike from './javadoclike'; import javascript from './javascript'; import typescript from './typescript'; -import type { LanguageProto } from '../types'; +import type { Grammar, GrammarOptions, LanguageProto } from '../types'; export default { id: 'jsdoc', - require: [javascript, javadoclike, typescript], - grammar ({ extend, getLanguage }) { - const javascript = getLanguage('javascript'); - const typescript = getLanguage('typescript'); - + require: [javascript, typescript], + base: javadoclike, + grammar ({ languages }: GrammarOptions): Grammar { const type = /\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})+\}/.source; const parameterPrefix = '(@(?:arg|argument|param|property)\\s+(?:' + type + '\\s+)?)'; - const jsdoc = extend('javadoclike', { + return { 'parameter': { // @param {string} foo - foo bar pattern: RegExp(parameterPrefix + /(?:(?!\s)[$\w\xA0-\uFFFF.])+(?=\s|$)/.source), @@ -23,72 +20,76 @@ export default { 'punctuation': /\./, }, }, - }); - - insertBefore(jsdoc, 'keyword', { - 'optional-parameter': { - // @param {string} [baz.foo="bar"] foo bar - pattern: RegExp( - parameterPrefix + /\[(?:(?!\s)[$\w\xA0-\uFFFF.])+(?:=[^[\]]+)?\](?=\s|$)/.source - ), - lookbehind: true, - inside: { - 'parameter': { - pattern: /(^\[)[$\w\xA0-\uFFFF\.]+/, + $insertBefore: { + 'keyword': { + 'optional-parameter': { + // @param {string} [baz.foo="bar"] foo bar + pattern: RegExp( + parameterPrefix + + /\[(?:(?!\s)[$\w\xA0-\uFFFF.])+(?:=[^[\]]+)?\](?=\s|$)/.source + ), lookbehind: true, inside: { - 'punctuation': /\./, + 'parameter': { + pattern: /(^\[)[$\w\xA0-\uFFFF\.]+/, + lookbehind: true, + inside: { + 'punctuation': /\./, + }, + }, + 'code': { + pattern: /(=)[\s\S]*(?=\]$)/, + lookbehind: true, + inside: 'javascript', + }, + 'punctuation': /[=[\]]/, }, }, - 'code': { - pattern: /(=)[\s\S]*(?=\]$)/, - lookbehind: true, - alias: 'language-javascript', - inside: 'javascript', - }, - 'punctuation': /[=[\]]/, - }, - }, - 'class-name': [ - { - pattern: RegExp( - /(@(?:augments|class|extends|interface|memberof!?|template|this|typedef)\s+(?:\s+)?)[A-Z]\w*(?:\.[A-Z]\w*)*/.source.replace( - //g, - () => type - ) - ), - lookbehind: true, - inside: { - 'punctuation': /\./, - }, - }, - { - pattern: RegExp('(@[a-z]+\\s+)' + type), - lookbehind: true, - inside: { - 'string': javascript.string, - 'number': javascript.number, - 'boolean': javascript.boolean, - 'keyword': typescript.keyword, - 'operator': /=>|\.\.\.|[&|?:*]/, - 'punctuation': /[.,;=<>{}()[\]]/, - }, - }, - ], - 'example': { - pattern: /(@example\s+(?!\s))(?:[^@\s]|\s+(?!\s))+?(?=\s*(?:\*\s*)?(?:@\w|\*\/))/, - lookbehind: true, - inside: { - 'code': { - pattern: /^([\t ]*(?:\*\s*)?)\S.*$/m, + 'class-name': [ + { + pattern: RegExp( + /(@(?:augments|class|extends|interface|memberof!?|template|this|typedef)\s+(?:\s+)?)[A-Z]\w*(?:\.[A-Z]\w*)*/.source.replace( + //g, + () => type + ) + ), + lookbehind: true, + inside: { + 'punctuation': /\./, + }, + }, + { + pattern: RegExp('(@[a-z]+\\s+)' + type), + lookbehind: true, + get inside () { + // Lazily evaluated + let { javascript, typescript } = languages; + delete this.inside; + return (this.inside = { + 'string': javascript.string, + 'number': javascript.number, + 'boolean': javascript.boolean, + 'keyword': typescript.keyword, + 'operator': /=>|\.\.\.|[&|?:*]/, + 'punctuation': /[.,;=<>{}()[\]]/, + }); + }, + }, + ], + 'example': { + pattern: + /(@example\s+(?!\s))(?:[^@\s]|\s+(?!\s))+?(?=\s*(?:\*\s*)?(?:@\w|\*\/))/, lookbehind: true, - alias: 'language-javascript', - inside: 'javascript', + inside: { + 'code': { + pattern: /^([\t ]*(?:\*\s*)?)\S.*$/m, + lookbehind: true, + inside: 'javascript', + }, + }, }, }, }, - }); - - return jsdoc; + }; }, } as LanguageProto<'jsdoc'>; diff --git a/src/languages/json.ts b/src/languages/json.ts index 0344444ea7..8a0794a693 100644 --- a/src/languages/json.ts +++ b/src/languages/json.ts @@ -3,6 +3,8 @@ import type { LanguageProto } from '../types'; export default { id: 'json', alias: 'webmanifest', + media: ['application/json', 'application/manifest+json'], + extensions: ['json', 'webmanifest'], grammar () { // https://www.json.org/json-en.html return { diff --git a/src/languages/json5.ts b/src/languages/json5.ts index cdc7abadf9..fd96bb757a 100644 --- a/src/languages/json5.ts +++ b/src/languages/json5.ts @@ -3,11 +3,11 @@ import type { LanguageProto } from '../types'; export default { id: 'json5', - require: json, - grammar ({ extend }) { + base: json, + grammar () { const string = /("|')(?:\\(?:\r\n?|\n|.)|(?!\1)[^\\\r\n])*\1/; - return extend('json', { + return { 'property': [ { pattern: RegExp(string.source + '(?=\\s*:)'), @@ -24,6 +24,6 @@ export default { }, 'number': /[+-]?\b(?:NaN|Infinity|0x[a-fA-F\d]+)\b|[+-]?(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:[eE][+-]?\d+\b)?/, - }); + }; }, } as LanguageProto<'json5'>; diff --git a/src/languages/jsonp.ts b/src/languages/jsonp.ts index 463779e351..dcc4ae0d51 100644 --- a/src/languages/jsonp.ts +++ b/src/languages/jsonp.ts @@ -1,19 +1,17 @@ -import { insertBefore } from '../util/language-util'; import json from './json'; import type { LanguageProto } from '../types'; export default { id: 'jsonp', - require: json, - grammar ({ extend }) { - const jsonp = extend('json', { + base: json, + grammar () { + return { 'punctuation': /[{}[\]();,.]/, - }); - - insertBefore(jsonp, 'punctuation', { - 'function': /(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*\()/, - }); - - return jsonp; + $insertBefore: { + 'punctuation': { + 'function': /(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*\()/, + }, + }, + }; }, } as LanguageProto<'jsonp'>; diff --git a/src/languages/jsx.ts b/src/languages/jsx.ts index d962fa1207..5d5e72e5fb 100644 --- a/src/languages/jsx.ts +++ b/src/languages/jsx.ts @@ -1,5 +1,6 @@ import { getTextContent, Token } from '../core/classes/token'; -import { insertBefore, withoutTokenize } from '../util/language-util'; +import { insertBefore } from '../util/insert'; +import { withoutTokenize } from '../util/without-tokenize'; import javascript from './javascript'; import markup from './markup'; import type { TokenStream } from '../core/classes/token'; @@ -9,9 +10,8 @@ function stringifyToken (token: string | Token | TokenStream | undefined): strin if (!token) { return ''; } - else { - return getTextContent(token); - } + + return getTextContent(token); } function walkTokens (tokens: TokenStream) { diff --git a/src/languages/kotlin.ts b/src/languages/kotlin.ts index 448ac33c49..02535967f7 100644 --- a/src/languages/kotlin.ts +++ b/src/languages/kotlin.ts @@ -1,13 +1,23 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'kotlin', - require: clike, + base: clike, alias: ['kt', 'kts'], - grammar ({ extend }) { - const kotlin = extend('clike', { + grammar () { + const interpolationInside = { + 'interpolation-punctuation': { + pattern: /^\$\{?|\}$/, + alias: 'punctuation', + }, + 'expression': { + pattern: /[\s\S]+/, + inside: 'kotlin', + }, + }; + + return { 'keyword': { // The lookbehind prevents wrong highlighting of e.g. kotlin.properties.get pattern: @@ -29,71 +39,54 @@ export default { /\b(?:0[xX][\da-fA-F]+(?:_[\da-fA-F]+)*|0[bB][01]+(?:_[01]+)*|\d+(?:_\d+)*(?:\.\d+(?:_\d+)*)?(?:[eE][+-]?\d+(?:_\d+)*)?[fFL]?)\b/, 'operator': /\+[+=]?|-[-=>]?|==?=?|!(?:!|==?)?|[\/*%<>]=?|[?:]:?|\.\.|&&|\|\||\b(?:and|inv|or|shl|shr|ushr|xor)\b/, - }); - - delete kotlin['class-name']; - - const interpolationInside = { - 'interpolation-punctuation': { - pattern: /^\$\{?|\}$/, - alias: 'punctuation', - }, - 'expression': { - pattern: /[\s\S]+/, - inside: 'kotlin', - }, - }; - - insertBefore(kotlin, 'string', { - // https://kotlinlang.org/spec/expressions.html#string-interpolation-expressions - 'string-literal': [ - { - pattern: /"""(?:[^$]|\$(?:(?!\{)|\{[^{}]*\}))*?"""/, - alias: 'multiline', - inside: { - 'interpolation': { - pattern: /\$(?:[a-z_]\w*|\{[^{}]*\})/i, - inside: interpolationInside, + $insertBefore: { + 'string': { + // https://kotlinlang.org/spec/expressions.html#string-interpolation-expressions + 'string-literal': [ + { + pattern: /"""(?:[^$]|\$(?:(?!\{)|\{[^{}]*\}))*?"""/, + alias: 'multiline', + inside: { + 'interpolation': { + pattern: /\$(?:[a-z_]\w*|\{[^{}]*\})/i, + inside: interpolationInside, + }, + 'string': /[\s\S]+/, + }, + }, + { + pattern: /"(?:[^"\\\r\n$]|\\.|\$(?:(?!\{)|\{[^{}]*\}))*"/, + alias: 'singleline', + inside: { + 'interpolation': { + pattern: /((?:^|[^\\])(?:\\{2})*)\$(?:[a-z_]\w*|\{[^{}]*\})/i, + lookbehind: true, + inside: interpolationInside, + }, + 'string': /[\s\S]+/, + }, }, - 'string': /[\s\S]+/, + ], + 'char': { + // https://kotlinlang.org/spec/expressions.html#character-literals + pattern: /'(?:[^'\\\r\n]|\\(?:.|u[a-fA-F0-9]{0,4}))'/, + greedy: true, }, }, - { - pattern: /"(?:[^"\\\r\n$]|\\.|\$(?:(?!\{)|\{[^{}]*\}))*"/, - alias: 'singleline', - inside: { - 'interpolation': { - pattern: /((?:^|[^\\])(?:\\{2})*)\$(?:[a-z_]\w*|\{[^{}]*\})/i, - lookbehind: true, - inside: interpolationInside, - }, - 'string': /[\s\S]+/, + 'keyword': { + 'annotation': { + pattern: /\B@(?:\w+:)?(?:[A-Z]\w*|\[[^\]]+\])/, + alias: 'builtin', + }, + }, + 'function': { + 'label': { + pattern: /\b\w+@|@\w+\b/, + alias: 'symbol', }, }, - ], - 'char': { - // https://kotlinlang.org/spec/expressions.html#character-literals - pattern: /'(?:[^'\\\r\n]|\\(?:.|u[a-fA-F0-9]{0,4}))'/, - greedy: true, - }, - }); - - delete kotlin['string']; - - insertBefore(kotlin, 'keyword', { - 'annotation': { - pattern: /\B@(?:\w+:)?(?:[A-Z]\w*|\[[^\]]+\])/, - alias: 'builtin', - }, - }); - - insertBefore(kotlin, 'function', { - 'label': { - pattern: /\b\w+@|@\w+\b/, - alias: 'symbol', }, - }); - - return kotlin; + $delete: ['class-name', 'string'], + } as unknown as Grammar; }, } as LanguageProto<'kotlin'>; diff --git a/src/languages/latte.ts b/src/languages/latte.ts index 78dda2eabd..71ce5cd452 100644 --- a/src/languages/latte.ts +++ b/src/languages/latte.ts @@ -1,43 +1,13 @@ import { embeddedIn } from '../shared/languages/templating'; -import { insertBefore } from '../util/language-util'; import markup from './markup'; import php from './php'; -import type { Grammar, GrammarToken, LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'latte', - require: [markup, php], - grammar ({ extend }) { - const markupLatte = extend('markup', {}); - const tag = markupLatte.tag as GrammarToken & { inside: Grammar }; - insertBefore(tag.inside, 'attr-value', { - 'n-attr': { - pattern: /n:[\w-]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+))?/, - inside: { - 'attr-name': { - pattern: /^[^\s=]+/, - alias: 'important', - }, - 'attr-value': { - pattern: /=[\s\S]+/, - inside: { - 'punctuation': [ - /^=/, - { - pattern: /^(\s*)["']|["']$/, - lookbehind: true, - }, - ], - 'php': { - pattern: /\S(?:[\s\S]*\S)?/, - inside: 'php', - }, - }, - }, - }, - }, - }); - + base: markup, + require: php, + grammar () { return { 'latte-comment': { pattern: /\{\*[\s\S]*?\*\}/, @@ -66,7 +36,36 @@ export default { }, }, }, - $tokenize: embeddedIn(markupLatte) as Grammar['$tokenize'], - }; + $insertBefore: { + 'tag/attr-value': { + 'n-attr': { + pattern: /n:[\w-]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+))?/, + inside: { + 'attr-name': { + pattern: /^[^\s=]+/, + alias: 'important', + }, + 'attr-value': { + pattern: /=[\s\S]+/, + inside: { + 'punctuation': [ + /^=/, + { + pattern: /^(\s*)["']|["']$/, + lookbehind: true, + }, + ], + 'php': { + pattern: /\S(?:[\s\S]*\S)?/, + inside: 'php', + }, + }, + }, + }, + }, + }, + }, + $tokenize: embeddedIn('markup'), + } as unknown as Grammar; }, } as LanguageProto<'latte'>; diff --git a/src/languages/less.ts b/src/languages/less.ts index 2a093bcde3..f38a6a50b3 100644 --- a/src/languages/less.ts +++ b/src/languages/less.ts @@ -1,11 +1,10 @@ -import { insertBefore } from '../util/language-util'; import css from './css'; import type { LanguageProto } from '../types'; export default { id: 'less', - require: css, - grammar ({ extend }) { + base: css, + grammar () { /* FIXME : :extend() is not handled specifically : its highlighting is buggy. Mixin usage must be inside a ruleset to be highlighted. @@ -14,7 +13,7 @@ export default { A comment before a mixin usage prevents the latter to be properly highlighted. */ - const less = extend('css', { + return { 'comment': [ /\/\*[\s\S]*?\*\//, { @@ -40,28 +39,28 @@ export default { 'property': /(?:@\{[\w-]+\}|[\w-])+(?:\+_?)?(?=\s*:)/, 'operator': /[+\-*\/]/, - }); - insertBefore(less, 'property', { - 'variable': [ - // Variable declaration (the colon must be consumed!) - { - pattern: /@[\w-]+\s*:/, - inside: { - 'punctuation': /:/, + $insertBefore: { + 'property': { + 'variable': [ + // Variable declaration (the colon must be consumed!) + { + pattern: /@[\w-]+\s*:/, + inside: { + 'punctuation': /:/, + }, + }, + + // Variable usage + /@@?[\w-]+/, + ], + 'mixin-usage': { + pattern: /([{;]\s*)[.#](?!\d)[\w-].*?(?=[(;])/, + lookbehind: true, + alias: 'function', }, }, - - // Variable usage - /@@?[\w-]+/, - ], - 'mixin-usage': { - pattern: /([{;]\s*)[.#](?!\d)[\w-].*?(?=[(;])/, - lookbehind: true, - alias: 'function', }, - }); - - return less; + }; }, } as LanguageProto<'less'>; diff --git a/src/languages/lisp.ts b/src/languages/lisp.ts index 97c761fce9..235d6f3249 100644 --- a/src/languages/lisp.ts +++ b/src/languages/lisp.ts @@ -25,7 +25,7 @@ export default { const symbol = /(?!\d)[-+*/~!@$%^=<>{}\w]+/.source; // symbol starting with & used in function arguments const marker = '&' + symbol; - // Open parenthesis for look-behind + // Open parenthesis for lookbehind const par = '(\\()'; const endpar = '(?=\\))'; // End the pattern with look-ahead space diff --git a/src/languages/log.ts b/src/languages/log.ts index f75e16c7b5..92056f2bf7 100644 --- a/src/languages/log.ts +++ b/src/languages/log.ts @@ -3,7 +3,7 @@ import type { LanguageProto } from '../types'; export default { id: 'log', optional: 'javastacktrace', - grammar ({ getOptionalLanguage }) { + grammar ({ languages }) { // This is a language definition for generic log files. // Since there is no one log format, this language definition has to support all formats to some degree. // @@ -22,7 +22,7 @@ export default { lookbehind: true, greedy: true, alias: ['javastacktrace', 'language-javastacktrace'], - inside: getOptionalLanguage('javastacktrace') || { + inside: languages['javastacktrace'] || { 'keyword': /\bat\b/, 'function': /[a-z_][\w$]*(?=\()/, 'punctuation': /[.:()]/, diff --git a/src/languages/markdown.ts b/src/languages/markdown.ts index daca0b49e7..1aa9954386 100644 --- a/src/languages/markdown.ts +++ b/src/languages/markdown.ts @@ -1,15 +1,11 @@ -import { getTextContent } from '../core/classes/token'; -import { insertBefore, withoutTokenize } from '../util/language-util'; import markup from './markup'; import type { Grammar, GrammarToken, LanguageProto } from '../types'; -import type { Prism } from '../core/prism'; -import type { Autoloader } from '../plugins/autoloader/prism-autoloader'; export default { id: 'markdown', - require: markup, + base: markup, alias: 'md', - grammar ({ extend }) { + grammar () { // Allow only one line break const inner = /(?:\\.|[^\\\n\r]|(?:\n|\r\n?)(?![\r\n]))/.source; @@ -33,8 +29,7 @@ export default { const tableLine = /\|?[ \t]*:?-{3,}:?[ \t]*(?:\|[ \t]*:?-{3,}:?[ \t]*)+\|?(?:\n|\r\n?)/ .source; - const markdown = extend('markup', {}); - insertBefore(markdown, 'prolog', { + let markdown = { 'front-matter-block': { pattern: /(^(?:\s*[\r\n])?)---(?!.)[\s\S]*?[\r\n]---(?!.)/, lookbehind: true, @@ -99,71 +94,13 @@ export default { // ```optional language // code block // ``` - pattern: /^```[\s\S]*?^```$/m, - greedy: true, + pattern: + /^```(?:\s*)(?[a-z-]+)(?:.+)?(?:\n|\r\n?)(?[\s\S]*?)(?:\n|\r\n?)```$/im, inside: { - 'code-block': { - pattern: /^(```.*(?:\n|\r\n?))[\s\S]+?(?=(?:\n|\r\n?)^```$)/m, - lookbehind: true, - }, - 'code-language': { - pattern: /^(```).+/, - lookbehind: true, - }, + 'code-language': groups => groups.codeLanguage, + 'code-block': groups => groups.codeBlock, 'punctuation': /```/, - $tokenize (code: string, grammar: Grammar, Prism: Prism) { - const tokens = Prism.tokenize(code, withoutTokenize(grammar)); - - /* - * Add the correct `language-xxxx` class to this code block. Keep in mind that the `code-language` token - * is optional. But the grammar is defined so that there is only one case we have to handle: - * - * token.content = [ - * ```, - * xxxx, - * '\n', // exactly one new lines (\r or \n or \r\n) - * ..., - * '\n', // exactly one new lines again - * ``` - * ]; - */ - - const codeLang = tokens[1]; - const codeBlock = tokens[3]; - - if ( - typeof codeLang === 'object' && - typeof codeBlock === 'object' && - codeLang.type === 'code-language' && - codeBlock.type === 'code-block' - ) { - // this might be a language that Prism does not support - - // do some replacements to support C++, C#, and F# - const lang = getTextContent(codeLang.content) - .replace(/\b#/g, 'sharp') - .replace(/\b\+\+/g, 'pp'); - // only use the first word - const langName = /[a-z][\w-]*/i.exec(lang)?.[0].toLowerCase(); - if (langName) { - codeBlock.addAlias('language-' + langName); - - const grammar = Prism.components.getLanguage(lang); - if (grammar) { - codeBlock.content = Prism.tokenize( - getTextContent(codeBlock), - grammar - ); - } - else { - codeBlock.addAlias('needs-highlighting'); - } - } - } - - return tokens; - }, - } as unknown as Grammar, + }, }, ], 'title': [ @@ -320,61 +257,21 @@ export default { }, }, }, - }); + }; ['url', 'bold', 'italic', 'strike'].forEach(token => { ['url', 'bold', 'italic', 'strike', 'code-snippet'].forEach(inside => { - if (token !== inside) { - ( - ( - ((markdown[token] as GrammarToken).inside as Grammar) - .content as GrammarToken - ).inside as Grammar - )[inside] = markdown[inside]; + if (token === inside) { + return; } + + ( + (((markdown[token] as GrammarToken).inside as Grammar).content as GrammarToken) + .inside as Grammar + )[inside] = markdown[inside]; }); }); - return markdown; - }, - effect (Prism) { - return Prism.hooks.add('wrap', env => { - if ( - !Prism.plugins.autoloader || - env.type !== 'code-block' || - !env.classes.includes('needs-highlighting') - ) { - return; - } - - let codeLang = ''; - for (let i = 0, l = env.classes.length; i < l; i++) { - const cls = env.classes[i]; - const match = /language-(.+)/.exec(cls); - if (match) { - codeLang = match[1]; - break; - } - } - - if (codeLang && codeLang !== 'none' && typeof document !== 'undefined') { - const id = `md-${new Date().valueOf()}-${Math.floor(Math.random() * 1e16)}`; - env.attributes['id'] = id; - - const autoloader = Prism.plugins.autoloader as Autoloader; - autoloader.loadLanguages(codeLang).then( - () => { - const element = document.getElementById(id); - if (element) { - element.innerHTML = Prism.highlight( - element.textContent || '', - codeLang - ); - } - }, - error => console.error(error) - ); - } - }); + return { $insertBefore: { 'prolog': markdown } }; }, } as LanguageProto<'markdown'>; diff --git a/src/languages/markup.ts b/src/languages/markup.ts index 7e055b9c5c..b972488f89 100644 --- a/src/languages/markup.ts +++ b/src/languages/markup.ts @@ -1,4 +1,3 @@ -import { insertBefore } from '../util/language-util'; import xml from './xml'; import type { Grammar, GrammarToken, LanguageProto } from '../types'; @@ -92,30 +91,30 @@ function attributeEmbedded (attrName: string, lang: string): GrammarToken { export default { id: 'markup', - require: xml, + base: xml, alias: ['html', 'svg', 'mathml'], - grammar ({ extend }) { - const markup = extend('xml', {}); - - insertBefore(markup, 'cdata', { - 'style': inlineEmbedded('style', 'css'), - 'script': inlineEmbedded('script', 'javascript'), - }); - - const tag = markup.tag as GrammarToken & { inside: Grammar }; - insertBefore(tag.inside, 'attr-value', { - 'special-attr': [ - attributeEmbedded('style', 'css'), - // add attribute support for all DOM events. - // https://developer.mozilla.org/en-US/docs/Web/Events#Standard_events - attributeEmbedded( - /on(?:abort|blur|change|click|composition(?:end|start|update)|dblclick|error|focus(?:in|out)?|key(?:down|up)|load|mouse(?:down|enter|leave|move|out|over|up)|reset|resize|scroll|select|slotchange|submit|unload|wheel)/ - .source, - 'javascript' - ), - ], - }); - - return markup; + media: 'text/html', + extensions: ['html', 'htm', 'xhtml', 'svg'], + grammar () { + return { + $insertBefore: { + 'cdata': { + 'style': inlineEmbedded('style', 'css'), + 'script': inlineEmbedded('script', 'javascript'), + }, + 'tag/attr-value': { + 'special-attr': [ + attributeEmbedded('style', 'css'), + // add attribute support for all DOM events. + // https://developer.mozilla.org/en-US/docs/Web/Events#Standard_events + attributeEmbedded( + /on(?:abort|blur|change|click|composition(?:end|start|update)|dblclick|error|focus(?:in|out)?|key(?:down|up)|load|mouse(?:down|enter|leave|move|out|over|up)|reset|resize|scroll|select|slotchange|submit|unload|wheel)/ + .source, + 'javascript' + ), + ], + }, + }, + }; }, } as LanguageProto<'markup'>; diff --git a/src/languages/mongodb.ts b/src/languages/mongodb.ts index bad5b1b35d..686250b33d 100644 --- a/src/languages/mongodb.ts +++ b/src/languages/mongodb.ts @@ -1,11 +1,10 @@ -import { insertBefore } from '../util/language-util'; import javascript from './javascript'; -import type { GrammarToken, LanguageProto } from '../types'; +import type { LanguageProto } from '../types'; export default { id: 'mongodb', - require: javascript, - grammar ({ extend }) { + base: javascript, + grammar () { let operators = [ // query and projection '$eq', @@ -276,41 +275,43 @@ export default { const operatorsSource = '(?:' + operators.join('|') + ')\\b'; - const mongodb = extend('javascript', {}); - - insertBefore(mongodb, 'string', { - 'property': { - pattern: - /(?:(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1|(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)(?=\s*:)/, - greedy: true, - inside: { - 'keyword': RegExp('^([\'"])?' + operatorsSource + '(?:\\1)?$'), + return { + $insertBefore: { + 'string': { + 'property': { + pattern: + /(?:(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1|(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)(?=\s*:)/, + greedy: true, + inside: { + 'keyword': RegExp('^([\'"])?' + operatorsSource + '(?:\\1)?$'), + }, + }, + }, + 'constant': { + 'builtin': { + pattern: RegExp('\\b(?:' + builtinFunctions.join('|') + ')\\b'), + alias: 'keyword', + }, }, }, - }); - - const string = mongodb['string'] as GrammarToken; - string.inside = { - url: { - // url pattern - pattern: /https?:\/\/[-\w@:%.+~#=]{1,256}\.[a-z0-9()]{1,6}\b[-\w()@:%+.~#?&/=]*/i, - greedy: true, - }, - entity: { - // ipv4 - pattern: - /\b(?:(?:[01]?\d\d?|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d\d?|2[0-4]\d|25[0-5])\b/, - greedy: true, + $merge: { + 'string': { + inside: { + url: { + // url pattern + pattern: + /https?:\/\/[-\w@:%.+~#=]{1,256}\.[a-z0-9()]{1,6}\b[-\w()@:%+.~#?&/=]*/i, + greedy: true, + }, + entity: { + // ipv4 + pattern: + /\b(?:(?:[01]?\d\d?|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d\d?|2[0-4]\d|25[0-5])\b/, + greedy: true, + }, + }, + }, }, }; - - insertBefore(mongodb, 'constant', { - 'builtin': { - pattern: RegExp('\\b(?:' + builtinFunctions.join('|') + ')\\b'), - alias: 'keyword', - }, - }); - - return mongodb; }, } as LanguageProto<'mongodb'>; diff --git a/src/languages/n4js.ts b/src/languages/n4js.ts index ccef12154c..c840cd1d87 100644 --- a/src/languages/n4js.ts +++ b/src/languages/n4js.ts @@ -1,26 +1,23 @@ -import { insertBefore } from '../util/language-util'; import javascript from './javascript'; import type { LanguageProto } from '../types'; export default { id: 'n4js', - require: javascript, + base: javascript, alias: 'n4jsd', - grammar ({ extend }) { - const n4js = extend('javascript', { + grammar () { + return { // Keywords from N4JS language spec: https://numberfour.github.io/n4js/spec/N4JSSpec.html 'keyword': /\b(?:Array|any|boolean|break|case|catch|class|const|constructor|continue|debugger|declare|default|delete|do|else|enum|export|extends|false|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|module|new|null|number|package|private|protected|public|return|set|static|string|super|switch|this|throw|true|try|typeof|var|void|while|with|yield)\b/, - }); - - insertBefore(n4js, 'constant', { - // Annotations in N4JS spec: https://numberfour.github.io/n4js/spec/N4JSSpec.html#_annotations - 'annotation': { - pattern: /@+\w+/, - alias: 'operator', + $insert: { + // Annotations in N4JS spec: https://numberfour.github.io/n4js/spec/N4JSSpec.html#_annotations + 'annotation': { + $before: 'constant', + pattern: /@+\w+/, + alias: 'operator', + }, }, - }); - - return n4js; + }; }, } as LanguageProto<'n4js'>; diff --git a/src/languages/naniscript.ts b/src/languages/naniscript.ts index b0c5715d3c..5105e16700 100644 --- a/src/languages/naniscript.ts +++ b/src/languages/naniscript.ts @@ -1,7 +1,6 @@ import { getTextContent } from '../core/classes/token'; -import { withoutTokenize } from '../util/language-util'; -import type { Grammar, LanguageProto } from '../types'; -import type { Prism } from '../core/prism'; +import { withoutTokenize } from '../util/without-tokenize'; +import type { Grammar, LanguageProto, Prism } from '../types'; function isBracketsBalanced (input: string): boolean { const brackets = '[]{}'; diff --git a/src/languages/objectivec.ts b/src/languages/objectivec.ts index 7127fedb0a..f6308387e7 100644 --- a/src/languages/objectivec.ts +++ b/src/languages/objectivec.ts @@ -1,12 +1,12 @@ import c from './c'; -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'objectivec', - require: c, + base: c, alias: 'objc', - grammar ({ extend }) { - const objectivec = extend('c', { + grammar () { + return { 'string': { pattern: /@?"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"/, greedy: true, @@ -14,10 +14,7 @@ export default { 'keyword': /\b(?:asm|auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|in|inline|int|long|register|return|self|short|signed|sizeof|static|struct|super|switch|typedef|typeof|union|unsigned|void|volatile|while)\b|(?:@interface|@end|@implementation|@protocol|@class|@public|@protected|@private|@property|@try|@catch|@finally|@throw|@synthesize|@dynamic|@selector)\b/, 'operator': /-[->]?|\+\+?|!=?|<>?=?|==?|&&?|\|\|?|[~^%?*\/@]/, - }); - - delete objectivec['class-name']; - - return objectivec; + $delete: ['class-name'], + } as Grammar; }, } as LanguageProto<'objectivec'>; diff --git a/src/languages/opencl-extensions.ts b/src/languages/opencl-extensions.ts index 027c452b68..81f27eed84 100644 --- a/src/languages/opencl-extensions.ts +++ b/src/languages/opencl-extensions.ts @@ -1,37 +1,53 @@ import type { LanguageProto } from '../types'; -export default { +export const c = { id: 'opencl-extensions', + extends: 'c', grammar: { - // Extracted from http://streamcomputing.eu/downloads/?opencl_host.lang (opencl-types and opencl-host) - 'type-opencl-host': { - pattern: - /\b(?:cl_(?:GLenum|GLint|GLuin|addressing_mode|bitfield|bool|buffer_create_type|build_status|channel_(?:order|type)|(?:u?(?:char|int|long|short)|double|float)(?:2|3|4|8|16)?|command_(?:queue(?:_info|_properties)?|type)|context(?:_info|_properties)?|device_(?:exec_capabilities|fp_config|id|info|local_mem_type|mem_cache_type|type)|(?:event|sampler)(?:_info)?|filter_mode|half|image_info|kernel(?:_info|_work_group_info)?|map_flags|mem(?:_flags|_info|_object_type)?|platform_(?:id|info)|profiling_info|program(?:_build_info|_info)?))\b/, - alias: 'keyword', - }, - 'boolean-opencl-host': { - pattern: /\bCL_(?:FALSE|TRUE)\b/, - alias: 'boolean', - }, - // Extracted from cl.h (2.0) and http://streamcomputing.eu/downloads/?opencl_host.lang (opencl-const) - 'constant-opencl-host': { - pattern: - /\bCL_(?:A|ABGR|ADDRESS_(?:CLAMP(?:_TO_EDGE)?|MIRRORED_REPEAT|NONE|REPEAT)|ARGB|BGRA|BLOCKING|BUFFER_CREATE_TYPE_REGION|BUILD_(?:ERROR|IN_PROGRESS|NONE|PROGRAM_FAILURE|SUCCESS)|COMMAND_(?:ACQUIRE_GL_OBJECTS|BARRIER|COPY_(?:BUFFER(?:_RECT|_TO_IMAGE)?|IMAGE(?:_TO_BUFFER)?)|FILL_(?:BUFFER|IMAGE)|MAP(?:_BUFFER|_IMAGE)|MARKER|MIGRATE(?:_SVM)?_MEM_OBJECTS|NATIVE_KERNEL|NDRANGE_KERNEL|READ_(?:BUFFER(?:_RECT)?|IMAGE)|RELEASE_GL_OBJECTS|SVM_(?:FREE|MAP|MEMCPY|MEMFILL|UNMAP)|TASK|UNMAP_MEM_OBJECT|USER|WRITE_(?:BUFFER(?:_RECT)?|IMAGE))|COMPILER_NOT_AVAILABLE|COMPILE_PROGRAM_FAILURE|COMPLETE|CONTEXT_(?:DEVICES|INTEROP_USER_SYNC|NUM_DEVICES|PLATFORM|PROPERTIES|REFERENCE_COUNT)|DEPTH(?:_STENCIL)?|DEVICE_(?:ADDRESS_BITS|AFFINITY_DOMAIN_(?:L[1-4]_CACHE|NEXT_PARTITIONABLE|NUMA)|AVAILABLE|BUILT_IN_KERNELS|COMPILER_AVAILABLE|DOUBLE_FP_CONFIG|ENDIAN_LITTLE|ERROR_CORRECTION_SUPPORT|EXECUTION_CAPABILITIES|EXTENSIONS|GLOBAL_(?:MEM_(?:CACHELINE_SIZE|CACHE_SIZE|CACHE_TYPE|SIZE)|VARIABLE_PREFERRED_TOTAL_SIZE)|HOST_UNIFIED_MEMORY|IL_VERSION|IMAGE(?:2D_MAX_(?:HEIGHT|WIDTH)|3D_MAX_(?:DEPTH|HEIGHT|WIDTH)|_BASE_ADDRESS_ALIGNMENT|_MAX_ARRAY_SIZE|_MAX_BUFFER_SIZE|_PITCH_ALIGNMENT|_SUPPORT)|LINKER_AVAILABLE|LOCAL_MEM_SIZE|LOCAL_MEM_TYPE|MAX_(?:CLOCK_FREQUENCY|COMPUTE_UNITS|CONSTANT_ARGS|CONSTANT_BUFFER_SIZE|GLOBAL_VARIABLE_SIZE|MEM_ALLOC_SIZE|NUM_SUB_GROUPS|ON_DEVICE_(?:EVENTS|QUEUES)|PARAMETER_SIZE|PIPE_ARGS|READ_IMAGE_ARGS|READ_WRITE_IMAGE_ARGS|SAMPLERS|WORK_GROUP_SIZE|WORK_ITEM_DIMENSIONS|WORK_ITEM_SIZES|WRITE_IMAGE_ARGS)|MEM_BASE_ADDR_ALIGN|MIN_DATA_TYPE_ALIGN_SIZE|NAME|NATIVE_VECTOR_WIDTH_(?:CHAR|DOUBLE|FLOAT|HALF|INT|LONG|SHORT)|NOT_(?:AVAILABLE|FOUND)|OPENCL_C_VERSION|PARENT_DEVICE|PARTITION_(?:AFFINITY_DOMAIN|BY_AFFINITY_DOMAIN|BY_COUNTS|BY_COUNTS_LIST_END|EQUALLY|FAILED|MAX_SUB_DEVICES|PROPERTIES|TYPE)|PIPE_MAX_(?:ACTIVE_RESERVATIONS|PACKET_SIZE)|PLATFORM|PREFERRED_(?:GLOBAL_ATOMIC_ALIGNMENT|INTEROP_USER_SYNC|LOCAL_ATOMIC_ALIGNMENT|PLATFORM_ATOMIC_ALIGNMENT|VECTOR_WIDTH_(?:CHAR|DOUBLE|FLOAT|HALF|INT|LONG|SHORT))|PRINTF_BUFFER_SIZE|PROFILE|PROFILING_TIMER_RESOLUTION|QUEUE_(?:ON_(?:DEVICE_(?:MAX_SIZE|PREFERRED_SIZE|PROPERTIES)|HOST_PROPERTIES)|PROPERTIES)|REFERENCE_COUNT|SINGLE_FP_CONFIG|SUB_GROUP_INDEPENDENT_FORWARD_PROGRESS|SVM_(?:ATOMICS|CAPABILITIES|COARSE_GRAIN_BUFFER|FINE_GRAIN_BUFFER|FINE_GRAIN_SYSTEM)|TYPE(?:_ACCELERATOR|_ALL|_CPU|_CUSTOM|_DEFAULT|_GPU)?|VENDOR(?:_ID)?|VERSION)|DRIVER_VERSION|EVENT_(?:COMMAND_(?:EXECUTION_STATUS|QUEUE|TYPE)|CONTEXT|REFERENCE_COUNT)|EXEC_(?:KERNEL|NATIVE_KERNEL|STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST)|FILTER_(?:LINEAR|NEAREST)|FLOAT|FP_(?:CORRECTLY_ROUNDED_DIVIDE_SQRT|DENORM|FMA|INF_NAN|ROUND_TO_INF|ROUND_TO_NEAREST|ROUND_TO_ZERO|SOFT_FLOAT)|GLOBAL|HALF_FLOAT|IMAGE_(?:ARRAY_SIZE|BUFFER|DEPTH|ELEMENT_SIZE|FORMAT|FORMAT_MISMATCH|FORMAT_NOT_SUPPORTED|HEIGHT|NUM_MIP_LEVELS|NUM_SAMPLES|ROW_PITCH|SLICE_PITCH|WIDTH)|INTENSITY|INVALID_(?:ARG_INDEX|ARG_SIZE|ARG_VALUE|BINARY|BUFFER_SIZE|BUILD_OPTIONS|COMMAND_QUEUE|COMPILER_OPTIONS|CONTEXT|DEVICE|DEVICE_PARTITION_COUNT|DEVICE_QUEUE|DEVICE_TYPE|EVENT|EVENT_WAIT_LIST|GLOBAL_OFFSET|GLOBAL_WORK_SIZE|GL_OBJECT|HOST_PTR|IMAGE_DESCRIPTOR|IMAGE_FORMAT_DESCRIPTOR|IMAGE_SIZE|KERNEL|KERNEL_ARGS|KERNEL_DEFINITION|KERNEL_NAME|LINKER_OPTIONS|MEM_OBJECT|MIP_LEVEL|OPERATION|PIPE_SIZE|PLATFORM|PROGRAM|PROGRAM_EXECUTABLE|PROPERTY|QUEUE_PROPERTIES|SAMPLER|VALUE|WORK_DIMENSION|WORK_GROUP_SIZE|WORK_ITEM_SIZE)|KERNEL_(?:ARG_(?:ACCESS_(?:NONE|QUALIFIER|READ_ONLY|READ_WRITE|WRITE_ONLY)|ADDRESS_(?:CONSTANT|GLOBAL|LOCAL|PRIVATE|QUALIFIER)|INFO_NOT_AVAILABLE|NAME|TYPE_(?:CONST|NAME|NONE|PIPE|QUALIFIER|RESTRICT|VOLATILE))|ATTRIBUTES|COMPILE_NUM_SUB_GROUPS|COMPILE_WORK_GROUP_SIZE|CONTEXT|EXEC_INFO_SVM_FINE_GRAIN_SYSTEM|EXEC_INFO_SVM_PTRS|FUNCTION_NAME|GLOBAL_WORK_SIZE|LOCAL_MEM_SIZE|LOCAL_SIZE_FOR_SUB_GROUP_COUNT|MAX_NUM_SUB_GROUPS|MAX_SUB_GROUP_SIZE_FOR_NDRANGE|NUM_ARGS|PREFERRED_WORK_GROUP_SIZE_MULTIPLE|PRIVATE_MEM_SIZE|PROGRAM|REFERENCE_COUNT|SUB_GROUP_COUNT_FOR_NDRANGE|WORK_GROUP_SIZE)|LINKER_NOT_AVAILABLE|LINK_PROGRAM_FAILURE|LOCAL|LUMINANCE|MAP_(?:FAILURE|READ|WRITE|WRITE_INVALIDATE_REGION)|MEM_(?:ALLOC_HOST_PTR|ASSOCIATED_MEMOBJECT|CONTEXT|COPY_HOST_PTR|COPY_OVERLAP|FLAGS|HOST_NO_ACCESS|HOST_PTR|HOST_READ_ONLY|HOST_WRITE_ONLY|KERNEL_READ_AND_WRITE|MAP_COUNT|OBJECT_(?:ALLOCATION_FAILURE|BUFFER|IMAGE1D|IMAGE1D_ARRAY|IMAGE1D_BUFFER|IMAGE2D|IMAGE2D_ARRAY|IMAGE3D|PIPE)|OFFSET|READ_ONLY|READ_WRITE|REFERENCE_COUNT|SIZE|SVM_ATOMICS|SVM_FINE_GRAIN_BUFFER|TYPE|USES_SVM_POINTER|USE_HOST_PTR|WRITE_ONLY)|MIGRATE_MEM_OBJECT_(?:CONTENT_UNDEFINED|HOST)|MISALIGNED_SUB_BUFFER_OFFSET|NONE|NON_BLOCKING|OUT_OF_(?:HOST_MEMORY|RESOURCES)|PIPE_(?:MAX_PACKETS|PACKET_SIZE)|PLATFORM_(?:EXTENSIONS|HOST_TIMER_RESOLUTION|NAME|PROFILE|VENDOR|VERSION)|PROFILING_(?:COMMAND_(?:COMPLETE|END|QUEUED|START|SUBMIT)|INFO_NOT_AVAILABLE)|PROGRAM_(?:BINARIES|BINARY_SIZES|BINARY_TYPE(?:_COMPILED_OBJECT|_EXECUTABLE|_LIBRARY|_NONE)?|BUILD_(?:GLOBAL_VARIABLE_TOTAL_SIZE|LOG|OPTIONS|STATUS)|CONTEXT|DEVICES|IL|KERNEL_NAMES|NUM_DEVICES|NUM_KERNELS|REFERENCE_COUNT|SOURCE)|QUEUED|QUEUE_(?:CONTEXT|DEVICE|DEVICE_DEFAULT|ON_DEVICE|ON_DEVICE_DEFAULT|OUT_OF_ORDER_EXEC_MODE_ENABLE|PROFILING_ENABLE|PROPERTIES|REFERENCE_COUNT|SIZE)|R|RA|READ_(?:ONLY|WRITE)_CACHE|RG|RGB|RGBA|RGBx|RGx|RUNNING|Rx|SAMPLER_(?:ADDRESSING_MODE|CONTEXT|FILTER_MODE|LOD_MAX|LOD_MIN|MIP_FILTER_MODE|NORMALIZED_COORDS|REFERENCE_COUNT)|(?:UN)?SIGNED_INT(?:8|16|32)|SNORM_INT(?:8|16)|SUBMITTED|SUCCESS|UNORM_INT(?:8|16|24|_101010|_101010_2)|UNORM_SHORT_(?:555|565)|VERSION_(?:1_0|1_1|1_2|2_0|2_1)|sBGRA|sRGB|sRGBA|sRGBx)\b/, - alias: 'constant', - }, - // Extracted from cl.h (2.0) and http://streamcomputing.eu/downloads/?opencl_host.lang (opencl-host) - 'function-opencl-host': { - pattern: - /\bcl(?:BuildProgram|CloneKernel|CompileProgram|Create(?:Buffer|CommandQueue(?:WithProperties)?|Context|ContextFromType|Image|Image2D|Image3D|Kernel|KernelsInProgram|Pipe|ProgramWith(?:Binary|BuiltInKernels|IL|Source)|Sampler|SamplerWithProperties|SubBuffer|SubDevices|UserEvent)|Enqueue(?:(?:Barrier|Marker)(?:WithWaitList)?|Copy(?:Buffer(?:Rect|ToImage)?|Image(?:ToBuffer)?)|(?:Fill|Map)(?:Buffer|Image)|MigrateMemObjects|NDRangeKernel|NativeKernel|(?:Read|Write)(?:Buffer(?:Rect)?|Image)|SVM(?:Free|Map|MemFill|Memcpy|MigrateMem|Unmap)|Task|UnmapMemObject|WaitForEvents)|Finish|Flush|Get(?:CommandQueueInfo|ContextInfo|Device(?:AndHostTimer|IDs|Info)|Event(?:Profiling)?Info|ExtensionFunctionAddress(?:ForPlatform)?|HostTimer|ImageInfo|Kernel(?:ArgInfo|Info|SubGroupInfo|WorkGroupInfo)|MemObjectInfo|PipeInfo|Platform(?:IDs|Info)|Program(?:Build)?Info|SamplerInfo|SupportedImageFormats)|LinkProgram|(?:Release|Retain)(?:CommandQueue|Context|Device|Event|Kernel|MemObject|Program|Sampler)|SVM(?:Alloc|Free)|Set(?:CommandQueueProperty|DefaultDeviceCommandQueue|EventCallback|Kernel|Kernel(?:Arg(?:SVMPointer)?|ExecInfo)|MemObjectDestructorCallback|UserEventStatus)|Unload(?:Platform)?Compiler|WaitForEvents)\b/, - alias: 'function', + $insertBefore: { + 'keyword': { + // Extracted from http://streamcomputing.eu/downloads/?opencl_host.lang (opencl-types and opencl-host) + 'type-opencl-host': { + pattern: + /\b(?:cl_(?:GLenum|GLint|GLuin|addressing_mode|bitfield|bool|buffer_create_type|build_status|channel_(?:order|type)|(?:u?(?:char|int|long|short)|double|float)(?:2|3|4|8|16)?|command_(?:queue(?:_info|_properties)?|type)|context(?:_info|_properties)?|device_(?:exec_capabilities|fp_config|id|info|local_mem_type|mem_cache_type|type)|(?:event|sampler)(?:_info)?|filter_mode|half|image_info|kernel(?:_info|_work_group_info)?|map_flags|mem(?:_flags|_info|_object_type)?|platform_(?:id|info)|profiling_info|program(?:_build_info|_info)?))\b/, + alias: 'keyword', + }, + 'boolean-opencl-host': { + pattern: /\bCL_(?:FALSE|TRUE)\b/, + alias: 'boolean', + }, + // Extracted from cl.h (2.0) and http://streamcomputing.eu/downloads/?opencl_host.lang (opencl-const) + 'constant-opencl-host': { + pattern: + /\bCL_(?:A|ABGR|ADDRESS_(?:CLAMP(?:_TO_EDGE)?|MIRRORED_REPEAT|NONE|REPEAT)|ARGB|BGRA|BLOCKING|BUFFER_CREATE_TYPE_REGION|BUILD_(?:ERROR|IN_PROGRESS|NONE|PROGRAM_FAILURE|SUCCESS)|COMMAND_(?:ACQUIRE_GL_OBJECTS|BARRIER|COPY_(?:BUFFER(?:_RECT|_TO_IMAGE)?|IMAGE(?:_TO_BUFFER)?)|FILL_(?:BUFFER|IMAGE)|MAP(?:_BUFFER|_IMAGE)|MARKER|MIGRATE(?:_SVM)?_MEM_OBJECTS|NATIVE_KERNEL|NDRANGE_KERNEL|READ_(?:BUFFER(?:_RECT)?|IMAGE)|RELEASE_GL_OBJECTS|SVM_(?:FREE|MAP|MEMCPY|MEMFILL|UNMAP)|TASK|UNMAP_MEM_OBJECT|USER|WRITE_(?:BUFFER(?:_RECT)?|IMAGE))|COMPILER_NOT_AVAILABLE|COMPILE_PROGRAM_FAILURE|COMPLETE|CONTEXT_(?:DEVICES|INTEROP_USER_SYNC|NUM_DEVICES|PLATFORM|PROPERTIES|REFERENCE_COUNT)|DEPTH(?:_STENCIL)?|DEVICE_(?:ADDRESS_BITS|AFFINITY_DOMAIN_(?:L[1-4]_CACHE|NEXT_PARTITIONABLE|NUMA)|AVAILABLE|BUILT_IN_KERNELS|COMPILER_AVAILABLE|DOUBLE_FP_CONFIG|ENDIAN_LITTLE|ERROR_CORRECTION_SUPPORT|EXECUTION_CAPABILITIES|EXTENSIONS|GLOBAL_(?:MEM_(?:CACHELINE_SIZE|CACHE_SIZE|CACHE_TYPE|SIZE)|VARIABLE_PREFERRED_TOTAL_SIZE)|HOST_UNIFIED_MEMORY|IL_VERSION|IMAGE(?:2D_MAX_(?:HEIGHT|WIDTH)|3D_MAX_(?:DEPTH|HEIGHT|WIDTH)|_BASE_ADDRESS_ALIGNMENT|_MAX_ARRAY_SIZE|_MAX_BUFFER_SIZE|_PITCH_ALIGNMENT|_SUPPORT)|LINKER_AVAILABLE|LOCAL_MEM_SIZE|LOCAL_MEM_TYPE|MAX_(?:CLOCK_FREQUENCY|COMPUTE_UNITS|CONSTANT_ARGS|CONSTANT_BUFFER_SIZE|GLOBAL_VARIABLE_SIZE|MEM_ALLOC_SIZE|NUM_SUB_GROUPS|ON_DEVICE_(?:EVENTS|QUEUES)|PARAMETER_SIZE|PIPE_ARGS|READ_IMAGE_ARGS|READ_WRITE_IMAGE_ARGS|SAMPLERS|WORK_GROUP_SIZE|WORK_ITEM_DIMENSIONS|WORK_ITEM_SIZES|WRITE_IMAGE_ARGS)|MEM_BASE_ADDR_ALIGN|MIN_DATA_TYPE_ALIGN_SIZE|NAME|NATIVE_VECTOR_WIDTH_(?:CHAR|DOUBLE|FLOAT|HALF|INT|LONG|SHORT)|NOT_(?:AVAILABLE|FOUND)|OPENCL_C_VERSION|PARENT_DEVICE|PARTITION_(?:AFFINITY_DOMAIN|BY_AFFINITY_DOMAIN|BY_COUNTS|BY_COUNTS_LIST_END|EQUALLY|FAILED|MAX_SUB_DEVICES|PROPERTIES|TYPE)|PIPE_MAX_(?:ACTIVE_RESERVATIONS|PACKET_SIZE)|PLATFORM|PREFERRED_(?:GLOBAL_ATOMIC_ALIGNMENT|INTEROP_USER_SYNC|LOCAL_ATOMIC_ALIGNMENT|PLATFORM_ATOMIC_ALIGNMENT|VECTOR_WIDTH_(?:CHAR|DOUBLE|FLOAT|HALF|INT|LONG|SHORT))|PRINTF_BUFFER_SIZE|PROFILE|PROFILING_TIMER_RESOLUTION|QUEUE_(?:ON_(?:DEVICE_(?:MAX_SIZE|PREFERRED_SIZE|PROPERTIES)|HOST_PROPERTIES)|PROPERTIES)|REFERENCE_COUNT|SINGLE_FP_CONFIG|SUB_GROUP_INDEPENDENT_FORWARD_PROGRESS|SVM_(?:ATOMICS|CAPABILITIES|COARSE_GRAIN_BUFFER|FINE_GRAIN_BUFFER|FINE_GRAIN_SYSTEM)|TYPE(?:_ACCELERATOR|_ALL|_CPU|_CUSTOM|_DEFAULT|_GPU)?|VENDOR(?:_ID)?|VERSION)|DRIVER_VERSION|EVENT_(?:COMMAND_(?:EXECUTION_STATUS|QUEUE|TYPE)|CONTEXT|REFERENCE_COUNT)|EXEC_(?:KERNEL|NATIVE_KERNEL|STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST)|FILTER_(?:LINEAR|NEAREST)|FLOAT|FP_(?:CORRECTLY_ROUNDED_DIVIDE_SQRT|DENORM|FMA|INF_NAN|ROUND_TO_INF|ROUND_TO_NEAREST|ROUND_TO_ZERO|SOFT_FLOAT)|GLOBAL|HALF_FLOAT|IMAGE_(?:ARRAY_SIZE|BUFFER|DEPTH|ELEMENT_SIZE|FORMAT|FORMAT_MISMATCH|FORMAT_NOT_SUPPORTED|HEIGHT|NUM_MIP_LEVELS|NUM_SAMPLES|ROW_PITCH|SLICE_PITCH|WIDTH)|INTENSITY|INVALID_(?:ARG_INDEX|ARG_SIZE|ARG_VALUE|BINARY|BUFFER_SIZE|BUILD_OPTIONS|COMMAND_QUEUE|COMPILER_OPTIONS|CONTEXT|DEVICE|DEVICE_PARTITION_COUNT|DEVICE_QUEUE|DEVICE_TYPE|EVENT|EVENT_WAIT_LIST|GLOBAL_OFFSET|GLOBAL_WORK_SIZE|GL_OBJECT|HOST_PTR|IMAGE_DESCRIPTOR|IMAGE_FORMAT_DESCRIPTOR|IMAGE_SIZE|KERNEL|KERNEL_ARGS|KERNEL_DEFINITION|KERNEL_NAME|LINKER_OPTIONS|MEM_OBJECT|MIP_LEVEL|OPERATION|PIPE_SIZE|PLATFORM|PROGRAM|PROGRAM_EXECUTABLE|PROPERTY|QUEUE_PROPERTIES|SAMPLER|VALUE|WORK_DIMENSION|WORK_GROUP_SIZE|WORK_ITEM_SIZE)|KERNEL_(?:ARG_(?:ACCESS_(?:NONE|QUALIFIER|READ_ONLY|READ_WRITE|WRITE_ONLY)|ADDRESS_(?:CONSTANT|GLOBAL|LOCAL|PRIVATE|QUALIFIER)|INFO_NOT_AVAILABLE|NAME|TYPE_(?:CONST|NAME|NONE|PIPE|QUALIFIER|RESTRICT|VOLATILE))|ATTRIBUTES|COMPILE_NUM_SUB_GROUPS|COMPILE_WORK_GROUP_SIZE|CONTEXT|EXEC_INFO_SVM_FINE_GRAIN_SYSTEM|EXEC_INFO_SVM_PTRS|FUNCTION_NAME|GLOBAL_WORK_SIZE|LOCAL_MEM_SIZE|LOCAL_SIZE_FOR_SUB_GROUP_COUNT|MAX_NUM_SUB_GROUPS|MAX_SUB_GROUP_SIZE_FOR_NDRANGE|NUM_ARGS|PREFERRED_WORK_GROUP_SIZE_MULTIPLE|PRIVATE_MEM_SIZE|PROGRAM|REFERENCE_COUNT|SUB_GROUP_COUNT_FOR_NDRANGE|WORK_GROUP_SIZE)|LINKER_NOT_AVAILABLE|LINK_PROGRAM_FAILURE|LOCAL|LUMINANCE|MAP_(?:FAILURE|READ|WRITE|WRITE_INVALIDATE_REGION)|MEM_(?:ALLOC_HOST_PTR|ASSOCIATED_MEMOBJECT|CONTEXT|COPY_HOST_PTR|COPY_OVERLAP|FLAGS|HOST_NO_ACCESS|HOST_PTR|HOST_READ_ONLY|HOST_WRITE_ONLY|KERNEL_READ_AND_WRITE|MAP_COUNT|OBJECT_(?:ALLOCATION_FAILURE|BUFFER|IMAGE1D|IMAGE1D_ARRAY|IMAGE1D_BUFFER|IMAGE2D|IMAGE2D_ARRAY|IMAGE3D|PIPE)|OFFSET|READ_ONLY|READ_WRITE|REFERENCE_COUNT|SIZE|SVM_ATOMICS|SVM_FINE_GRAIN_BUFFER|TYPE|USES_SVM_POINTER|USE_HOST_PTR|WRITE_ONLY)|MIGRATE_MEM_OBJECT_(?:CONTENT_UNDEFINED|HOST)|MISALIGNED_SUB_BUFFER_OFFSET|NONE|NON_BLOCKING|OUT_OF_(?:HOST_MEMORY|RESOURCES)|PIPE_(?:MAX_PACKETS|PACKET_SIZE)|PLATFORM_(?:EXTENSIONS|HOST_TIMER_RESOLUTION|NAME|PROFILE|VENDOR|VERSION)|PROFILING_(?:COMMAND_(?:COMPLETE|END|QUEUED|START|SUBMIT)|INFO_NOT_AVAILABLE)|PROGRAM_(?:BINARIES|BINARY_SIZES|BINARY_TYPE(?:_COMPILED_OBJECT|_EXECUTABLE|_LIBRARY|_NONE)?|BUILD_(?:GLOBAL_VARIABLE_TOTAL_SIZE|LOG|OPTIONS|STATUS)|CONTEXT|DEVICES|IL|KERNEL_NAMES|NUM_DEVICES|NUM_KERNELS|REFERENCE_COUNT|SOURCE)|QUEUED|QUEUE_(?:CONTEXT|DEVICE|DEVICE_DEFAULT|ON_DEVICE|ON_DEVICE_DEFAULT|OUT_OF_ORDER_EXEC_MODE_ENABLE|PROFILING_ENABLE|PROPERTIES|REFERENCE_COUNT|SIZE)|R|RA|READ_(?:ONLY|WRITE)_CACHE|RG|RGB|RGBA|RGBx|RGx|RUNNING|Rx|SAMPLER_(?:ADDRESSING_MODE|CONTEXT|FILTER_MODE|LOD_MAX|LOD_MIN|MIP_FILTER_MODE|NORMALIZED_COORDS|REFERENCE_COUNT)|(?:UN)?SIGNED_INT(?:8|16|32)|SNORM_INT(?:8|16)|SUBMITTED|SUCCESS|UNORM_INT(?:8|16|24|_101010|_101010_2)|UNORM_SHORT_(?:555|565)|VERSION_(?:1_0|1_1|1_2|2_0|2_1)|sBGRA|sRGB|sRGBA|sRGBx)\b/, + alias: 'constant', + }, + // Extracted from cl.h (2.0) and http://streamcomputing.eu/downloads/?opencl_host.lang (opencl-host) + 'function-opencl-host': { + pattern: + /\bcl(?:BuildProgram|CloneKernel|CompileProgram|Create(?:Buffer|CommandQueue(?:WithProperties)?|Context|ContextFromType|Image|Image2D|Image3D|Kernel|KernelsInProgram|Pipe|ProgramWith(?:Binary|BuiltInKernels|IL|Source)|Sampler|SamplerWithProperties|SubBuffer|SubDevices|UserEvent)|Enqueue(?:(?:Barrier|Marker)(?:WithWaitList)?|Copy(?:Buffer(?:Rect|ToImage)?|Image(?:ToBuffer)?)|(?:Fill|Map)(?:Buffer|Image)|MigrateMemObjects|NDRangeKernel|NativeKernel|(?:Read|Write)(?:Buffer(?:Rect)?|Image)|SVM(?:Free|Map|MemFill|Memcpy|MigrateMem|Unmap)|Task|UnmapMemObject|WaitForEvents)|Finish|Flush|Get(?:CommandQueueInfo|ContextInfo|Device(?:AndHostTimer|IDs|Info)|Event(?:Profiling)?Info|ExtensionFunctionAddress(?:ForPlatform)?|HostTimer|ImageInfo|Kernel(?:ArgInfo|Info|SubGroupInfo|WorkGroupInfo)|MemObjectInfo|PipeInfo|Platform(?:IDs|Info)|Program(?:Build)?Info|SamplerInfo|SupportedImageFormats)|LinkProgram|(?:Release|Retain)(?:CommandQueue|Context|Device|Event|Kernel|MemObject|Program|Sampler)|SVM(?:Alloc|Free)|Set(?:CommandQueueProperty|DefaultDeviceCommandQueue|EventCallback|Kernel|Kernel(?:Arg(?:SVMPointer)?|ExecInfo)|MemObjectDestructorCallback|UserEventStatus)|Unload(?:Platform)?Compiler|WaitForEvents)\b/, + alias: 'function', + }, + }, }, + }, +} as LanguageProto<'opencl-extensions'>; - // C++ includes everything from the OpenCL C host API plus the classes defined in cl2.h - // Extracted from doxygen class list http://github.khronos.org/OpenCL-CLHPP/annotated.html - 'type-opencl-host-cpp': { - pattern: - /\b(?:Buffer|BufferGL|BufferRenderGL|CommandQueue|Context|Device|DeviceCommandQueue|EnqueueArgs|Event|Image|Image1D|Image1DArray|Image1DBuffer|Image2D|Image2DArray|Image2DGL|Image3D|Image3DGL|ImageFormat|ImageGL|Kernel|KernelFunctor|LocalSpaceArg|Memory|NDRange|Pipe|Platform|Program|SVMAllocator|SVMTraitAtomic|SVMTraitCoarse|SVMTraitFine|SVMTraitReadOnly|SVMTraitReadWrite|SVMTraitWriteOnly|Sampler|UserEvent)\b/, - alias: 'keyword', +export const cpp = { + id: 'opencl-extensions', + base: c, + extends: 'cpp', + grammar: { + $insertBefore: { + 'keyword': { + // C++ includes everything from the OpenCL C host API plus the classes defined in cl2.h + // Extracted from doxygen class list http://github.khronos.org/OpenCL-CLHPP/annotated.html + 'type-opencl-host-cpp': { + pattern: + /\b(?:Buffer|BufferGL|BufferRenderGL|CommandQueue|Context|Device|DeviceCommandQueue|EnqueueArgs|Event|Image|Image1D|Image1DArray|Image1DBuffer|Image2D|Image2DArray|Image2DGL|Image3D|Image3DGL|ImageFormat|ImageGL|Kernel|KernelFunctor|LocalSpaceArg|Memory|NDRange|Pipe|Platform|Program|SVMAllocator|SVMTraitAtomic|SVMTraitCoarse|SVMTraitFine|SVMTraitReadOnly|SVMTraitReadWrite|SVMTraitWriteOnly|Sampler|UserEvent)\b/, + alias: 'keyword', + }, + }, }, }, } as LanguageProto<'opencl-extensions'>; diff --git a/src/languages/opencl.ts b/src/languages/opencl.ts index 8d2e1e3695..b49cf4f76d 100644 --- a/src/languages/opencl.ts +++ b/src/languages/opencl.ts @@ -1,13 +1,12 @@ -import { insertBefore } from '../util/language-util'; import c from './c'; import type { LanguageProto } from '../types'; export default { id: 'opencl', - require: c, - grammar ({ extend }) { + base: c, + grammar () { /* OpenCL kernel language */ - const opencl = extend('c', { + return { // Extracted from the official specs (2.0) and http://streamcomputing.eu/downloads/?opencl.lang (opencl-keywords, opencl-types) and http://sourceforge.net/tracker/?func=detail&aid=2957794&group_id=95717&atid=612384 (Words2, partly Words3) 'keyword': /\b(?:__attribute__|(?:__)?(?:constant|global|kernel|local|private|read_only|read_write|write_only)|auto|(?:bool|u?(?:char|int|long|short)|half|quad)(?:2|3|4|8|16)?|break|case|complex|const|continue|default|do|(?:double|float)(?:16(?:x(?:1|2|4|8|16))?|1x(?:1|2|4|8|16)|2(?:x(?:1|2|4|8|16))?|3|4(?:x(?:1|2|4|8|16))?|8(?:x(?:1|2|4|8|16))?)?|else|enum|extern|for|goto|if|imaginary|inline|packed|pipe|register|restrict|return|signed|sizeof|static|struct|switch|typedef|uniform|union|unsigned|void|volatile|while)\b/, @@ -22,18 +21,16 @@ export default { /\b(?:CHAR_(?:BIT|MAX|MIN)|CLK_(?:ADDRESS_(?:CLAMP(?:_TO_EDGE)?|NONE|REPEAT)|FILTER_(?:LINEAR|NEAREST)|(?:GLOBAL|LOCAL)_MEM_FENCE|NORMALIZED_COORDS_(?:FALSE|TRUE))|CL_(?:A?R?G?B?[Ax]?|BGRA|(?:HALF_)?FLOAT|INTENSITY|LUMINANCE|(?:(?:UN)?SIGNED|[US]NORM)_(?:INT(?:8|16|32))|UNORM_(?:INT_101010|SHORT_(?:555|565)))|(?:DBL|FLT|HALF)_(?:DIG|EPSILON|MANT_DIG|(?:MAX|MIN)(?:(?:_10)?_EXP)?)|FLT_RADIX|HUGE_VALF?|INFINITY|(?:INT|LONG|SCHAR|SHRT)_(?:MAX|MIN)|MAXFLOAT|M_(?:[12]_PI|2_SQRTPI|E|LN(?:2|10)|LOG(?:2|10)E?|PI(?:_[24])?|SQRT(?:1_2|2))(?:_F|_H)?|NAN|(?:UCHAR|UINT|ULONG|USHRT)_MAX)\b/, alias: 'constant', }, - }); - - insertBefore(opencl, 'class-name', { - // https://www.khronos.org/registry/OpenCL/sdk/2.1/docs/man/xhtml/scalarDataTypes.html - // https://www.khronos.org/registry/OpenCL/sdk/2.1/docs/man/xhtml/otherDataTypes.html - 'builtin-type': { - pattern: - /\b(?:_cl_(?:command_queue|context|device_id|event|kernel|mem|platform_id|program|sampler)|cl_(?:image_format|mem_fence_flags)|clk_event_t|event_t|image(?:1d_(?:array_|buffer_)?t|2d_(?:array_(?:depth_|msaa_|msaa_depth_)?|depth_|msaa_|msaa_depth_)?t|3d_t)|intptr_t|ndrange_t|ptrdiff_t|queue_t|reserve_id_t|sampler_t|size_t|uintptr_t)\b/, - alias: 'keyword', + $insert: { + // https://www.khronos.org/registry/OpenCL/sdk/2.1/docs/man/xhtml/scalarDataTypes.html + // https://www.khronos.org/registry/OpenCL/sdk/2.1/docs/man/xhtml/otherDataTypes.html + 'builtin-type': { + $before: 'class-name', + pattern: + /\b(?:_cl_(?:command_queue|context|device_id|event|kernel|mem|platform_id|program|sampler)|cl_(?:image_format|mem_fence_flags)|clk_event_t|event_t|image(?:1d_(?:array_|buffer_)?t|2d_(?:array_(?:depth_|msaa_|msaa_depth_)?|depth_|msaa_|msaa_depth_)?t|3d_t)|intptr_t|ndrange_t|ptrdiff_t|queue_t|reserve_id_t|sampler_t|size_t|uintptr_t)\b/, + alias: 'keyword', + }, }, - }); - - return opencl; + }; }, } as LanguageProto<'opencl'>; diff --git a/src/languages/parser.ts b/src/languages/parser.ts index 20c08ab47f..2f8b8f82f0 100644 --- a/src/languages/parser.ts +++ b/src/languages/parser.ts @@ -1,14 +1,13 @@ -import { insertBefore } from '../util/language-util'; import markup from './markup'; -import type { Grammar, GrammarToken, LanguageProto } from '../types'; +import type { LanguageProto } from '../types'; export default { id: 'parser', - require: markup, - grammar ({ extend }) { + base: markup, + grammar ({ base }) { const punctuation = /[\[\](){};]/; - const parser = extend('markup', { + return { 'keyword': { pattern: /(^|[^^])(?:\^(?:case|eval|for|if|switch|throw)\b|@(?:BASE|CLASS|GET(?:_DEFAULT)?|OPTIONS|SET_DEFAULT|USE)\b)/, @@ -37,54 +36,49 @@ export default { alias: 'builtin', }, 'punctuation': punctuation, - }); - - insertBefore(parser, 'keyword', { - 'parser-comment': { - pattern: /(\s)#.*/, - lookbehind: true, - alias: 'comment', - }, - 'expression': { - // Allow for 3 levels of depth - pattern: /(^|[^^])\((?:[^()]|\((?:[^()]|\((?:[^()])*\))*\))*\)/, - greedy: true, - lookbehind: true, - inside: { - 'string': { - pattern: /(^|[^^])(["'])(?:(?!\2)[^^]|\^[\s\S])*\2/, + $insertBefore: { + 'keyword': { + 'parser-comment': { + pattern: /(\s)#.*/, + lookbehind: true, + alias: 'comment', + }, + 'expression': { + // Allow for 3 levels of depth + pattern: /(^|[^^])\((?:[^()]|\((?:[^()]|\((?:[^()])*\))*\))*\)/, + greedy: true, lookbehind: true, + inside: { + 'string': { + pattern: /(^|[^^])(["'])(?:(?!\2)[^^]|\^[\s\S])*\2/, + lookbehind: true, + }, + 'keyword': base!.keyword, + 'variable': base!.variable, + 'function': base!.function, + 'boolean': /\b(?:false|true)\b/, + 'number': /\b(?:0x[a-f\d]+|\d+(?:\.\d*)?(?:e[+-]?\d+)?)\b/i, + 'escape': base!.escape, + 'operator': + /[~+*\/\\%]|!(?:\|\|?|=)?|&&?|\|\|?|==|<[<=]?|>[>=]?|-[fd]?|\b(?:def|eq|ge|gt|in|is|le|lt|ne)\b/, + 'punctuation': punctuation, + }, }, - 'keyword': parser.keyword, - 'variable': parser.variable, - 'function': parser.function, - 'boolean': /\b(?:false|true)\b/, - 'number': /\b(?:0x[a-f\d]+|\d+(?:\.\d*)?(?:e[+-]?\d+)?)\b/i, - 'escape': parser.escape, - 'operator': - /[~+*\/\\%]|!(?:\|\|?|=)?|&&?|\|\|?|==|<[<=]?|>[>=]?|-[fd]?|\b(?:def|eq|ge|gt|in|is|le|lt|ne)\b/, - 'punctuation': punctuation, }, - }, - }); - - insertBefore( - (((parser['tag'] as GrammarToken).inside as Grammar)['attr-value'] as GrammarToken) - .inside as Grammar, - 'punctuation', - { - 'expression': parser.expression, - 'keyword': parser.keyword, - 'variable': parser.variable, - 'function': parser.function, - 'escape': parser.escape, - 'parser-punctuation': { - pattern: punctuation, - alias: 'punctuation', + 'tag/attr-value': { + 'punctuation': { + 'expression': base!.expression, + 'keyword': base!.keyword, + 'variable': base!.variable, + 'function': base!.function, + 'escape': base!.escape, + 'parser-punctuation': { + pattern: punctuation, + alias: 'punctuation', + }, + }, }, - } - ); - - return parser; + }, + }; }, } as LanguageProto<'parser'>; diff --git a/src/languages/php-extras.ts b/src/languages/php-extras.ts deleted file mode 100644 index e37969abf6..0000000000 --- a/src/languages/php-extras.ts +++ /dev/null @@ -1,20 +0,0 @@ -import type { LanguageProto } from '../types'; - -export default { - id: 'php-extras', - grammar: { - 'this': { - pattern: /\$this\b/, - alias: 'keyword', - }, - 'global': - /\$(?:GLOBALS|HTTP_RAW_POST_DATA|_(?:COOKIE|ENV|FILES|GET|POST|REQUEST|SERVER|SESSION)|argc|argv|http_response_header|php_errormsg)\b/, - 'scope': { - pattern: /\b[\w\\]+::/, - inside: { - 'keyword': /\b(?:parent|self|static)\b/, - 'punctuation': /::|\\/, - }, - }, - }, -} as LanguageProto<'php-extras'>; diff --git a/src/languages/php.ts b/src/languages/php.ts index f3c8c4d9d4..19b566b68d 100644 --- a/src/languages/php.ts +++ b/src/languages/php.ts @@ -1,14 +1,11 @@ import { embeddedIn } from '../shared/languages/templating'; -import { insertBefore } from '../util/language-util'; import markup from './markup'; -import type { Prism } from '../core/prism'; -import type { Grammar, LanguageProto } from '../types'; +import type { Grammar, LanguageProto, Prism } from '../types'; export default { id: 'php', - require: markup, - optional: 'php-extras', - grammar ({ getOptionalLanguage }) { + base: markup, + grammar (): Grammar { /** * Original by Aaron Harun: http://aahacreative.com/2012/07/31/php-syntax-highlighting-prism/ * Modified by Miles Johnson: http://milesj.me @@ -53,6 +50,19 @@ export default { inside: 'phpdoc', }, 'comment': comment, + 'this': { + pattern: /\$this\b/, + alias: 'keyword', + }, + 'global': + /\$(?:GLOBALS|HTTP_RAW_POST_DATA|_(?:COOKIE|ENV|FILES|GET|POST|REQUEST|SERVER|SESSION)|argc|argv|http_response_header|php_errormsg)\b/, + 'scope': { + pattern: /\b[\w\\]+::/, + inside: { + 'keyword': /\b(?:parent|self|static)\b/, + 'punctuation': /::|\\/, + }, + }, 'variable': /\$+(?:\w+\b|(?=\{))/, 'package': { pattern: /(namespace\s+|use\s+(?:function\s+)?)(?:\\?\b[a-z_]\w*)+\b(?!\\)/i, @@ -306,63 +316,60 @@ export default { }, ]; - insertBefore(php, 'variable', { - 'string': string, - 'attribute': { - pattern: - /#\[(?:[^"'\/#]|\/(?![*/])|\/\/.*$|#(?!\[).*$|\/\*(?:[^*]|\*(?!\/))*\*\/|"(?:\\[\s\S]|[^\\"])*"|'(?:\\[\s\S]|[^\\'])*')+\](?=\s*[a-z$#])/im, - greedy: true, - inside: { - 'attribute-content': { - pattern: /^(#\[)[\s\S]+(?=\]$)/, - lookbehind: true, - // inside can appear subset of php - inside: { - 'comment': comment, - 'string': string, - 'attribute-class-name': [ - { - pattern: /([^:]|^)\b[a-z_]\w*(?!\\)\b/i, - alias: 'class-name', - greedy: true, - lookbehind: true, - }, - { - pattern: /([^:]|^)(?:\\?\b[a-z_]\w*)+/i, - alias: ['class-name', 'class-name-fully-qualified'], - greedy: true, - lookbehind: true, - inside: { - 'punctuation': /\\/, - }, - }, - ], - 'constant': constant, - 'number': number, - 'operator': operator, - 'punctuation': punctuation, - }, - }, - 'delimiter': { - pattern: /^#\[|\]$/, - alias: 'punctuation', - }, - }, - }, - }); - - const extras = getOptionalLanguage('php-extras'); - if (extras) { - insertBefore(php, 'variable', extras); - } - const embedded = embeddedIn('markup'); return { + ...php, 'php': { pattern: /<\?(?:[^"'/#]|\/(?![*/])|("|')(?:\\[\s\S]|(?!\1)[^\\])*\1|(?:\/\/|#(?!\[))(?:[^?\n\r]|\?(?!>))*(?=$|\?>|[\r\n])|#\[|\/\*(?:[^*]|\*(?!\/))*(?:\*\/|$))*?(?:\?>|$)/, inside: php, }, + $insertBefore: { + 'variable': { + 'string': string, + 'attribute': { + pattern: + /#\[(?:[^"'\/#]|\/(?![*/])|\/\/.*$|#(?!\[).*$|\/\*(?:[^*]|\*(?!\/))*\*\/|"(?:\\[\s\S]|[^\\"])*"|'(?:\\[\s\S]|[^\\'])*')+\](?=\s*[a-z$#])/im, + greedy: true, + inside: { + 'attribute-content': { + pattern: /^(#\[)[\s\S]+(?=\]$)/, + lookbehind: true, + // inside can appear subset of php + inside: { + 'comment': comment, + 'string': string, + 'attribute-class-name': [ + { + pattern: /([^:]|^)\b[a-z_]\w*(?!\\)\b/i, + alias: 'class-name', + greedy: true, + lookbehind: true, + }, + { + pattern: /([^:]|^)(?:\\?\b[a-z_]\w*)+/i, + alias: ['class-name', 'class-name-fully-qualified'], + greedy: true, + lookbehind: true, + inside: { + 'punctuation': /\\/, + }, + }, + ], + 'constant': constant, + 'number': number, + 'operator': operator, + 'punctuation': punctuation, + }, + }, + 'delimiter': { + pattern: /^#\[|\]$/, + alias: 'punctuation', + }, + }, + }, + }, + }, $tokenize: (code: string, grammar: Grammar, Prism: Prism) => { if (!/<\?/.test(code)) { return Prism.tokenize(code, php); diff --git a/src/languages/phpdoc.ts b/src/languages/phpdoc.ts index 8fec95eeb7..4e98eff6c2 100644 --- a/src/languages/phpdoc.ts +++ b/src/languages/phpdoc.ts @@ -1,14 +1,13 @@ -import { insertBefore } from '../util/language-util'; import javadoclike from './javadoclike'; import type { LanguageProto } from '../types'; export default { id: 'phpdoc', - require: javadoclike, - grammar ({ extend }) { + base: javadoclike, + grammar () { const typeExpression = /(?:\b[a-zA-Z]\w*|[|\\[\]])+/.source; - const phpdoc = extend('javadoclike', { + return { 'parameter': { pattern: RegExp( '(@(?:global|param|property(?:-read|-write)?|var)\\s+(?:' + @@ -17,11 +16,9 @@ export default { ), lookbehind: true, }, - }); - - insertBefore(phpdoc, 'keyword', { - 'class-name': [ - { + $insert: { + 'class-name': { + $before: 'keyword', pattern: RegExp( '(@(?:global|package|param|property(?:-read|-write)?|return|subpackage|throws|var)\\s+)' + typeExpression @@ -33,9 +30,7 @@ export default { 'punctuation': /[|\\[\]()]/, }, }, - ], - }); - - return phpdoc; + }, + }; }, } as LanguageProto<'phpdoc'>; diff --git a/src/languages/plsql.ts b/src/languages/plsql.ts index 422b3867c0..99aa09ffba 100644 --- a/src/languages/plsql.ts +++ b/src/languages/plsql.ts @@ -1,12 +1,11 @@ -import { insertBefore } from '../util/language-util'; import sql from './sql'; import type { LanguageProto } from '../types'; export default { id: 'plsql', - require: sql, - grammar ({ extend }) { - const plsql = extend('sql', { + base: sql, + grammar () { + return { 'comment': { pattern: /\/\*[\s\S]*?\*\/|--.*/, greedy: true, @@ -16,15 +15,13 @@ export default { /\b(?:A|ACCESSIBLE|ADD|AGENT|AGGREGATE|ALL|ALTER|AND|ANY|ARRAY|AS|ASC|AT|ATTRIBUTE|AUTHID|AVG|BEGIN|BETWEEN|BFILE_BASE|BINARY|BLOB_BASE|BLOCK|BODY|BOTH|BOUND|BULK|BY|BYTE|C|CALL|CALLING|CASCADE|CASE|CHAR|CHARACTER|CHARSET|CHARSETFORM|CHARSETID|CHAR_BASE|CHECK|CLOB_BASE|CLONE|CLOSE|CLUSTER|CLUSTERS|COLAUTH|COLLECT|COLUMNS|COMMENT|COMMIT|COMMITTED|COMPILED|COMPRESS|CONNECT|CONSTANT|CONSTRUCTOR|CONTEXT|CONTINUE|CONVERT|COUNT|CRASH|CREATE|CREDENTIAL|CURRENT|CURSOR|CUSTOMDATUM|DANGLING|DATA|DATE|DATE_BASE|DAY|DECLARE|DEFAULT|DEFINE|DELETE|DESC|DETERMINISTIC|DIRECTORY|DISTINCT|DOUBLE|DROP|DURATION|ELEMENT|ELSE|ELSIF|EMPTY|END|ESCAPE|EXCEPT|EXCEPTION|EXCEPTIONS|EXCLUSIVE|EXECUTE|EXISTS|EXIT|EXTERNAL|FETCH|FINAL|FIRST|FIXED|FLOAT|FOR|FORALL|FORCE|FROM|FUNCTION|GENERAL|GOTO|GRANT|GROUP|HASH|HAVING|HEAP|HIDDEN|HOUR|IDENTIFIED|IF|IMMEDIATE|IMMUTABLE|IN|INCLUDING|INDEX|INDEXES|INDICATOR|INDICES|INFINITE|INSERT|INSTANTIABLE|INT|INTERFACE|INTERSECT|INTERVAL|INTO|INVALIDATE|IS|ISOLATION|JAVA|LANGUAGE|LARGE|LEADING|LENGTH|LEVEL|LIBRARY|LIKE|LIKE2|LIKE4|LIKEC|LIMIT|LIMITED|LOCAL|LOCK|LONG|LOOP|MAP|MAX|MAXLEN|MEMBER|MERGE|MIN|MINUS|MINUTE|MOD|MODE|MODIFY|MONTH|MULTISET|MUTABLE|NAME|NAN|NATIONAL|NATIVE|NCHAR|NEW|NOCOMPRESS|NOCOPY|NOT|NOWAIT|NULL|NUMBER_BASE|OBJECT|OCICOLL|OCIDATE|OCIDATETIME|OCIDURATION|OCIINTERVAL|OCILOBLOCATOR|OCINUMBER|OCIRAW|OCIREF|OCIREFCURSOR|OCIROWID|OCISTRING|OCITYPE|OF|OLD|ON|ONLY|OPAQUE|OPEN|OPERATOR|OPTION|OR|ORACLE|ORADATA|ORDER|ORGANIZATION|ORLANY|ORLVARY|OTHERS|OUT|OVERLAPS|OVERRIDING|PACKAGE|PARALLEL_ENABLE|PARAMETER|PARAMETERS|PARENT|PARTITION|PASCAL|PERSISTABLE|PIPE|PIPELINED|PLUGGABLE|POLYMORPHIC|PRAGMA|PRECISION|PRIOR|PRIVATE|PROCEDURE|PUBLIC|RAISE|RANGE|RAW|READ|RECORD|REF|REFERENCE|RELIES_ON|REM|REMAINDER|RENAME|RESOURCE|RESULT|RESULT_CACHE|RETURN|RETURNING|REVERSE|REVOKE|ROLLBACK|ROW|SAMPLE|SAVE|SAVEPOINT|SB1|SB2|SB4|SECOND|SEGMENT|SELECT|SELF|SEPARATE|SEQUENCE|SERIALIZABLE|SET|SHARE|SHORT|SIZE|SIZE_T|SOME|SPARSE|SQL|SQLCODE|SQLDATA|SQLNAME|SQLSTATE|STANDARD|START|STATIC|STDDEV|STORED|STRING|STRUCT|STYLE|SUBMULTISET|SUBPARTITION|SUBSTITUTABLE|SUBTYPE|SUM|SYNONYM|TABAUTH|TABLE|TDO|THE|THEN|TIME|TIMESTAMP|TIMEZONE_ABBR|TIMEZONE_HOUR|TIMEZONE_MINUTE|TIMEZONE_REGION|TO|TRAILING|TRANSACTION|TRANSACTIONAL|TRUSTED|TYPE|UB1|UB2|UB4|UNDER|UNION|UNIQUE|UNPLUG|UNSIGNED|UNTRUSTED|UPDATE|USE|USING|VALIST|VALUE|VALUES|VARIABLE|VARIANCE|VARRAY|VARYING|VIEW|VIEWS|VOID|WHEN|WHERE|WHILE|WITH|WORK|WRAPPED|WRITE|YEAR|ZONE)\b/i, // https://docs.oracle.com/en/database/oracle/oracle-database/21/lnpls/plsql-language-fundamentals.html#GUID-96A42F7C-7A71-4B90-8255-CA9C8BD9722E 'operator': /:=?|=>|[<>^~!]=|\.\.|\|\||\*\*|[-+*/%<>=@]/, - }); - - insertBefore(plsql, 'operator', { - 'label': { - pattern: /<<\s*\w+\s*>>/, - alias: 'symbol', + $insert: { + 'label': { + $before: 'operator', + pattern: /<<\s*\w+\s*>>/, + alias: 'symbol', + }, }, - }); - - return plsql; + }; }, } as LanguageProto<'plsql'>; diff --git a/src/languages/processing.ts b/src/languages/processing.ts index 53cee10c7f..c61d1439d8 100644 --- a/src/languages/processing.ts +++ b/src/languages/processing.ts @@ -1,28 +1,26 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; import type { LanguageProto } from '../types'; export default { id: 'processing', - require: clike, - grammar ({ extend }) { - const processing = extend('clike', { + base: clike, + grammar () { + return { 'keyword': /\b(?:break|case|catch|class|continue|default|else|extends|final|for|if|implements|import|new|null|private|public|return|static|super|switch|this|try|void|while)\b/, // Spaces are allowed between function name and parenthesis 'function': /\b\w+(?=\s*\()/, 'operator': /<[<=]?|>[>=]?|&&?|\|\|?|[%?]|[!=+\-*\/]=?/, - }); - - insertBefore(processing, 'number', { - // Special case: XML is a type - 'constant': /\b(?!XML\b)[A-Z][A-Z\d_]+\b/, - 'type': { - pattern: /\b(?:boolean|byte|char|color|double|float|int|[A-Z]\w*)\b/, - alias: 'class-name', + $insertBefore: { + 'number': { + // Special case: XML is a type + 'constant': /\b(?!XML\b)[A-Z][A-Z\d_]+\b/, + 'type': { + pattern: /\b(?:boolean|byte|char|color|double|float|int|[A-Z]\w*)\b/, + alias: 'class-name', + }, + }, }, - }); - - return processing; + }; }, } as LanguageProto<'processing'>; diff --git a/src/languages/protobuf.ts b/src/languages/protobuf.ts index fec136bf98..ff21634572 100644 --- a/src/languages/protobuf.ts +++ b/src/languages/protobuf.ts @@ -1,15 +1,14 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; import type { LanguageProto } from '../types'; export default { id: 'protobuf', - require: clike, - grammar ({ extend }) { + base: clike, + grammar () { const builtinTypes = /\b(?:bool|bytes|double|s?fixed(?:32|64)|float|[su]?int(?:32|64)|string)\b/; - const protobuf = extend('clike', { + return { 'class-name': [ { pattern: /(\b(?:enum|extend|message|service)\s+)[A-Za-z_]\w*(?=\s*\{)/, @@ -24,31 +23,30 @@ export default { 'keyword': /\b(?:enum|extend|extensions|import|message|oneof|option|optional|package|public|repeated|required|reserved|returns|rpc(?=\s+\w)|service|stream|syntax|to)\b(?!\s*=\s*\d)/, 'function': /\b[a-z_]\w*(?=\s*\()/i, - }); - - insertBefore(protobuf, 'operator', { - 'map': { - pattern: /\bmap<\s*[\w.]+\s*,\s*[\w.]+\s*>(?=\s+[a-z_]\w*\s*[=;])/i, - alias: 'class-name', - inside: { - 'punctuation': /[<>.,]/, + $insertBefore: { + 'operator': { + 'map': { + pattern: /\bmap<\s*[\w.]+\s*,\s*[\w.]+\s*>(?=\s+[a-z_]\w*\s*[=;])/i, + alias: 'class-name', + inside: { + 'punctuation': /[<>.,]/, + 'builtin': builtinTypes, + }, + }, 'builtin': builtinTypes, + 'positional-class-name': { + pattern: /(?:\b|\B\.)[a-z_]\w*(?:\.[a-z_]\w*)*(?=\s+[a-z_]\w*\s*[=;])/i, + alias: 'class-name', + inside: { + 'punctuation': /\./, + }, + }, + 'annotation': { + pattern: /(\[\s*)[a-z_]\w*(?=\s*=)/i, + lookbehind: true, + }, }, }, - 'builtin': builtinTypes, - 'positional-class-name': { - pattern: /(?:\b|\B\.)[a-z_]\w*(?:\.[a-z_]\w*)*(?=\s+[a-z_]\w*\s*[=;])/i, - alias: 'class-name', - inside: { - 'punctuation': /\./, - }, - }, - 'annotation': { - pattern: /(\[\s*)[a-z_]\w*(?=\s*=)/i, - lookbehind: true, - }, - }); - - return protobuf; + }; }, } as LanguageProto<'protobuf'>; diff --git a/src/languages/pug.ts b/src/languages/pug.ts index 5b0cea56df..45b03277e9 100644 --- a/src/languages/pug.ts +++ b/src/languages/pug.ts @@ -1,4 +1,4 @@ -import { insertBefore } from '../util/language-util'; +import { insertBefore } from '../util/insert'; import javascript from './javascript'; import markup from './markup'; import type { Grammar, GrammarTokens, LanguageProto } from '../types'; diff --git a/src/languages/pure.ts b/src/languages/pure.ts index 9c73cc3504..ae15c121d7 100644 --- a/src/languages/pure.ts +++ b/src/languages/pure.ts @@ -1,4 +1,4 @@ -import { insertBefore } from '../util/language-util'; +import { insertBefore } from '../util/insert'; import type { Grammar, GrammarToken, LanguageProto } from '../types'; export default { @@ -40,7 +40,7 @@ export default { greedy: true, }, 'number': { - // The look-behind prevents wrong highlighting of the .. operator + // The lookbehind prevents wrong highlighting of the .. operator pattern: /((?:\.\.)?)(?:\b(?:inf|nan)\b|\b0x[\da-f]+|(?:\b(?:0b)?\d+(?:\.\d+)?|\B\.\d+)(?:e[+-]?\d+)?L?)/i, lookbehind: true, diff --git a/src/languages/purebasic.ts b/src/languages/purebasic.ts index 7a49682b99..290f377cf5 100644 --- a/src/languages/purebasic.ts +++ b/src/languages/purebasic.ts @@ -1,12 +1,11 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'purebasic', - require: clike, + base: clike, alias: 'pbfasm', - grammar ({ extend }) { + grammar () { /* Original Code by Bas Groothedde !!MANY THANKS!! I never would have made this, regex and me will never be best friends ;) @@ -15,66 +14,66 @@ export default { */ // PureBasic support, steal stuff from ansi-c - const purebasic = extend('clike', { + return { 'comment': /;.*/, 'keyword': /\b(?:align|and|as|break|calldebugger|case|compilercase|compilerdefault|compilerelse|compilerelseif|compilerendif|compilerendselect|compilererror|compilerif|compilerselect|continue|data|datasection|debug|debuglevel|declare|declarec|declarecdll|declaredll|declaremodule|default|define|dim|disableasm|disabledebugger|disableexplicit|else|elseif|enableasm|enabledebugger|enableexplicit|end|enddatasection|enddeclaremodule|endenumeration|endif|endimport|endinterface|endmacro|endmodule|endprocedure|endselect|endstructure|endstructureunion|endwith|enumeration|extends|fakereturn|for|foreach|forever|global|gosub|goto|if|import|importc|includebinary|includefile|includepath|interface|macro|module|newlist|newmap|next|not|or|procedure|procedurec|procedurecdll|proceduredll|procedurereturn|protected|prototype|prototypec|read|redim|repeat|restore|return|runtime|select|shared|static|step|structure|structureunion|swap|threaded|to|until|wend|while|with|xincludefile|xor)\b/i, 'function': /\b\w+(?:\.\w+)?\s*(?=\()/, 'number': /(?:\$[\da-f]+|\b-?(?:\d+(?:\.\d+)?|\.\d+)(?:e[+-]?\d+)?)\b/i, 'operator': /(?:@\*?|\?|\*)\w+\$?|-[>-]?|\+\+?|!=?|<>?=?|==?|&&?|\|?\||[~^%?*/@]/, - }); - - insertBefore(purebasic, 'keyword', { - 'tag': /#\w+\$?/, - 'asm': { - pattern: /(^[\t ]*)!.*/m, - lookbehind: true, - alias: 'tag', - inside: { - 'comment': /;.*/, - 'string': { - pattern: /(["'`])(?:\\.|(?!\1)[^\\\r\n])*\1/, - greedy: true, - }, - // Anonymous label references, i.e.: jmp @b - 'label-reference-anonymous': { - pattern: /(!\s*j[a-z]+\s+)@[fb]/i, - lookbehind: true, - alias: 'fasm-label', - }, - // Named label reference, i.e.: jne label1 - 'label-reference-addressed': { - pattern: /(!\s*j[a-z]+\s+)[A-Z._?$@][\w.?$@~#]*/i, - lookbehind: true, - alias: 'fasm-label', - }, - 'keyword': [/\b(?:extern|global)\b[^;\r\n]*/i, /\b(?:CPU|DEFAULT|FLOAT)\b.*/], - 'function': { - pattern: /^([\t ]*!\s*)[\da-z]+(?=\s|$)/im, + $insertBefore: { + 'keyword': { + 'tag': /#\w+\$?/, + 'asm': { + pattern: /(^[\t ]*)!.*/m, lookbehind: true, + alias: 'tag', + inside: { + 'comment': /;.*/, + 'string': { + pattern: /(["'`])(?:\\.|(?!\1)[^\\\r\n])*\1/, + greedy: true, + }, + // Anonymous label references, i.e.: jmp @b + 'label-reference-anonymous': { + pattern: /(!\s*j[a-z]+\s+)@[fb]/i, + lookbehind: true, + alias: 'fasm-label', + }, + // Named label reference, i.e.: jne label1 + 'label-reference-addressed': { + pattern: /(!\s*j[a-z]+\s+)[A-Z._?$@][\w.?$@~#]*/i, + lookbehind: true, + alias: 'fasm-label', + }, + 'keyword': [ + /\b(?:extern|global)\b[^;\r\n]*/i, + /\b(?:CPU|DEFAULT|FLOAT)\b.*/, + ], + 'function': { + pattern: /^([\t ]*!\s*)[\da-z]+(?=\s|$)/im, + lookbehind: true, + }, + 'function-inline': { + pattern: /(:\s*)[\da-z]+(?=\s)/i, + lookbehind: true, + alias: 'function', + }, + 'label': { + pattern: /^([\t ]*!\s*)[A-Za-z._?$@][\w.?$@~#]*(?=:)/m, + lookbehind: true, + alias: 'fasm-label', + }, + 'register': + /\b(?:st\d|[xyz]mm\d\d?|[cdt]r\d|r\d\d?[bwd]?|[abcd][hl]|[er]?[abcd]x|[er]?(?:bp|di|si|sp)|[cdefgs]s|mm\d+)\b/i, + 'number': + /(?:\b|-|(?=\$))(?:0[hx](?:[\da-f]*\.)?[\da-f]+(?:p[+-]?\d+)?|\d[\da-f]+[hx]|\$\d[\da-f]*|0[oq][0-7]+|[0-7]+[oq]|0[by][01]+|[01]+[by]|0[dt]\d+|(?:\d+(?:\.\d+)?|\.\d+)(?:\.?e[+-]?\d+)?[dt]?)\b/i, + 'operator': /[\[\]*+\-/%<>=&|$!,.:]/, + }, }, - 'function-inline': { - pattern: /(:\s*)[\da-z]+(?=\s)/i, - lookbehind: true, - alias: 'function', - }, - 'label': { - pattern: /^([\t ]*!\s*)[A-Za-z._?$@][\w.?$@~#]*(?=:)/m, - lookbehind: true, - alias: 'fasm-label', - }, - 'register': - /\b(?:st\d|[xyz]mm\d\d?|[cdt]r\d|r\d\d?[bwd]?|[abcd][hl]|[er]?[abcd]x|[er]?(?:bp|di|si|sp)|[cdefgs]s|mm\d+)\b/i, - 'number': - /(?:\b|-|(?=\$))(?:0[hx](?:[\da-f]*\.)?[\da-f]+(?:p[+-]?\d+)?|\d[\da-f]+[hx]|\$\d[\da-f]*|0[oq][0-7]+|[0-7]+[oq]|0[by][01]+|[01]+[by]|0[dt]\d+|(?:\d+(?:\.\d+)?|\.\d+)(?:\.?e[+-]?\d+)?[dt]?)\b/i, - 'operator': /[\[\]*+\-/%<>=&|$!,.:]/, }, }, - }); - - delete purebasic['class-name']; - delete purebasic['boolean']; - - return purebasic; + $delete: ['class-name', 'boolean'], + } as unknown as Grammar; }, } as LanguageProto<'purebasic'>; diff --git a/src/languages/purescript.ts b/src/languages/purescript.ts index 541f98b30f..ed13ada6dd 100644 --- a/src/languages/purescript.ts +++ b/src/languages/purescript.ts @@ -3,10 +3,10 @@ import type { LanguageProto } from '../types'; export default { id: 'purescript', - require: haskell, + base: haskell, alias: 'purs', - grammar ({ extend }) { - return extend('haskell', { + grammar () { + return { 'keyword': /\b(?:ado|case|class|data|derive|do|else|forall|if|in|infixl|infixr|instance|let|module|newtype|of|primitive|then|type|where)\b|∀/, @@ -35,6 +35,6 @@ export default { // See https://github.com/PrismJS/prism/issues/3006 for more details. /[\xa2-\xa6\xa8\xa9\xac\xae-\xb1\xb4\xb8\xd7\xf7\u02c2-\u02c5\u02d2-\u02df\u02e5-\u02eb\u02ed\u02ef-\u02ff\u0375\u0384\u0385\u03f6\u0482\u058d-\u058f\u0606-\u0608\u060b\u060e\u060f\u06de\u06e9\u06fd\u06fe\u07f6\u07fe\u07ff\u09f2\u09f3\u09fa\u09fb\u0af1\u0b70\u0bf3-\u0bfa\u0c7f\u0d4f\u0d79\u0e3f\u0f01-\u0f03\u0f13\u0f15-\u0f17\u0f1a-\u0f1f\u0f34\u0f36\u0f38\u0fbe-\u0fc5\u0fc7-\u0fcc\u0fce\u0fcf\u0fd5-\u0fd8\u109e\u109f\u1390-\u1399\u166d\u17db\u1940\u19de-\u19ff\u1b61-\u1b6a\u1b74-\u1b7c\u1fbd\u1fbf-\u1fc1\u1fcd-\u1fcf\u1fdd-\u1fdf\u1fed-\u1fef\u1ffd\u1ffe\u2044\u2052\u207a-\u207c\u208a-\u208c\u20a0-\u20bf\u2100\u2101\u2103-\u2106\u2108\u2109\u2114\u2116-\u2118\u211e-\u2123\u2125\u2127\u2129\u212e\u213a\u213b\u2140-\u2144\u214a-\u214d\u214f\u218a\u218b\u2190-\u2307\u230c-\u2328\u232b-\u2426\u2440-\u244a\u249c-\u24e9\u2500-\u2767\u2794-\u27c4\u27c7-\u27e5\u27f0-\u2982\u2999-\u29d7\u29dc-\u29fb\u29fe-\u2b73\u2b76-\u2b95\u2b97-\u2bff\u2ce5-\u2cea\u2e50\u2e51\u2e80-\u2e99\u2e9b-\u2ef3\u2f00-\u2fd5\u2ff0-\u2ffb\u3004\u3012\u3013\u3020\u3036\u3037\u303e\u303f\u309b\u309c\u3190\u3191\u3196-\u319f\u31c0-\u31e3\u3200-\u321e\u322a-\u3247\u3250\u3260-\u327f\u328a-\u32b0\u32c0-\u33ff\u4dc0-\u4dff\ua490-\ua4c6\ua700-\ua716\ua720\ua721\ua789\ua78a\ua828-\ua82b\ua836-\ua839\uaa77-\uaa79\uab5b\uab6a\uab6b\ufb29\ufbb2-\ufbc1\ufdfc\ufdfd\ufe62\ufe64-\ufe66\ufe69\uff04\uff0b\uff1c-\uff1e\uff3e\uff40\uff5c\uff5e\uffe0-\uffe6\uffe8-\uffee\ufffc\ufffd]/, ], - }); + }; }, } as LanguageProto<'purescript'>; diff --git a/src/languages/qore.ts b/src/languages/qore.ts index c039973ee3..67e03a4102 100644 --- a/src/languages/qore.ts +++ b/src/languages/qore.ts @@ -3,9 +3,9 @@ import type { LanguageProto } from '../types'; export default { id: 'qore', - require: clike, - grammar ({ extend }) { - return extend('clike', { + base: clike, + grammar () { + return { 'comment': { pattern: /(^|[^\\])(?:\/\*[\s\S]*?\*\/|(?:\/\/|#).*)/, lookbehind: true, @@ -27,6 +27,6 @@ export default { lookbehind: true, }, 'variable': /\$(?!\d)\w+\b/, - }); + }; }, } as LanguageProto<'qore'>; diff --git a/src/languages/qsharp.ts b/src/languages/qsharp.ts index dd9f5f1c04..7fe9f955ae 100644 --- a/src/languages/qsharp.ts +++ b/src/languages/qsharp.ts @@ -1,12 +1,11 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; import type { LanguageProto } from '../types'; export default { id: 'qsharp', - require: clike, + base: clike, alias: 'qs', - grammar ({ extend }) { + grammar () { /** * Replaces all placeholders "<>" of given pattern with the n-th replacement (zero based). * @@ -62,7 +61,13 @@ export default { // strings const regularString = /"(?:\\.|[^\\"])*"/.source; - const qsharp = extend('clike', { + // single line + const interpolationExpr = nested( + replace(/\{(?:[^"{}]|<<0>>|<>)*\}/.source, [regularString]), + 2 + ); + + return { 'comment': /\/\/.*/, 'string': [ { @@ -92,43 +97,37 @@ export default { 'operator': /\band=|\bor=|\band\b|\bnot\b|\bor\b|<[-=]|[-=]>|>>>=?|<<<=?|\^\^\^=?|\|\|\|=?|&&&=?|w\/=?|~~~|[*\/+\-^=!%]=?/, 'punctuation': /::|[{}[\];(),.:]/, - }); - - insertBefore(qsharp, 'number', { - 'range': { - pattern: /\.\./, - alias: 'operator', - }, - }); - - // single line - const interpolationExpr = nested( - replace(/\{(?:[^"{}]|<<0>>|<>)*\}/.source, [regularString]), - 2 - ); - - insertBefore(qsharp, 'string', { - 'interpolation-string': { - pattern: re(/\$"(?:\\.|<<0>>|[^\\"{])*"/.source, [interpolationExpr]), - greedy: true, - inside: { - 'interpolation': { - pattern: re(/((?:^|[^\\])(?:\\\\)*)<<0>>/.source, [interpolationExpr]), - lookbehind: true, + $insertBefore: { + 'number': { + 'range': { + pattern: /\.\./, + alias: 'operator', + }, + }, + 'string': { + 'interpolation-string': { + pattern: re(/\$"(?:\\.|<<0>>|[^\\"{])*"/.source, [interpolationExpr]), + greedy: true, inside: { - 'punctuation': /^\{|\}$/, - 'expression': { - pattern: /[\s\S]+/, - alias: 'language-qsharp', - inside: 'qsharp', + 'interpolation': { + pattern: re(/((?:^|[^\\])(?:\\\\)*)<<0>>/.source, [ + interpolationExpr, + ]), + lookbehind: true, + inside: { + 'punctuation': /^\{|\}$/, + 'expression': { + pattern: /[\s\S]+/, + alias: 'language-qsharp', + inside: 'qsharp', + }, + }, }, + 'string': /[\s\S]+/, }, }, - 'string': /[\s\S]+/, }, }, - }); - - return qsharp; + }; }, } as LanguageProto<'qsharp'>; diff --git a/src/languages/racket.ts b/src/languages/racket.ts index 0032562ff0..d5be3a3f34 100644 --- a/src/languages/racket.ts +++ b/src/languages/racket.ts @@ -1,29 +1,26 @@ -import { insertBefore } from '../util/language-util'; import scheme from './scheme'; import type { LanguageProto } from '../types'; export default { id: 'racket', - require: scheme, + base: scheme, alias: 'rkt', - grammar ({ extend }) { - const racket = extend('scheme', { + grammar () { + return { 'lambda-parameter': { // the racket lambda syntax is a lot more complex, so we won't even attempt to capture it. // this will just prevent false positives of the `function` pattern pattern: /([(\[]lambda\s+[(\[])[^()\[\]'\s]+/, lookbehind: true, }, - }); - - insertBefore(racket, 'string', { - 'lang': { - pattern: /^#lang.+/m, - greedy: true, - alias: 'keyword', + $insert: { + 'lang': { + $before: 'string', + pattern: /^#lang.+/m, + greedy: true, + alias: 'keyword', + }, }, - }); - - return racket; + }; }, } as LanguageProto<'racket'>; diff --git a/src/languages/reason.ts b/src/languages/reason.ts index d28188ab2a..732c500681 100644 --- a/src/languages/reason.ts +++ b/src/languages/reason.ts @@ -1,12 +1,11 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'reason', - require: clike, - grammar ({ extend }) { - const reason = extend('clike', { + base: clike, + grammar () { + return { 'string': { pattern: /"(?:\\(?:\r\n|[\s\S])|[^\\\r\n"])*"/, greedy: true, @@ -17,23 +16,22 @@ export default { /\b(?:and|as|assert|begin|class|constraint|do|done|downto|else|end|exception|external|for|fun|function|functor|if|in|include|inherit|initializer|lazy|let|method|module|mutable|new|nonrec|object|of|open|or|private|rec|sig|struct|switch|then|to|try|type|val|virtual|when|while|with)\b/, 'operator': /\.{3}|:[:=]|\|>|->|=(?:==?|>)?|<=?|>=?|[|^?'#!~`]|[+\-*\/]\.?|\b(?:asr|land|lor|lsl|lsr|lxor|mod)\b/, - }); - insertBefore(reason, 'class-name', { - 'char': { - pattern: /'(?:\\x[\da-f]{2}|\\o[0-3][0-7][0-7]|\\\d{3}|\\.|[^'\\\r\n])'/, - greedy: true, - }, - // Negative look-ahead prevents from matching things like String.capitalize - 'constructor': /\b[A-Z]\w*\b(?!\s*\.)/, - 'label': { - pattern: /\b[a-z]\w*(?=::)/, - alias: 'symbol', + $insertBefore: { + 'class-name': { + 'char': { + pattern: /'(?:\\x[\da-f]{2}|\\o[0-3][0-7][0-7]|\\\d{3}|\\.|[^'\\\r\n])'/, + greedy: true, + }, + // Negative look-ahead prevents from matching things like String.capitalize + 'constructor': /\b[A-Z]\w*\b(?!\s*\.)/, + 'label': { + pattern: /\b[a-z]\w*(?=::)/, + alias: 'symbol', + }, + }, }, - }); - - // We can't match functions property, so let's not even try. - delete reason.function; - - return reason; + // We can't match functions property, so let's not even try. + $delete: ['function'], + } as unknown as Grammar; }, } as LanguageProto<'reason'>; diff --git a/src/languages/rescript.ts b/src/languages/rescript.ts index c21c32d9b6..06d227e268 100644 --- a/src/languages/rescript.ts +++ b/src/languages/rescript.ts @@ -1,11 +1,10 @@ -import { insertBefore } from '../util/language-util'; import type { Grammar, LanguageProto } from '../types'; export default { id: 'rescript', alias: 'res', grammar () { - const rescript = { + return { 'comment': { pattern: /\/\/.*|\/\*[\s\S]*?(?:\*\/|$)/, greedy: true, @@ -40,33 +39,34 @@ export default { 'operator': /\.{3}|:[:=]?|\|>|->|=(?:==?|>)?|<=?|>=?|[|^?'#!~`]|[+\-*\/]\.?|\b(?:asr|land|lor|lsl|lsr|lxor|mod)\b/, 'punctuation': /[(){}[\],;.]/, - }; - - insertBefore(rescript, 'string', { - 'template-string': { - pattern: /`(?:\\[\s\S]|\$\{(?:[^{}]|\{(?:[^{}]|\{[^}]*\})*\})+\}|(?!\$\{)[^\\`])*`/, - greedy: true, - inside: { - 'template-punctuation': { - pattern: /^`|`$/, - alias: 'string', - }, - 'interpolation': { - pattern: /((?:^|[^\\])(?:\\{2})*)\$\{(?:[^{}]|\{(?:[^{}]|\{[^}]*\})*\})+\}/, - lookbehind: true, + $insertBefore: { + 'string': { + 'template-string': { + pattern: + /`(?:\\[\s\S]|\$\{(?:[^{}]|\{(?:[^{}]|\{[^}]*\})*\})+\}|(?!\$\{)[^\\`])*`/, + greedy: true, inside: { - 'interpolation-punctuation': { - pattern: /^\$\{|\}$/, - alias: 'tag', + 'template-punctuation': { + pattern: /^`|`$/, + alias: 'string', + }, + 'interpolation': { + pattern: + /((?:^|[^\\])(?:\\{2})*)\$\{(?:[^{}]|\{(?:[^{}]|\{[^}]*\})*\})+\}/, + lookbehind: true, + inside: { + 'interpolation-punctuation': { + pattern: /^\$\{|\}$/, + alias: 'tag', + }, + $rest: 'rescript', + } as unknown as Grammar, }, - $rest: 'rescript', - } as unknown as Grammar, + 'string': /[\s\S]+/, + }, }, - 'string': /[\s\S]+/, }, }, - }); - - return rescript; + }; }, } as LanguageProto<'rescript'>; diff --git a/src/languages/ruby.ts b/src/languages/ruby.ts index f22afa063b..7c96097733 100644 --- a/src/languages/ruby.ts +++ b/src/languages/ruby.ts @@ -1,43 +1,17 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'ruby', - require: clike, + base: clike, alias: 'rb', - grammar ({ extend }) { + grammar () { /** * Original by Samuel Flores * * Adds the following new token classes: * constant, builtin, variable, symbol, regex */ - const ruby = extend('clike', { - 'comment': { - pattern: /#.*|^=begin\s[\s\S]*?^=end/m, - greedy: true, - }, - 'class-name': { - pattern: - /(\b(?:class|module)\s+|\bcatch\s+\()[\w.\\]+|\b[A-Z_]\w*(?=\s*\.\s*new\b)/, - lookbehind: true, - inside: { - 'punctuation': /[.\\]/, - }, - }, - 'keyword': - /\b(?:BEGIN|END|alias|and|begin|break|case|class|def|define_method|defined|do|each|else|elsif|end|ensure|extend|for|if|in|include|module|new|next|nil|not|or|prepend|private|protected|public|raise|redo|require|rescue|retry|return|self|super|then|throw|undef|unless|until|when|while|yield)\b/, - 'operator': /\.{2,3}|&\.|===||[!=]?~|(?:&&|\|\||<<|>>|\*\*|[+\-*/%<>!^&|=])=?|[?:]/, - 'punctuation': /[(){}[\].,;]/, - }); - - insertBefore(ruby, 'operator', { - 'double-colon': { - pattern: /::/, - alias: 'punctuation', - }, - }); const interpolation = { pattern: /((?:^|[^\\])(?:\\{2})*)#\{(?:[^{}]|\{[^{}]*\})*\}/, @@ -55,8 +29,6 @@ export default { }, }; - delete ruby.function; - const percentExpression = '(?:' + [ @@ -71,136 +43,162 @@ export default { const symbolName = /(?:"(?:\\.|[^"\\\r\n])*"|(?:\b[a-zA-Z_]\w*|[^\s\0-\x7F]+)[?!]?|\$.)/ .source; - insertBefore(ruby, 'keyword', { - 'regex-literal': [ - { - pattern: RegExp(/%r/.source + percentExpression + /[egimnosux]{0,6}/.source), - greedy: true, - inside: { - 'interpolation': interpolation, - 'regex': /[\s\S]+/, - }, - }, - { - pattern: - /(^|[^/])\/(?!\/)(?:\[[^\r\n\]]+\]|\\.|[^[/\\\r\n])+\/[egimnosux]{0,6}(?=\s*(?:$|[\r\n,.;})#]))/, - lookbehind: true, - greedy: true, - inside: { - 'interpolation': interpolation, - 'regex': /[\s\S]+/, - }, - }, - ], - 'variable': /[@$]+[a-zA-Z_]\w*(?:[?!]|\b)/, - 'symbol': [ - { - pattern: RegExp(/(^|[^:]):/.source + symbolName), - lookbehind: true, - greedy: true, - }, - { - pattern: RegExp(/([\r\n{(,][ \t]*)/.source + symbolName + /(?=:(?!:))/.source), - lookbehind: true, - greedy: true, - }, - ], - 'method-definition': { - pattern: /(\bdef\s+)\w+(?:\s*\.\s*\w+)?/, + return { + 'comment': { + pattern: /#.*|^=begin\s[\s\S]*?^=end/m, + greedy: true, + }, + 'class-name': { + pattern: + /(\b(?:class|module)\s+|\bcatch\s+\()[\w.\\]+|\b[A-Z_]\w*(?=\s*\.\s*new\b)/, lookbehind: true, inside: { - 'function': /\b\w+$/, - 'keyword': /^self\b/, - 'class-name': /^\w+/, - 'punctuation': /\./, + 'punctuation': /[.\\]/, }, }, - }); - - insertBefore(ruby, 'string', { - 'string-literal': [ - { - pattern: RegExp(/%[qQiIwWs]?/.source + percentExpression), - greedy: true, - inside: { - 'interpolation': interpolation, - 'string': /[\s\S]+/, - }, - }, - { - pattern: /("|')(?:#\{[^}]+\}|#(?!\{)|\\(?:\r\n|[\s\S])|(?!\1)[^\\#\r\n])*\1/, - greedy: true, - inside: { - 'interpolation': interpolation, - 'string': /[\s\S]+/, + 'keyword': + /\b(?:BEGIN|END|alias|and|begin|break|case|class|def|define_method|defined|do|each|else|elsif|end|ensure|extend|for|if|in|include|module|new|next|nil|not|or|prepend|private|protected|public|raise|redo|require|rescue|retry|return|self|super|then|throw|undef|unless|until|when|while|yield)\b/, + 'operator': /\.{2,3}|&\.|===||[!=]?~|(?:&&|\|\||<<|>>|\*\*|[+\-*/%<>!^&|=])=?|[?:]/, + 'punctuation': /[(){}[\].,;]/, + $insertBefore: { + 'operator': { + 'double-colon': { + pattern: /::/, + alias: 'punctuation', }, }, - { - pattern: /<<[-~]?([a-z_]\w*)[\r\n](?:.*[\r\n])*?[\t ]*\1/i, - alias: 'heredoc-string', - greedy: true, - inside: { - 'delimiter': { - pattern: /^<<[-~]?[a-z_]\w*|\b[a-z_]\w*$/i, + 'keyword': { + 'regex-literal': [ + { + pattern: RegExp( + /%r/.source + percentExpression + /[egimnosux]{0,6}/.source + ), + greedy: true, inside: { - 'symbol': /\b\w+/, - 'punctuation': /^<<[-~]?/, + 'interpolation': interpolation, + 'regex': /[\s\S]+/, }, }, - 'interpolation': interpolation, - 'string': /[\s\S]+/, - }, - }, - { - pattern: /<<[-~]?'([a-z_]\w*)'[\r\n](?:.*[\r\n])*?[\t ]*\1/i, - alias: 'heredoc-string', - greedy: true, - inside: { - 'delimiter': { - pattern: /^<<[-~]?'[a-z_]\w*'|\b[a-z_]\w*$/i, + { + pattern: + /(^|[^/])\/(?!\/)(?:\[[^\r\n\]]+\]|\\.|[^[/\\\r\n])+\/[egimnosux]{0,6}(?=\s*(?:$|[\r\n,.;})#]))/, + lookbehind: true, + greedy: true, inside: { - 'symbol': /\b\w+/, - 'punctuation': /^<<[-~]?'|'$/, + 'interpolation': interpolation, + 'regex': /[\s\S]+/, }, }, - 'string': /[\s\S]+/, - }, - }, - ], - 'command-literal': [ - { - pattern: RegExp(/%x/.source + percentExpression), - greedy: true, - inside: { - 'interpolation': interpolation, - 'command': { - pattern: /[\s\S]+/, - alias: 'string', + ], + 'variable': /[@$]+[a-zA-Z_]\w*(?:[?!]|\b)/, + 'symbol': [ + { + pattern: RegExp(/(^|[^:]):/.source + symbolName), + lookbehind: true, + greedy: true, + }, + { + pattern: RegExp( + /([\r\n{(,][ \t]*)/.source + symbolName + /(?=:(?!:))/.source + ), + lookbehind: true, + greedy: true, + }, + ], + 'method-definition': { + pattern: /(\bdef\s+)\w+(?:\s*\.\s*\w+)?/, + lookbehind: true, + inside: { + 'function': /\b\w+$/, + 'keyword': /^self\b/, + 'class-name': /^\w+/, + 'punctuation': /\./, }, }, }, - { - pattern: /`(?:#\{[^}]+\}|#(?!\{)|\\(?:\r\n|[\s\S])|[^\\`#\r\n])*`/, - greedy: true, - inside: { - 'interpolation': interpolation, - 'command': { - pattern: /[\s\S]+/, - alias: 'string', + 'string': { + 'string-literal': [ + { + pattern: RegExp(/%[qQiIwWs]?/.source + percentExpression), + greedy: true, + inside: { + 'interpolation': interpolation, + 'string': /[\s\S]+/, + }, }, - }, + { + pattern: + /("|')(?:#\{[^}]+\}|#(?!\{)|\\(?:\r\n|[\s\S])|(?!\1)[^\\#\r\n])*\1/, + greedy: true, + inside: { + 'interpolation': interpolation, + 'string': /[\s\S]+/, + }, + }, + { + pattern: /<<[-~]?([a-z_]\w*)[\r\n](?:.*[\r\n])*?[\t ]*\1/i, + alias: 'heredoc-string', + greedy: true, + inside: { + 'delimiter': { + pattern: /^<<[-~]?[a-z_]\w*|\b[a-z_]\w*$/i, + inside: { + 'symbol': /\b\w+/, + 'punctuation': /^<<[-~]?/, + }, + }, + 'interpolation': interpolation, + 'string': /[\s\S]+/, + }, + }, + { + pattern: /<<[-~]?'([a-z_]\w*)'[\r\n](?:.*[\r\n])*?[\t ]*\1/i, + alias: 'heredoc-string', + greedy: true, + inside: { + 'delimiter': { + pattern: /^<<[-~]?'[a-z_]\w*'|\b[a-z_]\w*$/i, + inside: { + 'symbol': /\b\w+/, + 'punctuation': /^<<[-~]?'|'$/, + }, + }, + 'string': /[\s\S]+/, + }, + }, + ], + 'command-literal': [ + { + pattern: RegExp(/%x/.source + percentExpression), + greedy: true, + inside: { + 'interpolation': interpolation, + 'command': { + pattern: /[\s\S]+/, + alias: 'string', + }, + }, + }, + { + pattern: /`(?:#\{[^}]+\}|#(?!\{)|\\(?:\r\n|[\s\S])|[^\\`#\r\n])*`/, + greedy: true, + inside: { + 'interpolation': interpolation, + 'command': { + pattern: /[\s\S]+/, + alias: 'string', + }, + }, + }, + ], }, - ], - }); - - delete ruby.string; - - insertBefore(ruby, 'number', { - 'builtin': - /\b(?:Array|Bignum|Binding|Class|Continuation|Dir|Exception|FalseClass|File|Fixnum|Float|Hash|IO|Integer|MatchData|Method|Module|NilClass|Numeric|Object|Proc|Range|Regexp|Stat|String|Struct|Symbol|TMS|Thread|ThreadGroup|Time|TrueClass)\b/, - 'constant': /\b[A-Z][A-Z0-9_]*(?:[?!]|\b)/, - }); - - return ruby; + 'number': { + 'builtin': + /\b(?:Array|Bignum|Binding|Class|Continuation|Dir|Exception|FalseClass|File|Fixnum|Float|Hash|IO|Integer|MatchData|Method|Module|NilClass|Numeric|Object|Proc|Range|Regexp|Stat|String|Struct|Symbol|TMS|Thread|ThreadGroup|Time|TrueClass)\b/, + 'constant': /\b[A-Z][A-Z0-9_]*(?:[?!]|\b)/, + }, + }, + $delete: ['function', 'string'], + } as unknown as Grammar; }, } as LanguageProto<'ruby'>; diff --git a/src/languages/sass.ts b/src/languages/sass.ts index 8129f9f748..85f5de3801 100644 --- a/src/languages/sass.ts +++ b/src/languages/sass.ts @@ -1,33 +1,10 @@ -import { insertBefore } from '../util/language-util'; import css from './css'; -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'sass', - require: css, - grammar ({ extend }) { - const sass = extend('css', { - // Sass comments don't need to be closed, only indented - 'comment': { - pattern: /^([ \t]*)\/[\/*].*(?:(?:\r?\n|\r)\1[ \t].+)*/m, - lookbehind: true, - greedy: true, - }, - }); - - insertBefore(sass, 'atrule', { - // We want to consume the whole line - 'atrule-line': { - // Includes support for = and + shortcuts - pattern: /^(?:[ \t]*)[@+=].+/m, - greedy: true, - inside: { - 'atrule': /(?:@[\w-]+|[+=])/, - }, - }, - }); - delete sass.atrule; - + base: css, + grammar ({ base }) { const variable = /\$[-\w]+|#\{\$[-\w]+\}/; const operator = [ /[+*\/%]|[=!]=|<=?|>=?|\b(?:and|not|or)\b/, @@ -37,50 +14,67 @@ export default { }, ]; - insertBefore(sass, 'property', { - // We want to consume the whole line - 'variable-line': { - pattern: /^[ \t]*\$.+/m, + return { + // Sass comments don't need to be closed, only indented + 'comment': { + pattern: /^([ \t]*)\/[\/*].*(?:(?:\r?\n|\r)\1[ \t].+)*/m, + lookbehind: true, greedy: true, - inside: { - 'punctuation': /:/, - 'variable': variable, - 'operator': operator, - }, }, - // We want to consume the whole line - 'property-line': { - pattern: /^[ \t]*(?:[^:\s]+ *:.*|:[^:\s].*)/m, - greedy: true, - inside: { - 'property': [ - /[^:\s]+(?=\s*:)/, - { - pattern: /(:)[^:\s]+/, - lookbehind: true, + $insertBefore: { + 'atrule': { + // We want to consume the whole line + 'atrule-line': { + // Includes support for = and + shortcuts + pattern: /^(?:[ \t]*)[@+=].+/m, + greedy: true, + inside: { + 'atrule': /(?:@[\w-]+|[+=])/, }, - ], - 'punctuation': /:/, - 'variable': variable, - 'operator': operator, - 'important': sass.important, + }, + }, + 'property': { + // We want to consume the whole line + 'variable-line': { + pattern: /^[ \t]*\$.+/m, + greedy: true, + inside: { + 'punctuation': /:/, + 'variable': variable, + 'operator': operator, + }, + }, + // We want to consume the whole line + 'property-line': { + pattern: /^[ \t]*(?:[^:\s]+ *:.*|:[^:\s].*)/m, + greedy: true, + inside: { + 'property': [ + /[^:\s]+(?=\s*:)/, + { + pattern: /(:)[^:\s]+/, + lookbehind: true, + }, + ], + 'punctuation': /:/, + 'variable': variable, + 'operator': operator, + 'important': base!.important, + }, + }, + }, + // Now that whole lines for other patterns are consumed, + // what's left should be selectors + 'punctuation': { + 'selector': { + pattern: + /^([ \t]*)\S(?:,[^,\r\n]+|[^,\r\n]*)(?:,[^,\r\n]+)*(?:,(?:\r?\n|\r)\1[ \t]+\S(?:,[^,\r\n]+|[^,\r\n]*)(?:,[^,\r\n]+)*)*/m, + lookbehind: true, + greedy: true, + }, }, }, - }); - delete sass.property; - delete sass.important; - - // Now that whole lines for other patterns are consumed, - // what's left should be selectors - insertBefore(sass, 'punctuation', { - 'selector': { - pattern: - /^([ \t]*)\S(?:,[^,\r\n]+|[^,\r\n]*)(?:,[^,\r\n]+)*(?:,(?:\r?\n|\r)\1[ \t]+\S(?:,[^,\r\n]+|[^,\r\n]*)(?:,[^,\r\n]+)*)*/m, - lookbehind: true, - greedy: true, - }, - }); - - return sass; + $delete: ['atrule', 'property', 'important'], + } as unknown as Grammar; }, } as LanguageProto<'sass'>; diff --git a/src/languages/scala.ts b/src/languages/scala.ts index 11d61a262c..50cf5f1f6e 100644 --- a/src/languages/scala.ts +++ b/src/languages/scala.ts @@ -1,12 +1,11 @@ -import { insertBefore } from '../util/language-util'; import java from './java'; -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'scala', - require: java, - grammar ({ extend }) { - const scala = extend('java', { + base: java, + grammar () { + return { 'triple-quoted-string': { pattern: /"""[\s\S]*?"""/, greedy: true, @@ -22,45 +21,39 @@ export default { 'builtin': /\b(?:Any|AnyRef|AnyVal|Boolean|Byte|Char|Double|Float|Int|Long|Nothing|Short|String|Unit)\b/, 'symbol': /'[^\d\s\\]\w*/, - }); - - insertBefore(scala, 'triple-quoted-string', { - 'string-interpolation': { - pattern: - /\b[a-z]\w*(?:"""(?:[^$]|\$(?:[^{]|\{(?:[^{}]|\{[^{}]*\})*\}))*?"""|"(?:[^$"\r\n]|\$(?:[^{]|\{(?:[^{}]|\{[^{}]*\})*\}))*")/i, - greedy: true, - inside: { - 'id': { - pattern: /^\w+/, - greedy: true, - alias: 'function', - }, - 'escape': { - pattern: /\\\$"|\$[$"]/, - greedy: true, - alias: 'symbol', - }, - 'interpolation': { - pattern: /\$(?:\w+|\{(?:[^{}]|\{[^{}]*\})*\})/, - greedy: true, - inside: { - 'punctuation': /^\$\{?|\}$/, - 'expression': { - pattern: /[\s\S]+/, - inside: 'scala', + $insert: { + 'string-interpolation': { + $before: 'triple-quoted-string', + pattern: + /\b[a-z]\w*(?:"""(?:[^$]|\$(?:[^{]|\{(?:[^{}]|\{[^{}]*\})*\}))*?"""|"(?:[^$"\r\n]|\$(?:[^{]|\{(?:[^{}]|\{[^{}]*\})*\}))*")/i, + greedy: true, + inside: { + 'id': { + pattern: /^\w+/, + greedy: true, + alias: 'function', + }, + 'escape': { + pattern: /\\\$"|\$[$"]/, + greedy: true, + alias: 'symbol', + }, + 'interpolation': { + pattern: /\$(?:\w+|\{(?:[^{}]|\{[^{}]*\})*\})/, + greedy: true, + inside: { + 'punctuation': /^\$\{?|\}$/, + 'expression': { + pattern: /[\s\S]+/, + inside: 'scala', + }, }, }, + 'string': /[\s\S]+/, }, - 'string': /[\s\S]+/, }, }, - }); - - delete scala['doc-comment']; - delete scala['class-name']; - delete scala['function']; - delete scala['constant']; - - return scala; + $delete: ['doc-comment', 'class-name', 'function', 'constant'], + } as unknown as Grammar; }, } as LanguageProto<'scala'>; diff --git a/src/languages/scss.ts b/src/languages/scss.ts index c3fc199e44..0618978b94 100644 --- a/src/languages/scss.ts +++ b/src/languages/scss.ts @@ -1,12 +1,11 @@ -import { insertBefore } from '../util/language-util'; import css from './css'; import type { Grammar, LanguageProto } from '../types'; export default { id: 'scss', - require: css, - grammar ({ extend }) { - const scss = extend('css', { + base: css, + grammar () { + return { 'comment': { pattern: /(^|[^\\])(?:\/\*[\s\S]*?\*\/|\/\/.*)/, lookbehind: true, @@ -46,47 +45,44 @@ export default { 'variable': /\$[-\w]+|#\{\$[-\w]+\}/, }, }, - }); - - insertBefore(scss, 'atrule', { - 'keyword': [ - /@(?:content|debug|each|else(?: if)?|extend|for|forward|function|if|import|include|mixin|return|use|warn|while)\b/i, - { - pattern: /( )(?:from|through)(?= )/, - lookbehind: true, + $insertBefore: { + 'atrule': { + 'keyword': [ + /@(?:content|debug|each|else(?: if)?|extend|for|forward|function|if|import|include|mixin|return|use|warn|while)\b/i, + { + pattern: /( )(?:from|through)(?= )/, + lookbehind: true, + }, + ], + }, + 'important': { + // var and interpolated vars + 'variable': /\$[-\w]+|#\{\$[-\w]+\}/, + }, + 'function': { + 'module-modifier': { + pattern: /\b(?:as|hide|show|with)\b/i, + alias: 'keyword', + }, + 'placeholder': { + pattern: /%[-\w]+/, + alias: 'selector', + }, + 'statement': { + pattern: /\B!(?:default|optional)\b/i, + alias: 'keyword', + }, + 'boolean': /\b(?:false|true)\b/, + 'null': { + pattern: /\bnull\b/, + alias: 'keyword', + }, + 'operator': { + pattern: /(\s)(?:[-+*\/%]|[=!]=|<=?|>=?|and|not|or)(?=\s)/, + lookbehind: true, + }, }, - ], - }); - - insertBefore(scss, 'important', { - // var and interpolated vars - 'variable': /\$[-\w]+|#\{\$[-\w]+\}/, - }); - - insertBefore(scss, 'function', { - 'module-modifier': { - pattern: /\b(?:as|hide|show|with)\b/i, - alias: 'keyword', - }, - 'placeholder': { - pattern: /%[-\w]+/, - alias: 'selector', - }, - 'statement': { - pattern: /\B!(?:default|optional)\b/i, - alias: 'keyword', - }, - 'boolean': /\b(?:false|true)\b/, - 'null': { - pattern: /\bnull\b/, - alias: 'keyword', - }, - 'operator': { - pattern: /(\s)(?:[-+*\/%]|[=!]=|<=?|>=?|and|not|or)(?=\s)/, - lookbehind: true, }, - }); - - return scss; + } as unknown as Grammar; }, } as LanguageProto<'scss'>; diff --git a/src/languages/solidity.ts b/src/languages/solidity.ts index fc760d6213..e6e9d6de4c 100644 --- a/src/languages/solidity.ts +++ b/src/languages/solidity.ts @@ -1,13 +1,12 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; import type { LanguageProto } from '../types'; export default { id: 'solidity', - require: clike, + base: clike, alias: 'sol', - grammar ({ extend }) { - const solidity = extend('clike', { + grammar () { + return { 'class-name': { pattern: /(\b(?:contract|enum|interface|library|new|struct|using)\s+)(?!\d)[\w$]+/, lookbehind: true, @@ -15,21 +14,19 @@ export default { 'keyword': /\b(?:_|anonymous|as|assembly|assert|break|calldata|case|constant|constructor|continue|contract|default|delete|do|else|emit|enum|event|external|for|from|function|if|import|indexed|inherited|interface|internal|is|let|library|mapping|memory|modifier|new|payable|pragma|private|public|pure|require|returns?|revert|selfdestruct|solidity|storage|struct|suicide|switch|this|throw|using|var|view|while)\b/, 'operator': /=>|->|:=|=:|\*\*|\+\+|--|\|\||&&|<<=?|>>=?|[-+*/%^&|<>!=]=?|[~?]/, - }); - - insertBefore(solidity, 'keyword', { - 'builtin': - /\b(?:address|bool|byte|u?int(?:8|16|24|32|40|48|56|64|72|80|88|96|104|112|120|128|136|144|152|160|168|176|184|192|200|208|216|224|232|240|248|256)?|string|bytes(?:[1-9]|[12]\d|3[0-2])?)\b/, - }); - - insertBefore(solidity, 'number', { - 'version': { - pattern: /([<>]=?|\^)\d+\.\d+\.\d+\b/, - lookbehind: true, - alias: 'number', + $insertBefore: { + 'keyword': { + 'builtin': + /\b(?:address|bool|byte|u?int(?:8|16|24|32|40|48|56|64|72|80|88|96|104|112|120|128|136|144|152|160|168|176|184|192|200|208|216|224|232|240|248|256)?|string|bytes(?:[1-9]|[12]\d|3[0-2])?)\b/, + }, + 'number': { + 'version': { + pattern: /([<>]=?|\^)\d+\.\d+\.\d+\b/, + lookbehind: true, + alias: 'number', + }, + }, }, - }); - - return solidity; + }; }, } as LanguageProto<'solidity'>; diff --git a/src/languages/sparql.ts b/src/languages/sparql.ts index db62ee2361..f8a4bdcc29 100644 --- a/src/languages/sparql.ts +++ b/src/languages/sparql.ts @@ -1,28 +1,26 @@ -import { insertBefore } from '../util/language-util'; import turtle from './turtle'; import type { LanguageProto } from '../types'; export default { id: 'sparql', - require: turtle, + base: turtle, alias: 'rq', - grammar ({ extend }) { - const sparql = extend('turtle', { + grammar () { + return { 'boolean': /\b(?:false|true)\b/i, 'variable': { pattern: /[?$]\w+/, greedy: true, }, - }); - - insertBefore(sparql, 'punctuation', { - 'keyword': [ - /\b(?:A|ADD|ALL|AS|ASC|ASK|BNODE|BY|CLEAR|CONSTRUCT|COPY|CREATE|DATA|DEFAULT|DELETE|DESC|DESCRIBE|DISTINCT|DROP|EXISTS|FILTER|FROM|GROUP|HAVING|INSERT|INTO|LIMIT|LOAD|MINUS|MOVE|NAMED|NOT|NOW|OFFSET|OPTIONAL|ORDER|RAND|REDUCED|SELECT|SEPARATOR|SERVICE|SILENT|STRUUID|UNION|USING|UUID|VALUES|WHERE)\b/i, - /\b(?:ABS|AVG|BIND|BOUND|CEIL|COALESCE|CONCAT|CONTAINS|COUNT|DATATYPE|DAY|ENCODE_FOR_URI|FLOOR|GROUP_CONCAT|HOURS|IF|IRI|isBLANK|isIRI|isLITERAL|isNUMERIC|isURI|LANG|LANGMATCHES|LCASE|MAX|MD5|MIN|MINUTES|MONTH|REGEX|REPLACE|ROUND|sameTerm|SAMPLE|SECONDS|SHA1|SHA256|SHA384|SHA512|STR|STRAFTER|STRBEFORE|STRDT|STRENDS|STRLANG|STRLEN|STRSTARTS|SUBSTR|SUM|TIMEZONE|TZ|UCASE|URI|YEAR)\b(?=\s*\()/i, - /\b(?:BASE|GRAPH|PREFIX)\b/i, - ], - }); - - return sparql; + $insertBefore: { + 'punctuation': { + 'keyword': [ + /\b(?:A|ADD|ALL|AS|ASC|ASK|BNODE|BY|CLEAR|CONSTRUCT|COPY|CREATE|DATA|DEFAULT|DELETE|DESC|DESCRIBE|DISTINCT|DROP|EXISTS|FILTER|FROM|GROUP|HAVING|INSERT|INTO|LIMIT|LOAD|MINUS|MOVE|NAMED|NOT|NOW|OFFSET|OPTIONAL|ORDER|RAND|REDUCED|SELECT|SEPARATOR|SERVICE|SILENT|STRUUID|UNION|USING|UUID|VALUES|WHERE)\b/i, + /\b(?:ABS|AVG|BIND|BOUND|CEIL|COALESCE|CONCAT|CONTAINS|COUNT|DATATYPE|DAY|ENCODE_FOR_URI|FLOOR|GROUP_CONCAT|HOURS|IF|IRI|isBLANK|isIRI|isLITERAL|isNUMERIC|isURI|LANG|LANGMATCHES|LCASE|MAX|MD5|MIN|MINUTES|MONTH|REGEX|REPLACE|ROUND|sameTerm|SAMPLE|SECONDS|SHA1|SHA256|SHA384|SHA512|STR|STRAFTER|STRBEFORE|STRDT|STRENDS|STRLANG|STRLEN|STRSTARTS|SUBSTR|SUM|TIMEZONE|TZ|UCASE|URI|YEAR)\b(?=\s*\()/i, + /\b(?:BASE|GRAPH|PREFIX)\b/i, + ], + }, + }, + }; }, } as LanguageProto<'sparql'>; diff --git a/src/languages/sqf.ts b/src/languages/sqf.ts index ea4a9423a8..90ea748995 100644 --- a/src/languages/sqf.ts +++ b/src/languages/sqf.ts @@ -1,12 +1,11 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; -import type { LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'sqf', - require: clike, - grammar ({ extend }) { - const sqf = extend('clike', { + base: clike, + grammar ({ base }) { + return { 'string': { pattern: /"(?:(?:"")?[^"])*"(?!")|'(?:[^'])*'/, greedy: true, @@ -24,26 +23,23 @@ export default { alias: 'keyword', }, 'constant': /\bDIK(?:_[a-z\d]+)+\b/i, - }); - - insertBefore(sqf, 'string', { - 'macro': { - pattern: /(^[ \t]*)#[a-z](?:[^\r\n\\]|\\(?:\r\n|[\s\S]))*/im, - lookbehind: true, - greedy: true, - alias: 'property', - inside: { - 'directive': { - pattern: /#[a-z]+\b/i, - alias: 'keyword', + $insert: { + 'macro': { + $before: 'string', + pattern: /(^[ \t]*)#[a-z](?:[^\r\n\\]|\\(?:\r\n|[\s\S]))*/im, + lookbehind: true, + greedy: true, + alias: 'property', + inside: { + 'directive': { + pattern: /#[a-z]+\b/i, + alias: 'keyword', + }, + 'comment': base!.comment, }, - 'comment': sqf.comment, }, }, - }); - - delete sqf['class-name']; - - return sqf; + $delete: ['class-name'], + } as unknown as Grammar; }, } as LanguageProto<'sqf'>; diff --git a/src/languages/squirrel.ts b/src/languages/squirrel.ts index f32f32aae0..9c8317a76c 100644 --- a/src/languages/squirrel.ts +++ b/src/languages/squirrel.ts @@ -1,15 +1,12 @@ import { toArray } from '../util/iterables'; -import { insertBefore } from '../util/language-util'; import clike from './clike'; import type { LanguageProto } from '../types'; export default { id: 'squirrel', - require: clike, - grammar ({ extend, getLanguage }) { - const clike = getLanguage('clike'); - - const squirrel = extend('clike', { + base: clike, + grammar () { + return { 'comment': [ ...toArray(clike['comment']), { @@ -36,27 +33,25 @@ export default { 'number': /\b(?:0x[0-9a-fA-F]+|\d+(?:\.(?:\d+|[eE][+-]?\d+))?)\b/, 'operator': /\+\+|--|<=>|<[-<]|>>>?|&&?|\|\|?|[-+*/%!=<>]=?|[~^]|::?/, 'punctuation': /[(){}\[\],;.]/, - }); - - insertBefore(squirrel, 'string', { - 'char': { - pattern: /(^|[^\\"'])'(?:[^\\']|\\(?:[xuU][0-9a-fA-F]{0,8}|[\s\S]))'/, - lookbehind: true, - greedy: true, - }, - }); - - insertBefore(squirrel, 'operator', { - 'attribute-punctuation': { - pattern: /<\/|\/>/, - alias: 'important', - }, - 'lambda': { - pattern: /@(?=\()/, - alias: 'operator', + $insertBefore: { + 'string': { + 'char': { + pattern: /(^|[^\\"'])'(?:[^\\']|\\(?:[xuU][0-9a-fA-F]{0,8}|[\s\S]))'/, + lookbehind: true, + greedy: true, + }, + }, + 'operator': { + 'attribute-punctuation': { + pattern: /<\/|\/>/, + alias: 'important', + }, + 'lambda': { + pattern: /@(?=\()/, + alias: 'operator', + }, + }, }, - }); - - return squirrel; + }; }, } as LanguageProto<'squirrel'>; diff --git a/src/languages/textile.ts b/src/languages/textile.ts index 71a185220b..cc5484b383 100644 --- a/src/languages/textile.ts +++ b/src/languages/textile.ts @@ -3,8 +3,8 @@ import type { GrammarToken, LanguageProto } from '../types'; export default { id: 'textile', - require: markup, - grammar ({ extend }) { + base: markup, + grammar () { // We don't allow for pipes inside parentheses // to not break table pattern |(. foo |). bar | const modifierRegex = /\([^|()\n]+\)|\[[^\]\n]+\]|\{[^}\n]+\}/.source; @@ -290,15 +290,14 @@ export default { // Allow some styles inside table cells Object.assign(phraseInside['table'].inside, nestedPatterns); - const textile = extend('markup', { + return { 'phrase': phrase, - }); - - // Only allow alpha-numeric HTML tags, not XML tags - const tag = textile.tag as GrammarToken; - tag.pattern = - /<\/?(?!\d)[a-z0-9]+(?:\s+[^\s>\/=]+(?:=(?:("|')(?:\\[\s\S]|(?!\1)[^\\])*\1|[^\s'">=]+))?)*\s*\/?>/i; - - return textile; + $merge: { + // Only allow alpha-numeric HTML tags, not XML tags + 'tag': { + pattern: /<\/?(?!\d)[a-z0-9]+(?:\s+[^\s>\/=]+(?:=(?:("|')(?:\\[\s\S]|(?!\1)[^\\])*\1|[^\s'">=]+))?)*\s*\/?>/i, + } + } + }; }, } as LanguageProto<'textile'>; diff --git a/src/languages/treeview.ts b/src/languages/treeview.ts index 8e270923c6..0dae024bd4 100644 --- a/src/languages/treeview.ts +++ b/src/languages/treeview.ts @@ -1,7 +1,6 @@ import { getTextContent } from '../core/classes/token'; -import { withoutTokenize } from '../util/language-util'; -import type { Prism } from '../core/prism'; -import type { Grammar, LanguageProto } from '../types'; +import { withoutTokenize } from '../util/without-tokenize'; +import type { Grammar, LanguageProto, Prism } from '../types'; export default { id: 'treeview', @@ -95,5 +94,5 @@ export default { }, }, }, - } as unknown as Grammar, -} as LanguageProto<'treeview'>; + }, +} satisfies LanguageProto<'treeview'>; diff --git a/src/languages/tt2.ts b/src/languages/tt2.ts index fee5da4d03..eb855837b7 100644 --- a/src/languages/tt2.ts +++ b/src/languages/tt2.ts @@ -1,61 +1,56 @@ import { embeddedIn } from '../shared/languages/templating'; -import { insertBefore } from '../util/language-util'; import clike from './clike'; import markup from './markup'; import type { Grammar, LanguageProto } from '../types'; export default { id: 'tt2', - require: [clike, markup], - grammar ({ extend }) { - const tt2 = extend('clike', { + require: markup, + base: clike, + grammar (): Grammar { + return { 'comment': /#.*|\[%#[\s\S]*?%\]/, 'keyword': /\b(?:BLOCK|CALL|CASE|CATCH|CLEAR|DEBUG|DEFAULT|ELSE|ELSIF|END|FILTER|FINAL|FOREACH|GET|IF|IN|INCLUDE|INSERT|LAST|MACRO|META|NEXT|PERL|PROCESS|RAWPERL|RETURN|SET|STOP|SWITCH|TAGS|THROW|TRY|UNLESS|USE|WHILE|WRAPPER)\b/, 'punctuation': /[[\]{},()]/, - }); - - insertBefore(tt2, 'number', { - 'operator': /=[>=]?|!=?|<=?|>=?|&&|\|\|?|\b(?:and|not|or)\b/, - 'variable': { - pattern: /\b[a-z]\w*(?:\s*\.\s*(?:\d+|\$?[a-z]\w*))*\b/i, - }, - }); - - insertBefore(tt2, 'keyword', { - 'delimiter': { - pattern: /^(?:\[%|%%)-?|-?%\]$/, - alias: 'punctuation', - }, - }); - - insertBefore(tt2, 'string', { - 'single-quoted-string': { - pattern: /'[^\\']*(?:\\[\s\S][^\\']*)*'/, - greedy: true, - alias: 'string', + 'tt2': { + pattern: /\[%[\s\S]+?%\]/, + inside: 'tt2', }, - 'double-quoted-string': { - pattern: /"[^\\"]*(?:\\[\s\S][^\\"]*)*"/, - greedy: true, - alias: 'string', - inside: { + $insertBefore: { + 'number': { + 'operator': /=[>=]?|!=?|<=?|>=?|&&|\|\|?|\b(?:and|not|or)\b/, 'variable': { - pattern: /\$(?:[a-z]\w*(?:\.(?:\d+|\$?[a-z]\w*))*)/i, + pattern: /\b[a-z]\w*(?:\s*\.\s*(?:\d+|\$?[a-z]\w*))*\b/i, + }, + }, + 'keyword': { + 'delimiter': { + pattern: /^(?:\[%|%%)-?|-?%\]$/, + alias: 'punctuation', + }, + }, + 'string': { + 'single-quoted-string': { + pattern: /'[^\\']*(?:\\[\s\S][^\\']*)*'/, + greedy: true, + alias: 'string', + }, + 'double-quoted-string': { + pattern: /"[^\\"]*(?:\\[\s\S][^\\"]*)*"/, + greedy: true, + alias: 'string', + inside: { + 'variable': { + pattern: /\$(?:[a-z]\w*(?:\.(?:\d+|\$?[a-z]\w*))*)/i, + }, + }, }, }, }, - }); - - // The different types of TT2 strings "replace" the C-like standard string - delete tt2.string; - - return { - 'tt2': { - pattern: /\[%[\s\S]+?%\]/, - inside: tt2, - }, - $tokenize: embeddedIn('markup') as Grammar['$tokenize'], - }; + // The different types of TT2 strings "replace" the C-like standard string + $delete: ['string'], + $tokenize: embeddedIn('markup'), + } as unknown as Grammar; }, } as LanguageProto<'tt2'>; diff --git a/src/languages/typescript.ts b/src/languages/typescript.ts index ee8470b6d8..f79b6ba187 100644 --- a/src/languages/typescript.ts +++ b/src/languages/typescript.ts @@ -1,5 +1,5 @@ +import { insertBefore } from '../util/insert'; import { toArray } from '../util/iterables'; -import { insertBefore } from '../util/language-util'; import javascript from './javascript'; import type { Grammar, LanguageProto } from '../types'; diff --git a/src/languages/v.ts b/src/languages/v.ts index 0197e517b5..f0061ccd57 100644 --- a/src/languages/v.ts +++ b/src/languages/v.ts @@ -1,12 +1,16 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; import type { LanguageProto } from '../types'; export default { id: 'v', - require: clike, - grammar ({ extend }) { - const v = extend('clike', { + base: clike, + grammar () { + const genericInside = { + 'punctuation': /[<>]/, + 'class-name': /\w+/, + }; + + return { 'string': { pattern: /r?(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/, alias: 'quoted-string', @@ -45,51 +49,43 @@ export default { /~|\?|[*\/%^!=]=?|\+[=+]?|-[=-]?|\|[=|]?|&(?:=|&|\^=?)?|>(?:>=?|=)?|<(?:<=?|=|-)?|:=|\.\.\.?/, 'builtin': /\b(?:any(?:_float|_int)?|bool|byte(?:ptr)?|charptr|f(?:32|64)|i(?:8|16|64|128|nt)|rune|size_t|string|u(?:16|32|64|128)|voidptr)\b/, - }); - - insertBefore(v, 'string', { - 'char': { - pattern: /`(?:\\`|\\?[^`]{1,2})`/, // using {1,2} instead of `u` flag for compatibility - alias: 'rune', - }, - }); - - const genericInside = { - 'punctuation': /[<>]/, - 'class-name': /\w+/, - }; - - insertBefore(v, 'operator', { - 'attribute': { - pattern: - /(^[\t ]*)\[(?:deprecated|direct_array_access|flag|inline|live|ref_only|typedef|unsafe_fn|windows_stdcall)\]/m, - lookbehind: true, - alias: 'annotation', - inside: { - 'punctuation': /[\[\]]/, - 'keyword': /\w+/, + $insertBefore: { + 'string': { + 'char': { + pattern: /`(?:\\`|\\?[^`]{1,2})`/, // using {1,2} instead of `u` flag for compatibility + alias: 'rune', + }, }, - }, - 'generic': { - pattern: /<\w+>(?=\s*[\)\{])/, - inside: genericInside, - }, - }); - - insertBefore(v, 'function', { - 'generic-function': { - // e.g. foo( ... - pattern: /\b\w+\s*<\w+>(?=\()/, - inside: { - 'function': /^\w+/, + 'operator': { + 'attribute': { + pattern: + /(^[\t ]*)\[(?:deprecated|direct_array_access|flag|inline|live|ref_only|typedef|unsafe_fn|windows_stdcall)\]/m, + lookbehind: true, + alias: 'annotation', + inside: { + 'punctuation': /[\[\]]/, + 'keyword': /\w+/, + }, + }, 'generic': { - pattern: /<\w+>/, + pattern: /<\w+>(?=\s*[\)\{])/, inside: genericInside, }, }, + 'function': { + 'generic-function': { + // e.g. foo( ... + pattern: /\b\w+\s*<\w+>(?=\()/, + inside: { + 'function': /^\w+/, + 'generic': { + pattern: /<\w+>/, + inside: genericInside, + }, + }, + }, + }, }, - }); - - return v; + }; }, } as LanguageProto<'v'>; diff --git a/src/languages/vala.ts b/src/languages/vala.ts index f93ba5f197..c297734dae 100644 --- a/src/languages/vala.ts +++ b/src/languages/vala.ts @@ -1,12 +1,11 @@ -import { insertBefore } from '../util/language-util'; import clike from './clike'; import type { Grammar, LanguageProto } from '../types'; export default { id: 'vala', - require: clike, - grammar ({ extend }) { - const vala = extend('clike', { + base: clike, + grammar () { + return { // Classes copied from csharp 'class-name': [ { @@ -50,51 +49,49 @@ export default { 'operator': /\+\+|--|&&|\|\||<<=?|>>=?|=>|->|~|[+\-*\/%&^|=!<>]=?|\?\??|\.\.\./, 'punctuation': /[{}[\];(),.:]/, 'constant': /\b[A-Z0-9_]+\b/, - }); - - insertBefore(vala, 'string', { - 'raw-string': { - pattern: /"""[\s\S]*?"""/, - greedy: true, - alias: 'string', - }, - 'template-string': { - pattern: /@"[\s\S]*?"/, - greedy: true, - inside: { - 'interpolation': { - pattern: /\$(?:\([^)]*\)|[a-zA-Z]\w*)/, + $insertBefore: { + 'string': { + 'raw-string': { + pattern: /"""[\s\S]*?"""/, + greedy: true, + alias: 'string', + }, + 'template-string': { + pattern: /@"[\s\S]*?"/, + greedy: true, inside: { - 'delimiter': { - pattern: /^\$\(?|\)$/, - alias: 'punctuation', + 'interpolation': { + pattern: /\$(?:\([^)]*\)|[a-zA-Z]\w*)/, + inside: { + 'delimiter': { + pattern: /^\$\(?|\)$/, + alias: 'punctuation', + }, + $rest: 'vala', + }, }, - $rest: vala, - } as unknown as Grammar, + 'string': /[\s\S]+/, + }, }, - 'string': /[\s\S]+/, }, - }, - }); - - insertBefore(vala, 'keyword', { - 'regex': { - pattern: - /\/(?:\[(?:[^\]\\\r\n]|\\.)*\]|\\.|[^/\\\[\r\n])+\/[imsx]{0,4}(?=\s*(?:$|[\r\n,.;})\]]))/, - greedy: true, - inside: { - 'regex-source': { - pattern: /^(\/)[\s\S]+(?=\/[a-z]*$)/, - lookbehind: true, - alias: 'language-regex', - inside: 'regex', + 'keyword': { + 'regex': { + pattern: + /\/(?:\[(?:[^\]\\\r\n]|\\.)*\]|\\.|[^/\\\[\r\n])+\/[imsx]{0,4}(?=\s*(?:$|[\r\n,.;})\]]))/, + greedy: true, + inside: { + 'regex-source': { + pattern: /^(\/)[\s\S]+(?=\/[a-z]*$)/, + lookbehind: true, + alias: 'language-regex', + inside: 'regex', + }, + 'regex-delimiter': /^\//, + 'regex-flags': /^[a-z]+$/, + }, }, - 'regex-delimiter': /^\//, - 'regex-flags': /^[a-z]+$/, }, }, - }); - - return vala; + } as unknown as Grammar; }, } as LanguageProto<'vala'>; diff --git a/src/languages/vbnet.ts b/src/languages/vbnet.ts index 647ec6ed90..0fd8045824 100644 --- a/src/languages/vbnet.ts +++ b/src/languages/vbnet.ts @@ -1,13 +1,20 @@ -import { insertBefore } from '../util/language-util'; import basic from './basic'; import type { LanguageProto } from '../types'; export default { id: 'vbnet', require: basic, - optional: 'xml-doc', - grammar ({ extend, getOptionalLanguage }) { - const vbnet = extend('basic', { + optional: 'markup', + grammar ({ languages }) { + return { + 'doc-comment': { + pattern: /'''.*/, + greedy: true, + alias: 'comment', + inside: { + 'tag': languages.markup?.tag, + }, + }, 'comment': [ { pattern: /(?:!|REM\b).+/i, @@ -29,12 +36,6 @@ export default { 'keyword': /(?:\b(?:ADDHANDLER|ADDRESSOF|ALIAS|AND|ANDALSO|AS|BEEP|BLOAD|BOOLEAN|BSAVE|BYREF|BYTE|BYVAL|CALL(?: ABSOLUTE)?|CASE|CATCH|CBOOL|CBYTE|CCHAR|CDATE|CDBL|CDEC|CHAIN|CHAR|CHDIR|CINT|CLASS|CLEAR|CLNG|CLOSE|CLS|COBJ|COM|COMMON|CONST|CONTINUE|CSBYTE|CSHORT|CSNG|CSTR|CTYPE|CUINT|CULNG|CUSHORT|DATA|DATE|DECIMAL|DECLARE|DEF(?: FN| SEG|DBL|INT|LNG|SNG|STR)|DEFAULT|DELEGATE|DIM|DIRECTCAST|DO|DOUBLE|ELSE|ELSEIF|END|ENUM|ENVIRON|ERASE|ERROR|EVENT|EXIT|FALSE|FIELD|FILES|FINALLY|FOR(?: EACH)?|FRIEND|FUNCTION|GET|GETTYPE|GETXMLNAMESPACE|GLOBAL|GOSUB|GOTO|HANDLES|IF|IMPLEMENTS|IMPORTS|IN|INHERITS|INPUT|INTEGER|INTERFACE|IOCTL|IS|ISNOT|KEY|KILL|LET|LIB|LIKE|LINE INPUT|LOCATE|LOCK|LONG|LOOP|LSET|ME|MKDIR|MOD|MODULE|MUSTINHERIT|MUSTOVERRIDE|MYBASE|MYCLASS|NAME|NAMESPACE|NARROWING|NEW|NEXT|NOT|NOTHING|NOTINHERITABLE|NOTOVERRIDABLE|OBJECT|OF|OFF|ON(?: COM| ERROR| KEY| TIMER)?|OPEN|OPERATOR|OPTION(?: BASE)?|OPTIONAL|OR|ORELSE|OUT|OVERLOADS|OVERRIDABLE|OVERRIDES|PARAMARRAY|PARTIAL|POKE|PRIVATE|PROPERTY|PROTECTED|PUBLIC|PUT|RAISEEVENT|READ|READONLY|REDIM|REM|REMOVEHANDLER|RESTORE|RESUME|RETURN|RMDIR|RSET|RUN|SBYTE|SELECT(?: CASE)?|SET|SHADOWS|SHARED|SHELL|SHORT|SINGLE|SLEEP|STATIC|STEP|STOP|STRING|STRUCTURE|SUB|SWAP|SYNCLOCK|SYSTEM|THEN|THROW|TIMER|TO|TROFF|TRON|TRUE|TRY|TRYCAST|TYPE|TYPEOF|UINTEGER|ULONG|UNLOCK|UNTIL|USHORT|USING|VIEW PRINT|WAIT|WEND|WHEN|WHILE|WIDENING|WITH|WITHEVENTS|WRITE|WRITEONLY|XOR)|\B(?:#CONST|#ELSE|#ELSEIF|#END|#IF))(?:\$|\b)/i, 'punctuation': /[,;:(){}]/, - }); - - insertBefore(vbnet, 'comment', { - 'doc-comment': getOptionalLanguage('xml-doc')?.tick, - }); - - return vbnet; + }; }, } as LanguageProto<'vbnet'>; diff --git a/src/languages/velocity.ts b/src/languages/velocity.ts index 50d2dbd386..0ead632cad 100644 --- a/src/languages/velocity.ts +++ b/src/languages/velocity.ts @@ -1,13 +1,10 @@ -import { insertBefore } from '../util/language-util'; import markup from './markup'; -import type { Grammar, GrammarToken, LanguageProto } from '../types'; +import type { Grammar, LanguageProto } from '../types'; export default { id: 'velocity', - require: markup, - grammar ({ extend }) { - const velocity = extend('markup', {}); - + base: markup, + grammar () { const vel = { 'variable': { pattern: @@ -36,51 +33,59 @@ export default { 'punctuation': vel['punctuation'], }; - insertBefore(velocity, 'comment', { - 'unparsed': { - pattern: /(^|[^\\])#\[\[[\s\S]*?\]\]#/, - lookbehind: true, - greedy: true, - inside: { - 'punctuation': /^#\[\[|\]\]#$/, + return { + $merge: { + 'tag': { + inside: { + 'attr-value': { + inside: { + $rest: 'velocity', + } as unknown as Grammar, + }, + }, }, }, - 'velocity-comment': [ - { - pattern: /(^|[^\\])#\*[\s\S]*?\*#/, - lookbehind: true, - greedy: true, - alias: 'comment', - }, - { - pattern: /(^|[^\\])##.*/, - lookbehind: true, - greedy: true, - alias: 'comment', - }, - ], - 'directive': { - pattern: - /(^|[^\\](?:\\\\)*)#@?(?:[a-z][\w-]*|\{[a-z][\w-]*\})(?:\s*\((?:[^()]|\([^()]*\))*\))?/i, - lookbehind: true, - inside: { - 'keyword': { - pattern: /^#@?(?:[a-z][\w-]*|\{[a-z][\w-]*\})|\bin\b/, + $insertBefore: { + 'comment': { + 'unparsed': { + pattern: /(^|[^\\])#\[\[[\s\S]*?\]\]#/, + lookbehind: true, + greedy: true, inside: { - 'punctuation': /[{}]/, + 'punctuation': /^#\[\[|\]\]#$/, }, }, - $rest: vel, - } as unknown as Grammar, + 'velocity-comment': [ + { + pattern: /(^|[^\\])#\*[\s\S]*?\*#/, + lookbehind: true, + greedy: true, + alias: 'comment', + }, + { + pattern: /(^|[^\\])##.*/, + lookbehind: true, + greedy: true, + alias: 'comment', + }, + ], + 'directive': { + pattern: + /(^|[^\\](?:\\\\)*)#@?(?:[a-z][\w-]*|\{[a-z][\w-]*\})(?:\s*\((?:[^()]|\([^()]*\))*\))?/i, + lookbehind: true, + inside: { + 'keyword': { + pattern: /^#@?(?:[a-z][\w-]*|\{[a-z][\w-]*\})|\bin\b/, + inside: { + 'punctuation': /[{}]/, + }, + }, + $rest: vel, + }, + }, + 'variable': vel['variable'], + }, }, - 'variable': vel['variable'], - }); - - ( - (((velocity['tag'] as GrammarToken).inside as Grammar)['attr-value'] as GrammarToken) - .inside as Grammar - ).$rest = velocity; - - return velocity; + }; }, } as LanguageProto<'velocity'>; diff --git a/src/languages/wiki.ts b/src/languages/wiki.ts index f50f2afdb4..ebffd8db7d 100644 --- a/src/languages/wiki.ts +++ b/src/languages/wiki.ts @@ -1,15 +1,13 @@ -import { insertBefore } from '../util/language-util'; import markup from './markup'; import type { Grammar, GrammarToken, LanguageProto } from '../types'; export default { id: 'wiki', - require: markup, - grammar ({ extend, getLanguage }) { - const markup = getLanguage('markup'); - const tag = markup['tag'] as GrammarToken; + base: markup, + grammar ({ base }) { + const tag = base!['tag'] as GrammarToken; - const wiki = extend('markup', { + return { 'block-comment': { pattern: /(^|[^\\])\/\*[\s\S]*?\*\//, lookbehind: true, @@ -74,21 +72,20 @@ export default { } as Grammar, }, 'punctuation': /^(?:\{\||\|\}|\|-|[*#:;!|])|\|\||!!/m, - }); - - insertBefore(wiki, 'tag', { - // Prevent highlighting inside , and 
 tags
-			'nowiki': {
-				pattern: /<(nowiki|pre|source)\b[^>]*>[\s\S]*?<\/\1>/i,
-				inside: {
-					'tag': {
-						pattern: /<(?:nowiki|pre|source)\b[^>]*>|<\/(?:nowiki|pre|source)>/i,
-						inside: tag.inside,
+			$insert: {
+				// Prevent highlighting inside ,  and 
 tags
+				'nowiki': {
+					$before: 'tag',
+					pattern: /<(nowiki|pre|source)\b[^>]*>[\s\S]*?<\/\1>/i,
+					inside: {
+						'tag': {
+							pattern:
+								/<(?:nowiki|pre|source)\b[^>]*>|<\/(?:nowiki|pre|source)>/i,
+							inside: tag.inside,
+						},
 					},
 				},
 			},
-		});
-
-		return wiki;
+		};
 	},
 } as LanguageProto<'wiki'>;
diff --git a/src/languages/xeora.ts b/src/languages/xeora.ts
index 3ca7604675..0f1d575815 100644
--- a/src/languages/xeora.ts
+++ b/src/languages/xeora.ts
@@ -3,9 +3,9 @@ import type { LanguageProto } from '../types';
 
 export default {
 	id: 'xeora',
-	require: markup,
+	base: markup,
 	alias: 'xeoracube',
-	grammar ({ extend }) {
+	grammar () {
 		const functionVariable = {
 			pattern: /(?:[,|])@?(?:#+|[-+*~=^])?[\w.]+/,
 			inside: {
@@ -18,7 +18,7 @@ export default {
 			},
 		};
 
-		return extend('markup', {
+		return {
 			'constant': {
 				pattern: /\$(?:DomainContents|PageRenderDuration)\$/,
 				inside: {
@@ -115,6 +115,6 @@ export default {
 				},
 				alias: 'function',
 			},
-		});
+		};
 	},
 } as LanguageProto<'xeora'>;
diff --git a/src/languages/xml-doc.ts b/src/languages/xml-doc.ts
deleted file mode 100644
index d13cde3998..0000000000
--- a/src/languages/xml-doc.ts
+++ /dev/null
@@ -1,29 +0,0 @@
-import markup from './markup';
-import type { LanguageProto } from '../types';
-
-export default {
-	id: 'xml-doc',
-	require: markup,
-	grammar ({ getLanguage }) {
-		const tag = getLanguage('markup').tag;
-
-		return {
-			'slash': {
-				pattern: /\/\/\/.*/,
-				greedy: true,
-				alias: 'comment',
-				inside: {
-					'tag': tag,
-				},
-			},
-			'tick': {
-				pattern: /'''.*/,
-				greedy: true,
-				alias: 'comment',
-				inside: {
-					'tag': tag,
-				},
-			},
-		};
-	},
-} as LanguageProto<'xml-doc'>;
diff --git a/src/languages/xml.ts b/src/languages/xml.ts
index 890932b5e1..c7ae9a3629 100644
--- a/src/languages/xml.ts
+++ b/src/languages/xml.ts
@@ -1,9 +1,13 @@
-import { MARKUP_TAG } from '../shared/languages/patterns';
 import type { LanguageProto } from '../types';
 
+export const MARKUP_TAG =
+	/<\/?(?!\d)[^\s>\/=$<%]+(?:\s(?:\s*[^\s>\/=]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+(?=[\s>]))|(?=[\s/>])))+)?\s*\/?>/;
+
 export default {
 	id: 'xml',
 	alias: ['ssml', 'atom', 'rss'],
+	media: ['application/xml', 'text/xml', 'application/rss+xml', 'application/atom+xml'],
+	extensions: ['xml', 'xsl', 'xsd', 'wsdl', 'svg', 'rss', 'atom'],
 	grammar () {
 		const entity = [
 			{
@@ -86,12 +90,4 @@ export default {
 			'entity': entity,
 		};
 	},
-	effect (Prism) {
-		// Plugin to make entity title show the real entity, idea by Roman Komarov
-		return Prism.hooks.add('wrap', env => {
-			if (env.type === 'entity') {
-				env.attributes['title'] = env.content.replace(/&/, '&');
-			}
-		});
-	},
-} as LanguageProto<'xml'>;
+} satisfies LanguageProto<'xml'>;
diff --git a/src/languages/xquery.ts b/src/languages/xquery.ts
index dbeafd9063..3a1d117691 100644
--- a/src/languages/xquery.ts
+++ b/src/languages/xquery.ts
@@ -1,11 +1,11 @@
 import { getTextContent, Token } from '../core/classes/token';
-import { withoutTokenize } from '../util/language-util';
+import { withoutTokenize } from '../util/without-tokenize';
 import markup from './markup';
 import type { TokenStream } from '../core/classes/token';
-import type { Grammar, GrammarToken, LanguageProto } from '../types';
+import type { Grammar, LanguageProto } from '../types';
 
 function walkTokens (tokens: TokenStream) {
-	const openedTags = [];
+	const openedTags: { tagName: string; openedBraces: number }[] = [];
 	for (let i = 0; i < tokens.length; i++) {
 		const token = tokens[i];
 		const isToken = typeof token !== 'string';
@@ -107,9 +107,30 @@ function walkTokens (tokens: TokenStream) {
 
 export default {
 	id: 'xquery',
-	require: markup,
-	grammar ({ extend }) {
-		const xquery = extend('markup', {
+	base: markup,
+	grammar () {
+		return {
+			$merge: {
+				'tag': {
+					pattern:
+						/<\/?(?!\d)[^\s>\/=$<%]+(?:\s+[^\s>\/=]+(?:=(?:("|')(?:\\[\s\S]|\{(?!\{)(?:\{(?:\{[^{}]*\}|[^{}])*\}|[^{}])+\}|(?!\1)[^\\])*\1|[^\s'">=]+))?)*\s*\/?>/,
+					inside: {
+						'attr-value': {
+							pattern:
+								/=(?:("|')(?:\\[\s\S]|\{(?!\{)(?:\{(?:\{[^{}]*\}|[^{}])*\}|[^{}])+\}|(?!\1)[^\\])*\1|[^\s'">=]+)/,
+							inside: {
+								'punctuation': /^="|"$/,
+								'expression': {
+									// Allow for two levels of nesting
+									pattern: /\{(?!\{)(?:\{(?:\{[^{}]*\}|[^{}])*\}|[^{}])+\}/,
+									alias: 'language-xquery',
+									inside: 'xquery',
+								},
+							},
+						},
+					},
+				},
+			},
 			'xquery-comment': {
 				pattern: /\(:[\s\S]*?:\)/,
 				greedy: true,
@@ -166,29 +187,11 @@ export default {
 				},
 			],
 			'punctuation': /[[\](){},;:/]/,
-		});
-
-		const tag = xquery['tag'] as GrammarToken;
-		tag.pattern =
-			/<\/?(?!\d)[^\s>\/=$<%]+(?:\s+[^\s>\/=]+(?:=(?:("|')(?:\\[\s\S]|\{(?!\{)(?:\{(?:\{[^{}]*\}|[^{}])*\}|[^{}])+\}|(?!\1)[^\\])*\1|[^\s'">=]+))?)*\s*\/?>/;
-		const attrValue = (tag.inside as Grammar)['attr-value'] as GrammarToken;
-		attrValue.pattern =
-			/=(?:("|')(?:\\[\s\S]|\{(?!\{)(?:\{(?:\{[^{}]*\}|[^{}])*\}|[^{}])+\}|(?!\1)[^\\])*\1|[^\s'">=]+)/;
-		const attrValueInside = attrValue.inside as Grammar;
-		attrValueInside['punctuation'] = /^="|"$/;
-		attrValueInside['expression'] = {
-			// Allow for two levels of nesting
-			pattern: /\{(?!\{)(?:\{(?:\{[^{}]*\}|[^{}])*\}|[^{}])+\}/,
-			alias: 'language-xquery',
-			inside: 'xquery',
-		};
-
-		xquery.$tokenize = (code, grammar, Prism) => {
-			const tokens = Prism.tokenize(code, withoutTokenize(grammar));
-			walkTokens(tokens);
-			return tokens;
-		};
-
-		return xquery;
+			$tokenize: (code, grammar, Prism) => {
+				const tokens = Prism.tokenize(code, withoutTokenize(grammar));
+				walkTokens(tokens);
+				return tokens;
+			},
+		} as unknown as Grammar;
 	},
 } as LanguageProto<'xquery'>;
diff --git a/src/plugins/autolinker/prism-autolinker.ts b/src/plugins/autolinker/prism-autolinker.ts
index 3cdc52475b..33519adafa 100644
--- a/src/plugins/autolinker/prism-autolinker.ts
+++ b/src/plugins/autolinker/prism-autolinker.ts
@@ -24,7 +24,7 @@ export default {
 
 		return Prism.hooks.add({
 			'after-tokenize': (env) => {
-				tokenizeStrings(env.tokens, (code) => Prism.tokenize(code, links));
+				tokenizeStrings(env.tokens!, (code) => Prism.tokenize(code, links));
 			},
 			'wrap': (env) => {
 				if (env.type.endsWith('-link')) {
diff --git a/src/plugins/diff-highlight/prism-diff-highlight.ts b/src/plugins/diff-highlight/prism-diff-highlight.ts
index 11e761eb53..398558fdec 100644
--- a/src/plugins/diff-highlight/prism-diff-highlight.ts
+++ b/src/plugins/diff-highlight/prism-diff-highlight.ts
@@ -1,8 +1,8 @@
 import { Token, getTextContent } from '../../core/classes/token';
 import diff, { PREFIXES } from '../../languages/diff';
+import type { HookEnv } from '../../core/classes/hooks';
 import type { TokenStream } from '../../core/classes/token';
 import type { PluginProto } from '../../types';
-import type { HookEnv } from '../../core/classes/hooks';
 
 export default {
 	id: 'diff-highlight',
@@ -11,9 +11,9 @@ export default {
 		const LANGUAGE_REGEX = /^diff-([\w-]+)/i;
 
 		const setMissingGrammar = (env: HookEnv) => {
-			const lang = env.language;
-			if (LANGUAGE_REGEX.test(lang) && !env.grammar) {
-				env.grammar = Prism.components.getLanguage('diff');
+			if (env.languageId.startsWith('diff-') && !env.language) {
+				env.language = Prism.languageRegistry.getLanguage('diff');
+				env.languageReady = Prism.languageRegistry.load('diff');
 			}
 		};
 
diff --git a/src/plugins/entity-title/prism-entity-title.ts b/src/plugins/entity-title/prism-entity-title.ts
new file mode 100644
index 0000000000..f3558baa41
--- /dev/null
+++ b/src/plugins/entity-title/prism-entity-title.ts
@@ -0,0 +1,11 @@
+export default {
+	id: 'entity-title',
+	effect (Prism) {
+		// Plugin to make entity title show the real entity, idea by Roman Komarov
+		return Prism.hooks.add('wrap', env => {
+			if (env.type === 'entity') {
+				env.attributes['title'] = env.content.replace(/&/, '&');
+			}
+		});
+	}
+}
diff --git a/src/plugins/inline-color/prism-inline-color.ts b/src/plugins/inline-color/prism-inline-color.ts
index c51a7546ff..7a7f5b4668 100644
--- a/src/plugins/inline-color/prism-inline-color.ts
+++ b/src/plugins/inline-color/prism-inline-color.ts
@@ -1,8 +1,8 @@
 import cssExtras from '../../languages/css-extras';
-import { MARKUP_TAG } from '../../shared/languages/patterns';
 import type { PluginProto } from '../../types';
 
-const HTML_TAG = RegExp(MARKUP_TAG, 'g');
+export const HTML_TAG =
+	/<\/?(?!\d)[^\s>\/=$<%]+(?:\s(?:\s*[^\s>\/=]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+(?=[\s>]))|(?=[\s/>])))+)?\s*\/?>/g;
 
 // a regex to validate hexadecimal colors
 const HEX_COLOR = /^#?((?:[\da-f]){3,4}|(?:[\da-f]{2}){3,4})$/i;
diff --git a/src/plugins/normalize-whitespace/prism-normalize-whitespace.ts b/src/plugins/normalize-whitespace/prism-normalize-whitespace.ts
index 304b08528c..a06d006be2 100644
--- a/src/plugins/normalize-whitespace/prism-normalize-whitespace.ts
+++ b/src/plugins/normalize-whitespace/prism-normalize-whitespace.ts
@@ -208,7 +208,7 @@ export default {
 				}
 			}
 
-			if (!env.element.children.length || !Prism.components.has('keep-markup')) {
+			if (!env.element.children.length || !Prism.plugins['keep-markup']) {
 				env.code = before + env.code + after;
 				env.code = Normalizer.normalize(env.code, settings);
 			} else {
diff --git a/src/plugins/previewers/prism-previewers.ts b/src/plugins/previewers/prism-previewers.ts
index 9975b65f46..f944157fed 100644
--- a/src/plugins/previewers/prism-previewers.ts
+++ b/src/plugins/previewers/prism-previewers.ts
@@ -187,7 +187,6 @@ export class PreviewerCollection {
 }
 
 // TODO: Filthy hack to be able to load this script
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
 const Prism = { languages: {} as Record };
 
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
diff --git a/src/plugins/toolbar/prism-toolbar.ts b/src/plugins/toolbar/prism-toolbar.ts
index 64e04358e3..f725c08028 100644
--- a/src/plugins/toolbar/prism-toolbar.ts
+++ b/src/plugins/toolbar/prism-toolbar.ts
@@ -1,6 +1,6 @@
 import { getParentPre } from '../../shared/dom-util';
 import { noop } from '../../shared/util';
-import type { HookEnv, HookCallback } from '../../core/classes/hooks';
+import type { HookCallback, HookEnv } from '../../core/classes/hooks';
 import type { PluginProto } from '../../types';
 
 /**
diff --git a/src/shared/languages/patterns.ts b/src/shared/languages/patterns.ts
deleted file mode 100644
index d52ac37152..0000000000
--- a/src/shared/languages/patterns.ts
+++ /dev/null
@@ -1,10 +0,0 @@
-export const MARKUP_TAG =
-	/<\/?(?!\d)[^\s>\/=$<%]+(?:\s(?:\s*[^\s>\/=]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+(?=[\s>]))|(?=[\s/>])))+)?\s*\/?>/;
-
-export const JS_TEMPLATE_INTERPOLATION = /\$\{(?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})+\}/;
-export const JS_TEMPLATE = RegExp(
-	/`(?:\\[\s\S]||[^\\`$]|\$(?!\{))*`/.source.replace(
-		'',
-		() => JS_TEMPLATE_INTERPOLATION.source
-	)
-);
diff --git a/src/shared/languages/templating.ts b/src/shared/languages/templating.ts
index 8f6858cfbb..52e796961c 100644
--- a/src/shared/languages/templating.ts
+++ b/src/shared/languages/templating.ts
@@ -1,5 +1,5 @@
 import { getTextContent } from '../../core/classes/token';
-import { withoutTokenize } from '../../util/language-util';
+import { withoutTokenize } from '../../util/without-tokenize';
 import type { Prism } from '../../core';
 import type { Token, TokenStream } from '../../core/classes/token';
 import type { Registry } from '../../core/registry';
@@ -105,6 +105,7 @@ function insertIntoHostToken (hostTokens: TokenStream, tokenStack: TokenStack) {
 
 type GrammarRef = Grammar | string | undefined | null;
 
+// TODO use resolve() from tokenize/util
 function resolve (ref: GrammarRef, components: Registry): Grammar | undefined {
 	if (!ref) {
 		return undefined;
diff --git a/src/shared/util.ts b/src/shared/util.ts
index 75dd94f92c..0bdc5986e5 100644
--- a/src/shared/util.ts
+++ b/src/shared/util.ts
@@ -18,10 +18,7 @@ export function lazy (supplier: () => T): () => T {
 }
 
 export function htmlEncode (text: string): string {
-	return text
-		.replace(/&/g, '&')
-		.replace(/ (value: T): value is T & {} {
 	return value != null;
 }
 
+declare global {
+	interface RegExpConstructor {
+		/**
+		 * Escapes special characters in a string for use in a regular expression.
+		 *
+		 * @param str The string to escape.
+		 */
+		escape (str: string): string;
+	}
+}
+
 /**
  * Escapes all special regex characters in the given string.
  */
-export function regexEscape (string: string): string {
-	return string.replace(/([\\[\](){}+*?|^$.])/g, '\\$1');
-}
-
+export const regexEscape: (string: string) => string =
+	RegExp.escape?.bind(RegExp) ??
+	((str: string) => {
+		return str.replace(/([\\[\](){}+*?|^$.])/g, '\\$1');
+	});
 
 export function capitalize (string: T): Capitalize {
 	// This is the internal implementation of `Capitalize` by TS.
@@ -56,3 +65,7 @@ export function kebabToCamelCase (kebab: T): KebabToCamelCase<
 	const [first, ...others] = kebab.split(/-/);
 	return (first + others.map(capitalize).join('')) as KebabToCamelCase;
 }
+
+export function camelToKebabCase (str: string) {
+	return (str + '').replace(/[A-Z]/g, l => '-' + l.toLowerCase());
+}
diff --git a/src/types.d.ts b/src/types.d.ts
index b973c0bd98..e9cd275ff4 100644
--- a/src/types.d.ts
+++ b/src/types.d.ts
@@ -1,65 +1,41 @@
-import type { Prism } from './core/prism';
-import type { TokenStream } from './core/classes/token';
+import type {
+	Language,
+	LanguageGrammars,
+	LanguageLike,
+	LanguageProto,
+	Languages,
+} from './core/classes/language';
+import type { PluginProto } from './core/classes/plugin-registry';
+import type { Prism } from './core/classes/prism';
+import type { Token, TokenName, TokenStream } from './core/classes/token';
+
+export type { Token, TokenName, TokenStream };
+
+export type { Prism } from './core/classes/prism';
+
+export type { Language, Languages, LanguageGrammars, LanguageProto, LanguageLike };
+
+export type {
+	ComponentRegistryOptions,
+	ComponentProtoBase,
+} from './core/classes/component-registry';
+
+export type { PluginProto };
+
+export type { KebabToCamelCase } from './util/types';
+
+export type ComponentProto = LanguageProto | PluginProto;
+
+export interface PlainObject {
+	[key: string]: unknown;
+}
 
 export interface GrammarOptions {
 	readonly getLanguage: (id: string) => Grammar;
-	readonly getOptionalLanguage: (id: string) => Grammar | undefined;
-	readonly extend: (id: string, ref: GrammarTokens) => Grammar;
-}
-export interface ComponentProtoBase {
-	id: Id;
-	require?: ComponentProto | readonly ComponentProto[];
-	optional?: string | readonly string[];
-	alias?: string | readonly string[];
-	effect?: (Prism: Prism & { plugins: Record, {}> }) => () => void;
-}
-export interface LanguageProto extends ComponentProtoBase {
-	grammar: Grammar | ((options: GrammarOptions) => Grammar);
-	plugin?: undefined;
+	readonly base?: Grammar;
+	readonly languages: LanguageGrammars;
+	readonly whenDefined: (id: string) => Promise;
 }
-type PluginType = unknown;
-export interface PluginProto extends ComponentProtoBase {
-	grammar?: undefined;
-	plugin?: (
-		Prism: Prism & { plugins: Record, undefined> }
-	) => PluginType> & {};
-}
-export type ComponentProto = LanguageProto | PluginProto;
-
-export type StandardTokenName =
-	| 'atrule'
-	| 'attr-name'
-	| 'attr-value'
-	| 'bold'
-	| 'boolean'
-	| 'builtin'
-	| 'cdata'
-	| 'char'
-	| 'class-name'
-	| 'comment'
-	| 'constant'
-	| 'deleted'
-	| 'doctype'
-	| 'entity'
-	| 'function'
-	| 'important'
-	| 'inserted'
-	| 'italic'
-	| 'keyword'
-	| 'namespace'
-	| 'number'
-	| 'operator'
-	| 'prolog'
-	| 'property'
-	| 'punctuation'
-	| 'regex'
-	| 'selector'
-	| 'string'
-	| 'symbol'
-	| 'tag'
-	| 'url';
-
-export type TokenName = (string & {}) | StandardTokenName;
 
 export type RegExpLike = RegExp & { readonly pattern?: never };
 
@@ -70,23 +46,27 @@ export interface GrammarToken {
 	/**
 	 * The regular expression of the token.
 	 */
-	pattern: RegExpLike;
+	pattern: RegExp;
+
 	/**
 	 * If `true`, then the first capturing group of `pattern` will (effectively) behave as a lookbehind group meaning that the captured text will not be part of the matched text of the new token.
 	 *
 	 * @default false
 	 */
 	lookbehind?: boolean;
+
 	/**
 	 * Whether the token is greedy.
 	 *
 	 * @default false
 	 */
 	greedy?: boolean;
+
 	/**
 	 * An optional alias or list of aliases.
 	 */
 	alias?: TokenName | TokenName[];
+
 	/**
 	 * The nested grammar of this token.
 	 *
@@ -100,10 +80,6 @@ export interface GrammarToken {
 	inside?: string | Grammar | null;
 }
 
-export type GrammarTokens = Partial<
-	Record
->;
-
 export type GrammarSpecial = {
 	/**
 	 * An optional grammar object that will be appended to this grammar.
@@ -112,12 +88,82 @@ export type GrammarSpecial = {
 	$tokenize?: (code: string, grammar: Grammar, Prism: Prism) => TokenStream;
 };
 
-export type Grammar = GrammarTokens & GrammarSpecial;
+export type GrammarTokens = GrammarTokensRec & GrammarSpecial;
 
-export interface PlainObject {
-	[key: string]: unknown;
-}
+/**
+ * A grammar that is defined as its delta from another grammar
+ */
+export type GrammarPatch = {
+	$insert?: GrammarTokens;
+	$before?: TokenName | TokenName[];
+	$after?: TokenName | TokenName[];
+	$insertBefore?: GrammarTokens;
+	$insertAfter?: GrammarTokens;
+	$delete?: TokenName[];
+	$merge?: GrammarTokens;
+};
+
+export type Grammar = GrammarTokens & GrammarSpecial & GrammarPatch;
+
+/**
+ * Recursive type helpers for GrammarTokens
+ *
+ * Why do we need this?
+ * The structure of a Prism grammar is inherently recursive: a token can contain an “inside” grammar, which itself is a set of tokens, and so on.
+ * To accurately type this, we need a recursive type definition.
+ *
+ * Why not just use the old type?
+ * ```ts
+ * export type GrammarTokens = Partial>;
+ * ```
+ * This type definition is infinitely recursive, which caused TypeScript to give up and treat it as “any”.
+ * By introducing a recursion depth limit (using a decrement tuple), we keep the type safe and precise, and prevent TypeScript from running into infinite recursion.
+ *
+ * The helpers below allow us to control the recursion depth in one place, making the type both maintainable and type-safe.
+ */
+
+// Centralize the depth
+type GrammarTokensDepth = 5;
 
-export type KebabToCamelCase = S extends `${infer T}-${infer U}`
-	? `${T}${Capitalize>}`
-	: S;
+// Helper to build a tuple of length N
+type BuildTuple = T['length'] extends N
+	? T
+	: BuildTuple;
+
+/**
+ * Helper to generate a decrement tuple
+ *
+ * TypeScript can't subtract numbers at the type level, so we use a tuple (array) to "count down" the depth for recursion.
+ * This lets us limit how many times GrammarTokensRec can call itself, which prevents infinite recursion and keeps the type safe.
+ */
+type DecrementTuple = T extends [
+	unknown,
+	...infer Rest,
+]
+	? DecrementTuple
+	: R;
+
+// The actual decrement tuple, derived from the depth
+type Decrement = DecrementTuple>;
+
+/**
+ * When the recursion limit is reached, the type system will use DeepRecursion instead of recursing further.
+ * It prevents the type from becoming “any” or “unknown” without explanation.
+ */
+type DeepRecursion = { __deepRecursion: true };
+
+/**
+ * GrammarTokensRec is a recursive type that builds a partial record of token names as keys, with values that can be a RegExp, GrammarToken, or an array of those types.
+ * It uses the decrement tuple to count down the depth of recursion, preventing infinite loops.
+ */
+type GrammarTokensRec = Depth extends 0
+	? DeepRecursion
+	: Partial<
+			Record<
+				TokenName,
+				| RegExpLike
+				| GrammarToken
+				| GrammarTokensRec
+				| (RegExpLike | GrammarToken | GrammarTokensRec)[]
+			>
+		>;
diff --git a/src/util/async.ts b/src/util/async.ts
index 019ef8f089..d0493c8150 100644
--- a/src/util/async.ts
+++ b/src/util/async.ts
@@ -24,3 +24,67 @@ export function documentReady (document = globalThis.document) {
 
 	return Promise.resolve();
 }
+
+export function nextTick () {
+	return new Promise(resolve => {
+		if (typeof requestAnimationFrame === 'function') {
+			requestAnimationFrame(resolve);
+		}
+		else if (typeof setImmediate === 'function') {
+			setImmediate(resolve);
+		}
+		else {
+			setTimeout(resolve, 0);
+		}
+	});
+}
+
+// In addition to waiting for all promises to settle, handle post-hoc additions/removals.
+export async function allSettled (promises: Promise[]): Promise<(T | null)[]> {
+	return Promise.allSettled(promises).then(outcomes => {
+		if (promises.length > 0 && promises.length !== outcomes.length) {
+			// The list of promises changed. Return a new Promise.
+			// The original promise won't resolve until the new one does.
+			return allSettled(promises);
+		}
+
+		// The list of promises either empty or stayed the same.
+		// Return results immediately.
+		return outcomes.map(o => (o.status === 'fulfilled' ? o.value : null));
+	});
+}
+
+export class Deferred extends Promise {
+	executor?: ConstructorParameters>[0];
+	resolve: (value: T | Promise) => void = () => {};
+	reject: (reason?: Error) => void = () => {};
+
+	constructor (executor?: ConstructorParameters>[0]) {
+		super((resolve, reject) => {
+			this.resolve = resolve;
+			this.reject = reject;
+			executor?.(resolve, reject);
+		});
+		this.executor = executor;
+	}
+}
+
+type DeferredPromise = Promise & {
+	resolve: (value: T) => void;
+	reject: (reason?: any) => void;
+};
+
+export function defer (): DeferredPromise {
+	let res!: (value: T) => void;
+	let rej!: (reason?: any) => void;
+
+	let promise = new Promise((resolve, reject) => {
+		res = resolve;
+		rej = reject;
+	}) as DeferredPromise;
+
+	promise.resolve = res;
+	promise.reject = rej;
+
+	return promise;
+}
diff --git a/src/util/extend.ts b/src/util/extend.ts
index db31718a9b..b0490e8232 100644
--- a/src/util/extend.ts
+++ b/src/util/extend.ts
@@ -1,4 +1,5 @@
-import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types';
+import { betterAssign, deepClone } from './objects';
+import type { Grammar, GrammarSpecial } from '../types';
 
 /**
  * Creates a deep copy of the language with the given id and appends the given tokens.
@@ -15,9 +16,8 @@ import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types'
  * Therefore, it is encouraged to order overwriting tokens according to the positions of the overwritten tokens.
  * Furthermore, all non-overwriting tokens should be placed after the overwriting ones.
  *
- * @param grammar The grammar of the language to extend.
- * @param id The id of the language to extend.
- * @param reDef The new tokens to append.
+ * @param base The grammar of the language to extend.
+ * @param grammar The new tokens to append.
  * @returns The new language created.
  * @example
  * Prism.languages['css-with-colors'] = Prism.languages.extend('css', {
@@ -28,90 +28,63 @@ import type { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../types'
  *     'color': /\b(?:red|green|blue)\b/
  * });
  */
-export function extend (grammar: Grammar, id: string, reDef: Grammar): Grammar {
-	const lang = cloneGrammar(grammar, id);
+export function extend (base: Grammar, grammar: Grammar): Grammar {
+	const lang = deepClone(base);
 
-	for (const key in reDef) {
-		lang[key] = reDef[key];
+	for (const key in grammar) {
+		if (typeof key !== 'string' || key.startsWith('$')) {
+			// ignore special keys
+			continue;
+		}
+
+		lang[key] = grammar[key];
 	}
 
-	return lang;
-}
+	if (grammar.$insertBefore) {
+		lang.$insertBefore = betterAssign(lang.$insertBefore ?? {}, grammar.$insertBefore);
+	}
 
-function cloneGrammar (grammar: Grammar, id: string): Grammar {
-	const result: Grammar = {};
+	if (grammar.$insertAfter) {
+		lang.$insertAfter = betterAssign(lang.$insertAfter ?? {}, grammar.$insertAfter);
+	}
 
-	const visited = new Map();
+	if (grammar.$insert) {
+		// Syntactic sugar for $insertBefore/$insertAfter
+		for (let tokenName in grammar.$insert) {
+			let def = grammar.$insert[tokenName] as Grammar;
+			let { $before, $after, ...token } = def;
+			let relToken = $before || $after;
+			let all = $before ? '$insertBefore' : '$insertAfter';
+			lang[all] ??= {};
 
-	function cloneToken (value: GrammarToken | RegExpLike) {
-		if (!value.pattern) {
-			return value;
-		}
-		else {
-			const copy: GrammarToken = { pattern: value.pattern };
-			if (value.lookbehind) {
-				copy.lookbehind = value.lookbehind;
-			}
-			if (value.greedy) {
-				copy.greedy = value.greedy;
+			if (Array.isArray(relToken)) {
+				// Insert in multiple places
+				for (let t of relToken) {
+					lang[all][t][tokenName] = token;
+				}
 			}
-			if (value.alias) {
-				copy.alias = Array.isArray(value.alias) ? [...value.alias] : value.alias;
+			else if (relToken) {
+				(lang[all][relToken] ??= {})[tokenName] = token;
 			}
-			if (value.inside) {
-				copy.inside = cloneRef(value.inside);
+			else {
+				lang[tokenName] = token;
 			}
-			return copy;
 		}
 	}
-	function cloneTokens (value: GrammarTokens[string]) {
-		if (!value) {
-			return undefined;
-		}
-		else if (Array.isArray(value)) {
-			return value.map(cloneToken);
-		}
-		else {
-			return cloneToken(value);
-		}
-	}
-	function cloneRef (ref: NonNullable) {
-		if (ref === id) {
-			// self ref
-			return result;
-		}
-		else if (typeof ref === 'string') {
-			return ref;
+
+	if (grammar.$delete) {
+		if (lang.$delete) {
+			// base also had $delete
+			lang.$delete.push(...grammar.$delete);
 		}
 		else {
-			return clone(ref);
+			lang.$delete = [...grammar.$delete];
 		}
 	}
-	function clone (value: Grammar) {
-		let mapped = visited.get(value);
-		if (mapped === undefined) {
-			mapped = value === grammar ? result : {};
-			visited.set(value, mapped);
-
-			// tokens
-			for (const [key, tokens] of Object.entries(value)) {
-				mapped[key] = cloneTokens(tokens as GrammarToken[]);
-			}
 
-			// rest
-			const r = value.$rest;
-			if (r != null) {
-				mapped.$rest = cloneRef(r);
-			}
-
-			// tokenize
-			const t = value.$tokenize;
-			if (t) {
-				mapped.$tokenize = t;
-			}
-		}
-		return mapped;
+	if (grammar.$merge) {
+		lang.$merge = betterAssign(lang.$merge ?? {}, grammar.$merge);
 	}
 
-	return clone(grammar);
+	return lang;
 }
diff --git a/src/util/grammar-patch.ts b/src/util/grammar-patch.ts
new file mode 100644
index 0000000000..56ba8f1865
--- /dev/null
+++ b/src/util/grammar-patch.ts
@@ -0,0 +1,82 @@
+import { insertAfter, insertBefore } from './insert';
+import { deepMerge } from './objects';
+import type { Grammar, GrammarTokens } from '../types';
+
+/**
+ * Apply a patch to a grammar to modify it.
+ * The patch and the grammar may be the same object.
+ *
+ * @param grammar
+ * @param patch
+ * @returns
+ */
+export function grammarPatch (grammar: Grammar, patch: Grammar = grammar): Grammar {
+	if (patch.$insertBefore) {
+		for (const key in patch.$insertBefore) {
+			const tokens = patch.$insertBefore[key];
+
+			if (key?.includes('/')) {
+				// Deep key
+				let path = key.split('/');
+				const lastKey = path.pop();
+				path = path.flatMap(key => [key, 'inside']); // add `inside` after each key
+				let obj = path.reduce((acc, key) => acc?.[key] as Grammar, grammar);
+
+				if (obj) {
+					insertBefore(obj, lastKey as string, tokens as GrammarTokens);
+				}
+			}
+			else if (tokens) {
+				insertBefore(grammar, key, tokens as GrammarTokens);
+			}
+		}
+		delete grammar.$insertBefore;
+	}
+
+	if (patch.$insertAfter) {
+		for (const key in patch.$insertAfter) {
+			const tokens = patch.$insertAfter[key];
+
+			if (key?.includes('/')) {
+				// Deep key
+				let path = key.split('/');
+				const lastKey = path.pop();
+				path = path.flatMap(key => [key, 'inside']); // add `inside` after each key
+				let obj = path.reduce((acc, key) => acc?.[key] as Grammar, grammar);
+
+				if (obj) {
+					insertAfter(obj, lastKey as string, tokens as GrammarTokens);
+				}
+			}
+			else if (tokens) {
+				insertAfter(grammar, key, tokens as GrammarTokens);
+			}
+		}
+		delete grammar.$insertAfter;
+	}
+
+	if (patch.$delete) {
+		for (const key of patch.$delete) {
+			// TODO support deep keys
+			delete grammar[key as string];
+		}
+		delete grammar.$delete;
+	}
+
+	if (patch.$merge) {
+		for (const key in patch.$merge) {
+			const tokens = patch.$merge[key];
+
+			if (grammar[key]) {
+				deepMerge(grammar[key], tokens);
+			}
+			else {
+				grammar[key] = tokens;
+			}
+		}
+
+		delete grammar.$merge;
+	}
+
+	return grammar;
+}
diff --git a/src/util/insert.ts b/src/util/insert.ts
index 5f132877ef..d7d637c5ef 100644
--- a/src/util/insert.ts
+++ b/src/util/insert.ts
@@ -1,4 +1,5 @@
-import type { Grammar, GrammarToken, GrammarTokens } from '../types';
+import { betterAssign } from './objects';
+import type { Grammar, GrammarTokens } from '../types';
 
 /**
  * Inserts tokens _before_ another token in the given grammar.
@@ -28,47 +29,51 @@ import type { Grammar, GrammarToken, GrammarTokens } from '../types';
  * });
  * ```
  *
- * ## Special cases
- *
- * If the grammars of `grammar` and `insert` have tokens with the same name, the tokens in `grammar`'s grammar
- * will be ignored.
- *
- * This behavior can be used to insert tokens after `before`:
- *
- * ```js
- * insertBefore(markup, 'comment', {
- *     'comment': markup.comment,
- *     // tokens after 'comment'
- * });
- * ```
- *
  * @param grammar The grammar to be modified.
- * @param before The key to insert before.
- * @param insert An object containing the key-value pairs to be inserted.
+ * @param beforeKey The key to insert before.
+ * @param tokens An object containing the key-value pairs to be inserted.
  */
-export function insertBefore (grammar: Grammar, before: string, insert: GrammarTokens) {
-	if (!(before in grammar)) {
-		throw new Error(`"${before}" has to be a key of grammar.`);
+export function insertBefore (grammar: Grammar, beforeKey: string, tokens: GrammarTokens) {
+	insert(grammar, beforeKey, tokens, 'before');
+}
+
+export function insertAfter (grammar: Grammar, afterKey: string, tokens: GrammarTokens) {
+	insert(grammar, afterKey, tokens);
+}
+
+export function insert (
+	grammar: Grammar,
+	atKey: string,
+	insert: GrammarTokens,
+	position: 'before' | 'after' = 'after'
+) {
+	if (!(atKey in grammar)) {
+		// TODO support deep keys
+		throw new Error(`"${atKey}" has to be a key of grammar.`);
 	}
 
-	const grammarEntries = Object.entries(grammar);
+	const descriptors = Object.getOwnPropertyDescriptors(grammar);
 
 	// delete all keys in `grammar`
-	for (const [key] of grammarEntries) {
-		delete grammar[key];
+	for (const key in descriptors) {
+		if (Object.hasOwn(descriptors, key)) {
+			delete grammar[key];
+		}
 	}
 
 	// insert keys again
-	for (const [key, value] of grammarEntries) {
-		if (key === before) {
-			for (const insertKey of Object.keys(insert)) {
-				grammar[insertKey] = insert[insertKey];
-			}
+	for (const key in descriptors) {
+		if (position === 'before' && key === atKey) {
+			betterAssign(grammar, insert);
 		}
 
 		// Do not insert tokens which also occur in `insert`. See #1525
-		if (!insert.hasOwnProperty(key)) {
-			grammar[key] = value as unknown as GrammarToken;
+		if (!Object.hasOwn(insert, key)) {
+			Object.defineProperty(grammar, key, descriptors[key]);
+		}
+
+		if (position === 'after' && key === atKey) {
+			betterAssign(grammar, insert);
 		}
 	}
 }
diff --git a/src/util/iterables.ts b/src/util/iterables.ts
index a985d7f4fc..f51d0027af 100644
--- a/src/util/iterables.ts
+++ b/src/util/iterables.ts
@@ -35,3 +35,15 @@ export function forEach (
 		callbackFn(value as T, 0);
 	}
 }
+
+export function isIterable (value: any): value is Iterable {
+	return typeof value === 'object' && Symbol.iterator in Object(value);
+}
+
+export function toIterable (value: any): Iterable {
+	if (isIterable(value)) {
+		return value;
+	}
+
+	return toArray(value);
+}
diff --git a/src/util/language-util.ts b/src/util/language-util.ts
index bbfc1b3c80..60e27ac430 100644
--- a/src/util/language-util.ts
+++ b/src/util/language-util.ts
@@ -1,3 +1,4 @@
 export { extend } from './extend';
-export { insertBefore } from './insert';
+export { grammarPatch as applyPatch } from './grammar-patch';
+export { insertAfter, insertBefore } from './insert';
 export { withoutTokenize } from './without-tokenize';
diff --git a/src/util/objects.ts b/src/util/objects.ts
new file mode 100644
index 0000000000..f87e08d165
--- /dev/null
+++ b/src/util/objects.ts
@@ -0,0 +1,149 @@
+import { toArray } from './iterables';
+
+export async function defineLazyProperty (
+	obj: T,
+	key: K,
+	getter: () => T[K],
+	waitFor?: any
+) {
+	if (waitFor) {
+		await waitFor;
+	}
+
+	Object.defineProperty(obj, key, {
+		enumerable: true,
+		configurable: true,
+		get () {
+			const value = getter.call(this);
+			// Replace the getter with a writable property
+			defineSimpleProperty(this, key, value);
+			return value;
+		},
+		set (value) {
+			defineSimpleProperty(this, key, value);
+		},
+	});
+}
+
+export function defineSimpleProperty (
+	obj: T,
+	key: K,
+	value: T[K]
+): void {
+	Object.defineProperty(obj, key, {
+		value,
+		writable: true,
+		enumerable: true,
+		configurable: false,
+	});
+}
+
+type Property = string | number | symbol;
+type EachCallback = (value: any, key: Property, parent: any, path: Property[]) => void;
+
+export function isPlainObject (obj: unknown) {
+	return isObject(obj, 'Object');
+}
+
+export function isObject (obj: unknown, type: string) {
+	if (!obj || typeof obj !== 'object') {
+		return false;
+	}
+
+	let proto = Object.getPrototypeOf(obj);
+	return proto.constructor?.name === type;
+}
+
+export interface MergeOptions {
+	emptyValues?: any[];
+	containers?: string[];
+	isContainer?: (value: any, key?: Property, parent?: any) => boolean;
+	mergeArrays?: boolean;
+}
+
+export function deepMerge (target: any, source: any, options: MergeOptions = {}) {
+	let {
+		emptyValues = [undefined],
+		containers = ['Object', 'EventTarget'],
+		isContainer = value => containers.some(type => isObject(value, type)),
+		mergeArrays = false,
+	} = options;
+
+	if (mergeArrays && (Array.isArray(target) || Array.isArray(source))) {
+		target = toArray(target);
+		source = toArray(source);
+		return target.concat(source);
+	}
+
+	if (isContainer(target) && isContainer(source)) {
+		for (let key in source) {
+			target[key] = deepMerge(target[key], source[key], options);
+		}
+
+		return target;
+	}
+
+	if (emptyValues.includes(target)) {
+		return source;
+	}
+
+	return target ?? source;
+}
+
+export interface CloneOptions {
+	/*
+	 * Used internally to store clones of objects,
+	 * both for performance but mainly to avoid getting tripped up in circular references
+	 */
+	_clones?: WeakMap;
+}
+
+export function deepClone (obj: any, options: CloneOptions = {}) {
+	if (!obj || typeof obj !== 'object') {
+		return obj;
+	}
+
+	options._clones ??= new WeakMap();
+	let { _clones } = options;
+
+	if (_clones.has(obj)) {
+		return _clones.get(obj);
+	}
+
+	let ret = obj;
+
+	if (Array.isArray(obj)) {
+		ret = obj.map(item => deepClone(item, options));
+	}
+	else if (isPlainObject(obj)) {
+		ret = { ...obj };
+
+		for (let key in obj) {
+			ret[key] = deepClone(obj[key], options);
+		}
+	}
+
+	_clones.set(obj, ret);
+	return ret;
+}
+
+/**
+ * Like Object.assign() but preserves accessors
+ * @param target
+ * @param sources
+ */
+export function betterAssign (target, ...sources) {
+	for (const source of sources) {
+		let descriptors = Object.getOwnPropertyDescriptors(source);
+		for (const key in descriptors) {
+			if (Object.hasOwn(target, key)) {
+				continue;
+			}
+
+			const descriptor = descriptors[key];
+			Object.defineProperty(target, key, descriptor);
+		}
+	}
+
+	return target;
+}
diff --git a/src/util/types.d.ts b/src/util/types.d.ts
new file mode 100644
index 0000000000..d72041d469
--- /dev/null
+++ b/src/util/types.d.ts
@@ -0,0 +1,3 @@
+export type KebabToCamelCase = S extends `${infer T}-${infer U}`
+	? `${T}${Capitalize>}`
+	: S;
diff --git a/tests/components-test.ts b/tests/components-test.ts
index ba14d7f173..7c4cd50ced 100644
--- a/tests/components-test.ts
+++ b/tests/components-test.ts
@@ -1,10 +1,11 @@
-import { assert } from 'chai';
 import { readFileSync } from 'fs';
 import path from 'path';
 import { fileURLToPath } from 'url';
+import { assert } from 'chai';
 import { noop } from '../src/shared/util';
 import { forEach, toArray } from '../src/util/iterables';
 import { getComponent, getComponentIds, getLanguageIds } from './helper/prism-loader';
+import type { LanguageProto } from '../src/types';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
@@ -42,7 +43,7 @@ describe('Components', () => {
 		for (const id of getComponentIds()) {
 			const proto = await getComponent(id).catch(noop);
 			add(id, 'a component id');
-			forEach(proto?.alias, a => add(a, `an alias of ${id}`));
+			forEach((proto as LanguageProto)?.alias, a => add(a, `an alias of ${id}`));
 		}
 	});
 });
@@ -101,7 +102,7 @@ describe('components.json', () => {
 	describe('- should have valid alias titles', () => {
 		for (const lang of getLanguageIds()) {
 			it(`- ${lang} should have all alias titles registered as alias`, async () => {
-				const aliases = new Set(toArray((await getComponent(lang)).alias));
+				const aliases = new Set(toArray(((await getComponent(lang)) as LanguageProto)?.alias));
 				const aliasTitles =
 					(components.languages[lang] as ComponentEntry | undefined)?.aliasTitles ?? {};
 
@@ -124,7 +125,6 @@ describe('components.json', () => {
 			.map((key): { id: string; title: string } => {
 				return {
 					id: key,
-					// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
 					title: (components.languages[key] as ComponentEntry).title!,
 				};
 			});
diff --git a/tests/core/registry.ts b/tests/core/registry.ts
index d2afb7a9cd..6dae213aa1 100644
--- a/tests/core/registry.ts
+++ b/tests/core/registry.ts
@@ -1,60 +1,34 @@
 import { assert } from 'chai';
 import { Prism } from '../../src/core/prism';
+import type { Grammar } from '../../src/types';
 
 describe('Registry', () => {
 	it('should resolve aliases', () => {
-		const { components } = new Prism();
+		const { languageRegistry } = new Prism();
 
-		const grammar = {};
-		components.add({ id: 'a', alias: 'b', grammar });
+		const grammar = {} as Grammar;
+		languageRegistry.add({ id: 'a', alias: 'b', grammar });
 
-		assert.isTrue(components.has('a'));
-		assert.isTrue(components.has('b'));
+		assert.equal(languageRegistry.resolveRef('a').id, 'a');
+		assert.equal(languageRegistry.resolveRef('b').id, 'a');
 
-		assert.strictEqual(components.resolveAlias('a'), 'a');
-		assert.strictEqual(components.resolveAlias('b'), 'a');
+		assert.strictEqual(languageRegistry.aliases['b'], 'a');
 
-		assert.strictEqual(components.getLanguage('a'), grammar);
-		assert.strictEqual(components.getLanguage('b'), grammar);
+		assert.deepStrictEqual(languageRegistry.getLanguage('a')?.resolvedGrammar, grammar);
+		assert.deepStrictEqual(languageRegistry.getLanguage('b')?.resolvedGrammar, grammar);
 	});
 
 	it('should resolve aliases in optional dependencies', () => {
-		const { components } = new Prism();
+		const { languageRegistry } = new Prism();
 
-		const grammar = {};
-		components.add({ id: 'a', alias: 'b', grammar });
-		components.add({
+		const grammar = {} as Grammar;
+		languageRegistry.add({ id: 'a', alias: 'b', grammar });
+		languageRegistry.add({
 			id: 'c',
 			optional: 'b',
-			grammar ({ getOptionalLanguage }) {
-				return getOptionalLanguage('b') ?? {};
-			},
+			grammar: ({ getLanguage }) => getLanguage('b') ?? {},
 		});
 
-		assert.strictEqual(components.getLanguage('c'), grammar);
-	});
-
-	it('should throw on circular dependencies', () => {
-		assert.throws(() => {
-			const { components } = new Prism();
-
-			components.add({ id: 'a', optional: 'b', grammar: {} });
-			components.add({ id: 'b', optional: 'a', grammar: {} });
-		}, /Circular dependency a -> b -> a not allowed/);
-
-		assert.throws(() => {
-			const { components } = new Prism();
-
-			components.add(
-				{ id: 'a', optional: 'b', grammar: {} },
-				{ id: 'b', optional: 'a', grammar: {} }
-			);
-		}, /Circular dependency a -> b -> a not allowed/);
-
-		assert.throws(() => {
-			const { components } = new Prism();
-
-			components.add({ id: 'a', optional: 'a', grammar: {} });
-		}, /Circular dependency a -> a not allowed/);
+		assert.deepStrictEqual(languageRegistry.getLanguage('c')?.resolvedGrammar, grammar);
 	});
 });
diff --git a/tests/coverage.ts b/tests/coverage.ts
index 2738a52c2d..872da0d341 100644
--- a/tests/coverage.ts
+++ b/tests/coverage.ts
@@ -17,14 +17,13 @@ describe('Pattern test coverage', () => {
 		const Prism = await PrismLoader.createInstance(languages);
 
 		const root = Object.fromEntries(
-			[...Prism.components['entries'].keys()].map(id => [
+			Object.keys(Prism.languageRegistry.cache).map(id => [
 				id,
-				Prism.components.getLanguage(id),
+				Prism.languageRegistry.getLanguage(id)?.resolvedGrammar,
 			])
 		);
 
 		BFS(root, (path, object) => {
-			// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
 			const { key, value } = path[path.length - 1];
 			const tokenPath = BFSPathToPrismTokenPath(path);
 
@@ -78,8 +77,8 @@ describe('Pattern test coverage', () => {
 					try {
 						await runTestCase(languageIdentifier, filePath, 'none', createInstance);
 					}
-					catch (error) {
-						// we don't case about whether the test succeeds,
+					catch (error) { // eslint-disable-line @typescript-eslint/no-unused-vars
+						// we don't care about whether the test succeeds,
 						// we just want to gather usage data
 					}
 				}
@@ -112,7 +111,7 @@ describe('Pattern test coverage', () => {
 				});
 
 				it(`- should exhaustively cover all keywords in keyword lists`, () => {
-					const problems = [];
+					const problems: string[] = [];
 
 					for (const data of getAllOf(language)) {
 						if (data.matches.length === 0) {
diff --git a/tests/helper/prism-dom-util.ts b/tests/helper/prism-dom-util.ts
index 4f802be1b8..020104d5c4 100644
--- a/tests/helper/prism-dom-util.ts
+++ b/tests/helper/prism-dom-util.ts
@@ -1,7 +1,7 @@
 import { createPrismDOM } from './prism-loader';
 import { assertEqual, useSnapshot } from './snapshot';
 import { formatHtml } from './util';
-import type { KebabToCamelCase } from '../../src/types';
+import type { KebabToCamelCase } from '../../src/util/types';
 import type { PrismDOM, PrismWindow } from './prism-loader';
 
 interface AssertOptions {
diff --git a/tests/helper/prism-loader.ts b/tests/helper/prism-loader.ts
index a10cd4afc0..9abd069e07 100644
--- a/tests/helper/prism-loader.ts
+++ b/tests/helper/prism-loader.ts
@@ -1,12 +1,12 @@
 import { readdirSync } from 'fs';
-import { JSDOM } from 'jsdom';
 import path from 'path';
+import { fileURLToPath } from 'url';
+import { JSDOM } from 'jsdom';
 import { Prism } from '../../src/core/prism';
 import { isNonNull, lazy, noop } from '../../src/shared/util';
 import { toArray } from '../../src/util/iterables';
 import type { ComponentProto, LanguageProto, PluginProto } from '../../src/types';
 import type { DOMWindow } from 'jsdom';
-import { fileURLToPath } from 'url';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const SRC_DIR = path.join(__dirname, '../../src');
@@ -63,7 +63,9 @@ export async function createInstance (languages?: string | string[]) {
 	const instance = new Prism();
 
 	const protos = await Promise.all(toArray(languages).map(getComponent));
-	instance.components.add(...protos);
+	protos.filter(Boolean).forEach(proto => {
+		instance.languageRegistry.add(proto as LanguageProto);
+	});
 
 	return instance;
 }
@@ -142,7 +144,14 @@ export function createPrismDOM (): PrismDOM<{}> {
 	const load = async (languagesOrPlugins: string | string[]) => {
 		const protos = await Promise.all(toArray(languagesOrPlugins).map(getComponent));
 		withGlobals(() => {
-			instance.components.add(...protos);
+			protos.filter(Boolean).forEach(proto => {
+				if (proto.grammar) {
+					instance.languageRegistry.add(proto);
+				}
+				else {
+					instance.pluginRegistry.add(proto);
+				}
+			});
 		});
 	};
 
diff --git a/tests/helper/test-case.ts b/tests/helper/test-case.ts
index 29f31378c8..fdd59a9922 100644
--- a/tests/helper/test-case.ts
+++ b/tests/helper/test-case.ts
@@ -3,8 +3,8 @@ import fs from 'fs';
 import { createInstance } from './prism-loader';
 import * as TokenStreamTransformer from './token-stream-transformer';
 import { formatHtml, getLeadingSpaces } from './util';
-import type { Prism } from '../../src/core';
-import type { TokenStream } from '../../src/core/classes/token';
+import type { Prism } from '../../src/types';
+import type { TokenStream } from '../../src/types';
 
 const defaultCreateInstance = createInstance;
 
@@ -134,7 +134,7 @@ interface Runner {
 }
 const jsonRunner: Runner = {
 	run (Prism, code, language) {
-		const grammar = Prism.components.getLanguage(language);
+		const grammar = Prism.languageRegistry.getLanguage(language)?.resolvedGrammar;
 		return Prism.tokenize(code, grammar ?? {});
 	},
 	print (actual) {
diff --git a/tests/helper/util.ts b/tests/helper/util.ts
index bc56816d1f..d93b640a86 100644
--- a/tests/helper/util.ts
+++ b/tests/helper/util.ts
@@ -12,13 +12,11 @@ const astCache = new Map();
 
 export interface PathItem {
 	key: string | null;
-	// eslint-disable-next-line @typescript-eslint/no-explicit-any
-	value: any;
+		value: any;
 }
 /**
  * Performs a breadth-first search on the given start element.
  */
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
 export function BFS (start: any, callback: (path: PathItem[], obj: Record) => void) {
 	const visited = new Set();
 	let toVisit: PathItem[][] = [
diff --git a/tests/identifier-test.ts b/tests/identifier-test.ts
index 157ee24776..7b65af2729 100644
--- a/tests/identifier-test.ts
+++ b/tests/identifier-test.ts
@@ -3,7 +3,7 @@ import { toArray } from '../src/util/iterables';
 import { createInstance, getComponent, getLanguageIds } from './helper/prism-loader';
 import { prettyprint } from './helper/token-stream-transformer';
 import type { Prism, Token } from '../src/core';
-import type { TokenStream } from '../src/core/classes/token';
+import type { TokenStream } from '../src/types';
 
 // This is where you can exclude a language from the identifier test.
 //
@@ -89,7 +89,7 @@ for (const lang of getLanguageIds()) {
 	describe(`Patterns of '${lang}' with optional dependencies`, () => {
 		const getPrism = async () => {
 			const component = await getComponent(lang);
-			const optional = toArray(component.optional);
+			const optional = toArray(component?.optional);
 			const Prism = await createInstance([lang, ...optional]);
 			return Prism;
 		};
@@ -122,8 +122,8 @@ function testLiterals (getPrism: Promise, lang: string) {
 		identifierType: keyof IdentifierTestOptions
 	) {
 		const Prism = await getPrism;
-		for (const id of Prism.components['entries'].keys()) {
-			const grammar = Prism.components.getLanguage(id);
+		for (const id of Object.keys(Prism.languageRegistry.cache)) {
+			const grammar = Prism.languageRegistry.getLanguage(id)?.resolvedGrammar;
 			if (!grammar) {
 				continue;
 			}
diff --git a/tests/pattern-tests.ts b/tests/pattern-tests.ts
index 9f8007e410..65cd6457be 100644
--- a/tests/pattern-tests.ts
+++ b/tests/pattern-tests.ts
@@ -1,13 +1,13 @@
 import { assert } from 'chai';
 import {
-	JS,
-	NFA,
-	Transformers,
-	Words,
 	combineTransformers,
 	getIntersectionWordSets,
 	isDisjointWith,
+	JS,
+	NFA,
 	transform,
+	Transformers,
+	Words,
 } from 'refa';
 import * as RAA from 'regexp-ast-analysis';
 import { visitRegExpAST } from 'regexpp';
@@ -16,7 +16,7 @@ import { lazy } from '../src/shared/util';
 import { toArray } from '../src/util/iterables';
 import * as args from './helper/args';
 import { createInstance, getComponent, getLanguageIds } from './helper/prism-loader';
-import { TestCaseFile, parseLanguageNames } from './helper/test-case';
+import { parseLanguageNames, TestCaseFile } from './helper/test-case';
 import { loadAllTests } from './helper/test-discovery';
 import { BFS, BFSPathToPrismTokenPath, isRegExp, parseRegex } from './helper/util';
 import type { Prism } from '../src/core';
@@ -91,7 +91,6 @@ function testPatterns (getPrism: () => Promise, mainLanguage:
 		ast: LiteralAST;
 		tokenPath: string;
 		name: string;
-		// eslint-disable-next-line @typescript-eslint/no-explicit-any
 		parent: any;
 		lookbehind: boolean;
 		lookbehindGroup: CapturingGroup | undefined;
@@ -109,7 +108,6 @@ function testPatterns (getPrism: () => Promise, mainLanguage:
 			visited.add(grammar);
 
 			BFS(grammar, path => {
-				// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
 				const { key, value } = path[path.length - 1];
 				const tokenPath = BFSPathToPrismTokenPath(path, rootStr);
 				visited.add(value);
@@ -126,9 +124,7 @@ function testPatterns (getPrism: () => Promise, mainLanguage:
 							);
 						}
 
-						// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
 						const parent = path.length > 1 ? path[path.length - 2].value : undefined;
-						// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
 						const lookbehind =
 							key === 'pattern' && !!parent && !!(parent as GrammarToken).lookbehind;
 						const lookbehindGroup = lookbehind
@@ -139,7 +135,6 @@ function testPatterns (getPrism: () => Promise, mainLanguage:
 							ast,
 							tokenPath,
 							name: key,
-							// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
 							parent,
 							path,
 							lookbehind,
@@ -160,8 +155,8 @@ function testPatterns (getPrism: () => Promise, mainLanguage:
 		}
 
 		// static analysis
-		for (const id of Prism.components['entries'].keys()) {
-			const grammar = Prism.components.getLanguage(id);
+		for (const id of Object.keys(Prism.languageRegistry.cache)) {
+			const grammar = Prism.languageRegistry.getLanguage(id)?.resolvedGrammar;
 			if (grammar) {
 				traverse(grammar, id);
 			}
@@ -170,7 +165,7 @@ function testPatterns (getPrism: () => Promise, mainLanguage:
 		// dynamic analysis
 		for (const lang of getRelevantLanguages()) {
 			const snippets = testSnippets.get(lang);
-			const grammar = Prism.components.getLanguage(lang);
+			const grammar = Prism.languageRegistry.getLanguage(lang)?.resolvedGrammar;
 
 			// eslint-disable-next-line @typescript-eslint/unbound-method
 			const oldTokenize = Prism.tokenize;
@@ -283,7 +278,7 @@ function testPatterns (getPrism: () => Promise, mainLanguage:
 			forEachCapturingGroup(ast.pattern, ({ group, number }) => {
 				const isLookbehindGroup = group === lookbehindGroup;
 				if (group.references.length === 0 && !isLookbehindGroup) {
-					const fixes = [];
+					const fixes: string[] = [];
 					fixes.push(
 						`Make this group a non-capturing group ('(?:...)' instead of '(...)'). (It's usually this option.)`
 					);
@@ -407,7 +402,7 @@ function testPatterns (getPrism: () => Promise, mainLanguage:
  * Returns the first capturing group in the given pattern.
  */
 function getFirstCapturingGroup (pattern: Pattern): CapturingGroup | undefined {
-	let cap = undefined;
+	let cap: CapturingGroup | undefined = undefined;
 
 	try {
 		visitRegExpAST(pattern, {
@@ -417,7 +412,7 @@ function getFirstCapturingGroup (pattern: Pattern): CapturingGroup | undefined {
 			},
 		});
 	}
-	catch (error) {
+	catch (error) { // eslint-disable-line @typescript-eslint/no-unused-vars
 		// ignore errors
 	}
 
@@ -706,7 +701,10 @@ function checkPolynomialBacktracking (path: string, pattern: RegExp, ast?: Liter
 		ast = parseRegex(pattern);
 	}
 
-	const result = scslre.analyse(ast as Readonly, { maxReports: 1, reportTypes: { 'Move': false } });
+	const result = scslre.analyse(ast as Readonly, {
+		maxReports: 1,
+		reportTypes: { 'Move': false },
+	});
 	if (result.reports.length > 0) {
 		const report = result.reports[0];
 
@@ -782,9 +780,9 @@ interface Highlight {
 function highlight (highlights: Highlight[], offset = 0) {
 	highlights.sort((a, b) => a.start - b.start);
 
-	const lines = [];
+	const lines: string[] = [];
 	while (highlights.length > 0) {
-		const newHighlights = [];
+		const newHighlights: Highlight[] = [];
 		let l = '';
 		for (const highlight of highlights) {
 			const start = highlight.start + offset;
diff --git a/tsconfig.json b/tsconfig.json
index 1be6e5effd..e6c1d4efb3 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -11,8 +11,8 @@
 		// "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
 
 		/* Language and Environment */
-		"target": "es2020" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
-		// "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
+		"target": "es2022" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
+		"lib": ["es2022", "dom"] /* Specify a set of bundled library declaration files that describe the target runtime environment. */,
 		// "jsx": "preserve",                                /* Specify what JSX code is generated. */
 		// "experimentalDecorators": true,                   /* Enable experimental support for TC39 stage 2 draft decorators. */
 		// "emitDecoratorMetadata": true,                    /* Emit design-type metadata for decorated declarations in source files. */
@@ -76,8 +76,8 @@
 		"forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
 
 		/* Type Checking */
-		"strict": true /* Enable all strict type-checking options. */,
-		// "noImplicitAny": true,                            /* Enable error reporting for expressions and declarations with an implied 'any' type. */
+		"strict": true                                       /* Enable all strict type-checking options. */,
+		"noImplicitAny": false                               /* Enable error reporting for expressions and declarations with an implied 'any' type. */,
 		// "strictNullChecks": true,                         /* When type checking, take into account 'null' and 'undefined'. */
 		// "strictFunctionTypes": true,                      /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
 		// "strictBindCallApply": true,                      /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */