diff --git a/src/grammar/grammar.ts b/src/grammar/grammar.ts index da0a9f7e..4ebf4bae 100644 --- a/src/grammar/grammar.ts +++ b/src/grammar/grammar.ts @@ -9,6 +9,7 @@ import { createMatchers, Matcher } from '../matcher'; import { disposeOnigString, IOnigLib, OnigScanner, OnigString } from '../onigLib'; import { IRawGrammar, IRawRepository, IRawRule } from '../rawGrammar'; import { ruleIdFromNumber, IRuleFactoryHelper, IRuleRegistry, Rule, RuleFactory, RuleId, ruleIdToNumber } from '../rule'; +import { COMPUTE_FONTS } from '../tests/themes.test'; import { FontStyle, ScopeName, ScopePath, ScopeStack, StyleAttributes } from '../theme'; import { clone, containsRTL } from '../utils'; import { BasicScopeAttributes, BasicScopeAttributesProvider } from './basicScopesAttributeProvider'; @@ -1135,6 +1136,14 @@ export class FontInfo implements IFontInfo { } } +export let TOTAL_PROPERTY_COUNT = 0; +export let TOTAL_GET_ATTR_CALLS = 0; + +export function resetVariables() { + TOTAL_PROPERTY_COUNT = 0; + TOTAL_GET_ATTR_CALLS = 0; +} + export class LineFonts { private readonly _fonts: FontInfo[] = []; @@ -1151,14 +1160,12 @@ export class LineFonts { scopesList: AttributedScopeStack | null, endIndex: number ): void { - const styleAttributes = scopesList?.styleAttributes; - if (!styleAttributes) { - this._lastIndex = endIndex; + if (!COMPUTE_FONTS) { return; } - const fontFamily = styleAttributes.fontFamily; - const fontSizeMultiplier = styleAttributes.fontSize; - const lineHeightMultiplier = styleAttributes.lineHeight; + const fontFamily = this.getFontFamily(scopesList); + const fontSizeMultiplier = this.getFontSize(scopesList); + const lineHeightMultiplier = this.getLineHeight(scopesList); if (!fontFamily && !fontSizeMultiplier && !lineHeightMultiplier) { this._lastIndex = endIndex; return; @@ -1182,4 +1189,35 @@ export class LineFonts { public getResult(): IFontInfo[] { return this._fonts; } + + private getFontFamily(scopesList: AttributedScopeStack | null): string | null { + TOTAL_PROPERTY_COUNT++; + return this.getAttribute(scopesList, (styleAttributes) => { return styleAttributes.fontFamily; }); + } + + private getFontSize(scopesList: AttributedScopeStack | null): number | null { + TOTAL_PROPERTY_COUNT++; + return this.getAttribute(scopesList, (styleAttributes) => { return styleAttributes.fontSize; }); + } + + private getLineHeight(scopesList: AttributedScopeStack | null): number | null { + TOTAL_PROPERTY_COUNT++; + return this.getAttribute(scopesList, (styleAttributes) => { return styleAttributes.lineHeight; }); + } + + private getAttribute(scopesList: AttributedScopeStack | null, getAttr: (styleAttributes: StyleAttributes) => any | null): any | null { + TOTAL_GET_ATTR_CALLS++; + if (!scopesList) { + return null; + } + const styleAttributes = scopesList.styleAttributes; + if (!styleAttributes) { + return null; + } + const attribute = getAttr(styleAttributes); + if (attribute) { + return attribute; + } + return this.getAttribute(scopesList.parent, getAttr); + } } diff --git a/src/tests/themes.test.ts b/src/tests/themes.test.ts index 9da1ebc0..8e4920b4 100644 --- a/src/tests/themes.test.ts +++ b/src/tests/themes.test.ts @@ -14,11 +14,11 @@ import { fontStyleToString, StyleAttributes } from '../theme'; -import { ThemeTest } from './themeTest'; import { getOniguruma } from './onigLibs'; import { Resolver, IGrammarRegistration, ILanguageRegistration } from './resolver'; import { strArrCmp, strcmp } from '../utils'; import { parsePLIST } from '../plist'; +import { resetVariables, TOTAL_GET_ATTR_CALLS, TOTAL_PROPERTY_COUNT } from '../grammar'; const THEMES_TEST_PATH = path.join(__dirname, '../../test-cases/themes'); @@ -70,57 +70,79 @@ class ThemeInfo { } } -(function () { - let THEMES = [ - new ThemeInfo('abyss', 'Abyss.tmTheme'), - new ThemeInfo('dark_vs', 'dark_vs.json'), - new ThemeInfo('light_vs', 'light_vs.json'), - new ThemeInfo('hc_black', 'hc_black.json'), - new ThemeInfo('dark_plus', 'dark_plus.json', 'dark_vs.json'), - new ThemeInfo('light_plus', 'light_plus.json', 'light_vs.json'), - new ThemeInfo('kimbie_dark', 'Kimbie_dark.tmTheme'), - new ThemeInfo('monokai', 'Monokai.tmTheme'), - new ThemeInfo('monokai_dimmed', 'dimmed-monokai.tmTheme'), - new ThemeInfo('quietlight', 'QuietLight.tmTheme'), - new ThemeInfo('red', 'red.tmTheme'), - new ThemeInfo('solarized_dark', 'Solarized-dark.tmTheme'), - new ThemeInfo('solarized_light', 'Solarized-light.tmTheme'), - new ThemeInfo('tomorrow_night_blue', 'Tomorrow-Night-Blue.tmTheme'), - ]; +export let COMPUTE_FONTS: boolean = false; + +test.only('Tokenize test.ts with TypeScript grammar and dark_vs theme', async () => { + + resetVariables(); + + // Load dark_vs theme + const themeFile = path.join(THEMES_TEST_PATH, 'dark_vs.json'); + const themeContent = fs.readFileSync(themeFile).toString(); + const theme: IRawTheme = JSON.parse(themeContent); + + // Load TypeScript grammar + const grammarsData: IGrammarRegistration[] = JSON.parse(fs.readFileSync(path.join(THEMES_TEST_PATH, 'grammars.json')).toString('utf8')); + const languagesData: ILanguageRegistration[] = JSON.parse(fs.readFileSync(path.join(THEMES_TEST_PATH, 'languages.json')).toString('utf8')); - // Load all language/grammar metadata - let _grammars: IGrammarRegistration[] = JSON.parse(fs.readFileSync(path.join(THEMES_TEST_PATH, 'grammars.json')).toString('utf8')); - for (let grammar of _grammars) { + // Update paths for grammars + for (let grammar of grammarsData) { grammar.path = path.join(THEMES_TEST_PATH, grammar.path); } - let _languages: ILanguageRegistration[] = JSON.parse(fs.readFileSync(path.join(THEMES_TEST_PATH, 'languages.json')).toString('utf8')); - - let _resolver = new Resolver(_grammars, _languages, getOniguruma()); - let _themeData = THEMES.map(theme => theme.create(_resolver)); - - // Discover all tests - let testFiles = fs.readdirSync(path.join(THEMES_TEST_PATH, 'tests')); - testFiles = testFiles.filter(testFile => !/\.result$/.test(testFile)); - testFiles = testFiles.filter(testFile => !/\.result.patch$/.test(testFile)); - testFiles = testFiles.filter(testFile => !/\.actual$/.test(testFile)); - testFiles = testFiles.filter(testFile => !/\.diff.html$/.test(testFile)); - - for (let testFile of testFiles) { - let tst = new ThemeTest(THEMES_TEST_PATH, testFile, _themeData, _resolver); - test(tst.testName, async function () { - this.timeout(20000); - try { - await tst.evaluate(); - assert.deepStrictEqual(tst.actual, tst.expected); - } catch(err) { - tst.writeExpected(); - throw err; - } - }); - } + const resolver = new Resolver(grammarsData, languagesData, getOniguruma()); + const registry = new Registry(resolver); + registry.setTheme(theme); + + // Load TypeScript grammar + const tsGrammar = await registry.loadGrammar('source.ts'); + assert.ok(tsGrammar, 'TypeScript grammar should be loaded'); + + // Read test.ts file + const testFilePath = path.join(THEMES_TEST_PATH, 'fixtures/test.ts'); + const testFileContent = fs.readFileSync(testFilePath).toString('utf8'); + const lines = testFileContent.split(/\r\n|\r|\n/); + + // Tokenize all lines + const tokenizeLine = () => { + let ruleStack = null; + const tokenizedLines: any[] = []; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const result = tsGrammar.tokenizeLine2(line, ruleStack); + ruleStack = result.ruleStack; + tokenizedLines.push({ + line: i + 1, + tokens: result.tokens, + fonts: result.fonts + }); + } + return tokenizedLines; + }; + + // Verify we tokenized all lines + COMPUTE_FONTS = false; + const startComputeFontsFalse = Date.now(); + tokenizeLine(); + const endComputeFontsFalse = Date.now(); + + console.log('Tokenization time without fonts (ms):', (endComputeFontsFalse - startComputeFontsFalse)); + + COMPUTE_FONTS = true; + const startComputeFontsTrue = Date.now(); + tokenizeLine(); + const endComputeFontsTrue = Date.now(); + + console.log('Tokenization time with fonts (ms):', (endComputeFontsTrue - startComputeFontsTrue)); + + const getAttrCallsPerPropertyCount = TOTAL_GET_ATTR_CALLS / TOTAL_PROPERTY_COUNT; + + console.log('TOTAL_PROPERTY_COUNT : ', TOTAL_PROPERTY_COUNT); + console.log('TOTAL_GET_ATTR_CALLS : ', TOTAL_GET_ATTR_CALLS); + + console.log('TOTAL_PROPERTY_COUNT / TOTAL_GET_ATTR_CALLS :', getAttrCallsPerPropertyCount); +}); -})(); test('Theme matching gives higher priority to deeper matches', () => { const theme = Theme.createFromRawTheme({ diff --git a/test-cases/themes/dark_vs.json b/test-cases/themes/dark_vs.json index b717ba64..0c1da29a 100644 --- a/test-cases/themes/dark_vs.json +++ b/test-cases/themes/dark_vs.json @@ -25,23 +25,28 @@ "foreground": "#000080" } }, - { "scope": "comment", "settings": { - "foreground": "#608b4e" + "foreground": "#608b4e", + "fontSize": 12, + "lineHeight": 18 } }, { "scope": "constant.language", "settings": { - "foreground": "#569cd6" + "foreground": "#569cd6", + "fontSize": 14, + "lineHeight": 20 } }, { "scope": "constant.numeric", "settings": { - "foreground": "#b5cea8" + "foreground": "#b5cea8", + "fontSize": 13, + "lineHeight": 19 } }, { @@ -63,7 +68,9 @@ { "scope": "entity.name.tag", "settings": { - "foreground": "#569cd6" + "foreground": "#569cd6", + "fontSize": 14, + "lineHeight": 21 } }, { @@ -75,7 +82,9 @@ { "scope": "entity.other.attribute-name", "settings": { - "foreground": "#9cdcfe" + "foreground": "#9cdcfe", + "fontSize": 13, + "lineHeight": 19 } }, { @@ -86,9 +95,7 @@ "entity.other.attribute-name.parent-selector.css", "entity.other.attribute-name.pseudo-class.css", "entity.other.attribute-name.pseudo-element.css", - "source.css.less entity.other.attribute-name.id", - "entity.other.attribute-name.attribute.scss", "entity.other.attribute-name.scss" ], @@ -119,7 +126,9 @@ "scope": "markup.heading", "settings": { "fontStyle": "bold", - "foreground": "#569cd6" + "foreground": "#569cd6", + "fontSize": 16, + "lineHeight": 24 } }, { @@ -180,7 +189,9 @@ { "scope": "meta.preprocessor", "settings": { - "foreground": "#569cd6" + "foreground": "#569cd6", + "fontSize": 13, + "lineHeight": 18 } }, { @@ -210,7 +221,9 @@ { "scope": "storage", "settings": { - "foreground": "#569cd6" + "foreground": "#569cd6", + "fontSize": 14, + "lineHeight": 20 } }, { @@ -228,7 +241,9 @@ { "scope": "string", "settings": { - "foreground": "#ce9178" + "foreground": "#ce9178", + "fontSize": 13, + "lineHeight": 19 } }, { @@ -262,13 +277,17 @@ { "scope": "support.type.property-name", "settings": { - "foreground": "#9cdcfe" + "foreground": "#9cdcfe", + "fontSize": 13, + "lineHeight": 19 } }, { "scope": "keyword", "settings": { - "foreground": "#569cd6" + "foreground": "#569cd6", + "fontSize": 14, + "lineHeight": 21 } }, { @@ -284,7 +303,10 @@ } }, { - "scope": ["keyword.operator.new", "keyword.operator.expression"], + "scope": [ + "keyword.operator.new", + "keyword.operator.expression" + ], "settings": { "foreground": "#569cd6" } @@ -318,7 +340,10 @@ }, { "name": "coloring of the Java import and package identifiers", - "scope": ["storage.modifier.import.java", "storage.modifier.package.java"], + "scope": [ + "storage.modifier.import.java", + "storage.modifier.package.java" + ], "settings": { "foreground": "#d4d4d4" } @@ -327,7 +352,9 @@ "name": "this.self", "scope": "variable.language", "settings": { - "foreground": "#569cd6" + "foreground": "#569cd6", + "fontSize": 14, + "lineHeight": 20 } } ] diff --git a/test-cases/themes/fixtures/test.ts b/test-cases/themes/fixtures/test.ts new file mode 100644 index 00000000..9deebfd0 --- /dev/null +++ b/test-cases/themes/fixtures/test.ts @@ -0,0 +1,1240 @@ +/*--------------------------------------------------------- + * Copyright (C) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------*/ + +import { DebugFlags } from '../debug'; +import { EncodedTokenAttributes, OptionalStandardTokenType, StandardTokenType, toOptionalTokenType } from '../encodedTokenAttributes'; +import { IEmbeddedLanguagesMap, IGrammar, IToken, ITokenizeLineResult, ITokenizeLineResult2, ITokenTypeMap, StateStack, IFontInfo } from '../main'; +import { createMatchers, Matcher } from '../matcher'; +import { disposeOnigString, IOnigLib, OnigScanner, OnigString } from '../onigLib'; +import { IRawGrammar, IRawRepository, IRawRule } from '../rawGrammar'; +import { ruleIdFromNumber, IRuleFactoryHelper, IRuleRegistry, Rule, RuleFactory, RuleId, ruleIdToNumber } from '../rule'; +import { FontStyle, ScopeName, ScopePath, ScopeStack, StyleAttributes } from '../theme'; +import { clone, containsRTL } from '../utils'; +import { BasicScopeAttributes, BasicScopeAttributesProvider } from './basicScopesAttributeProvider'; +import { _tokenizeString } from './tokenizeString'; + +export function createGrammar( + scopeName: ScopeName, + grammar: IRawGrammar, + initialLanguage: number, + embeddedLanguages: IEmbeddedLanguagesMap | null, + tokenTypes: ITokenTypeMap | null, + balancedBracketSelectors: BalancedBracketSelectors | null, + grammarRepository: IGrammarRepository & IThemeProvider, + onigLib: IOnigLib +): Grammar { + return new Grammar( + scopeName, + grammar, + initialLanguage, + embeddedLanguages, + tokenTypes, + balancedBracketSelectors, + grammarRepository, + onigLib + ); //TODO +} + +export interface IThemeProvider { + themeMatch(scopePath: ScopeStack): StyleAttributes | null; + getDefaults(): StyleAttributes; +} + +export interface IGrammarRepository { + lookup(scopeName: ScopeName): IRawGrammar | undefined; + injections(scopeName: ScopeName): ScopeName[]; +} + +export interface Injection { + readonly debugSelector: string; + readonly matcher: Matcher; + readonly priority: -1 | 0 | 1; // 0 is the default. -1 for 'L' and 1 for 'R' + readonly ruleId: RuleId; + readonly grammar: IRawGrammar; +} + +function collectInjections(result: Injection[], selector: string, rule: IRawRule, ruleFactoryHelper: IRuleFactoryHelper, grammar: IRawGrammar): void { + const matchers = createMatchers(selector, nameMatcher); + const ruleId = RuleFactory.getCompiledRuleId(rule, ruleFactoryHelper, grammar.repository); + for (const matcher of matchers) { + result.push({ + debugSelector: selector, + matcher: matcher.matcher, + ruleId: ruleId, + grammar: grammar, + priority: matcher.priority + }); + } +} + +function nameMatcher(identifers: ScopeName[], scopes: ScopeName[]): boolean { + if (scopes.length < identifers.length) { + return false; + } + let lastIndex = 0; + return identifers.every(identifier => { + for (let i = lastIndex; i < scopes.length; i++) { + if (scopesAreMatching(scopes[i], identifier)) { + lastIndex = i + 1; + return true; + } + } + return false; + }); +} + +function scopesAreMatching(thisScopeName: string, scopeName: string): boolean { + if (!thisScopeName) { + return false; + } + if (thisScopeName === scopeName) { + return true; + } + const len = scopeName.length; + return thisScopeName.length > len && thisScopeName.substr(0, len) === scopeName && thisScopeName[len] === '.'; +} + +export class Grammar implements IGrammar, IRuleFactoryHelper, IOnigLib { + private _rootId: RuleId | -1; + private _lastRuleId: number; + private readonly _ruleId2desc: Rule[]; + private readonly _includedGrammars: { [scopeName: string]: IRawGrammar }; + private readonly _grammarRepository: IGrammarRepository & IThemeProvider; + private readonly _grammar: IRawGrammar; + private _injections: Injection[] | null; + private readonly _basicScopeAttributesProvider: BasicScopeAttributesProvider; + private readonly _tokenTypeMatchers: TokenTypeMatcher[]; + + public get themeProvider(): IThemeProvider { return this._grammarRepository; } + + constructor( + private readonly _rootScopeName: ScopeName, + grammar: IRawGrammar, + initialLanguage: number, + embeddedLanguages: IEmbeddedLanguagesMap | null, + tokenTypes: ITokenTypeMap | null, + private readonly balancedBracketSelectors: BalancedBracketSelectors | null, + grammarRepository: IGrammarRepository & IThemeProvider, + private readonly _onigLib: IOnigLib + ) { + this._basicScopeAttributesProvider = new BasicScopeAttributesProvider( + initialLanguage, + embeddedLanguages + ); + + this._rootId = -1; + this._lastRuleId = 0; + this._ruleId2desc = [null!]; + this._includedGrammars = {}; + this._grammarRepository = grammarRepository; + this._grammar = initGrammar(grammar, null); + this._injections = null; + + this._tokenTypeMatchers = []; + if (tokenTypes) { + for (const selector of Object.keys(tokenTypes)) { + const matchers = createMatchers(selector, nameMatcher); + for (const matcher of matchers) { + this._tokenTypeMatchers.push({ + matcher: matcher.matcher, + type: tokenTypes[selector], + }); + } + } + } + } + + public dispose(): void { + for (const rule of this._ruleId2desc) { + if (rule) { + rule.dispose(); + } + } + } + + public createOnigScanner(sources: string[]): OnigScanner { + return this._onigLib.createOnigScanner(sources); + } + + public createOnigString(sources: string): OnigString { + return this._onigLib.createOnigString(sources); + } + + public getMetadataForScope(scope: string): BasicScopeAttributes { + return this._basicScopeAttributesProvider.getBasicScopeAttributes(scope); + } + + private _collectInjections(): Injection[] { + const grammarRepository: IGrammarRepository = { + lookup: (scopeName: string): IRawGrammar | undefined => { + if (scopeName === this._rootScopeName) { + return this._grammar; + } + return this.getExternalGrammar(scopeName); + }, + injections: (scopeName: string): string[] => { + return this._grammarRepository.injections(scopeName); + }, + }; + + const result: Injection[] = []; + + const scopeName = this._rootScopeName; + + const grammar = grammarRepository.lookup(scopeName); + if (grammar) { + // add injections from the current grammar + const rawInjections = grammar.injections; + if (rawInjections) { + for (let expression in rawInjections) { + collectInjections( + result, + expression, + rawInjections[expression], + this, + grammar + ); + } + } + + // add injection grammars contributed for the current scope + + const injectionScopeNames = this._grammarRepository.injections(scopeName); + if (injectionScopeNames) { + injectionScopeNames.forEach((injectionScopeName) => { + const injectionGrammar = + this.getExternalGrammar(injectionScopeName); + if (injectionGrammar) { + const selector = injectionGrammar.injectionSelector; + if (selector) { + collectInjections( + result, + selector, + injectionGrammar, + this, + injectionGrammar + ); + } + } + }); + } + } + + result.sort((i1, i2) => i1.priority - i2.priority); // sort by priority + + return result; + } + + public getInjections(): Injection[] { + if (this._injections === null) { + this._injections = this._collectInjections(); + + if (DebugFlags.InDebugMode && this._injections.length > 0) { + console.log( + `Grammar ${this._rootScopeName} contains the following injections:` + ); + for (const injection of this._injections) { + console.log(` - ${injection.debugSelector}`); + } + } + } + return this._injections; + } + + public registerRule(factory: (id: RuleId) => T): T { + const id = ++this._lastRuleId; + const result = factory(ruleIdFromNumber(id)); + this._ruleId2desc[id] = result; + return result; + } + + public getRule(ruleId: RuleId): Rule { + return this._ruleId2desc[ruleIdToNumber(ruleId)]; + } + + public getExternalGrammar( + scopeName: string, + repository?: IRawRepository + ): IRawGrammar | undefined { + if (this._includedGrammars[scopeName]) { + return this._includedGrammars[scopeName]; + } else if (this._grammarRepository) { + const rawIncludedGrammar = + this._grammarRepository.lookup(scopeName); + if (rawIncludedGrammar) { + // console.log('LOADED GRAMMAR ' + pattern.include); + this._includedGrammars[scopeName] = initGrammar( + rawIncludedGrammar, + repository && repository.$base + ); + return this._includedGrammars[scopeName]; + } + } + return undefined; + } + + public tokenizeLine( + lineText: string, + prevState: StateStackImpl | null, + timeLimit: number = 0 + ): ITokenizeLineResult { + const r = this._tokenize(lineText, prevState, false, timeLimit); + return { + tokens: r.lineTokens.getResult(r.ruleStack, r.lineLength), + ruleStack: r.ruleStack, + stoppedEarly: r.stoppedEarly, + fonts: r.lineFonts.getResult() + }; + } + + public tokenizeLine2( + lineText: string, + prevState: StateStackImpl | null, + timeLimit: number = 0 + ): ITokenizeLineResult2 { + const r = this._tokenize(lineText, prevState, true, timeLimit); + return { + tokens: r.lineTokens.getBinaryResult(r.ruleStack, r.lineLength), + ruleStack: r.ruleStack, + stoppedEarly: r.stoppedEarly, + fonts: r.lineFonts.getResult() + }; + } + + private _tokenize( + lineText: string, + prevState: StateStackImpl | null, + emitBinaryTokens: boolean, + timeLimit: number + ): { + lineLength: number; + lineTokens: LineTokens; + lineFonts: LineFonts; + ruleStack: StateStackImpl; + stoppedEarly: boolean; + } { + if (this._rootId === -1) { + this._rootId = RuleFactory.getCompiledRuleId( + this._grammar.repository.$self, + this, + this._grammar.repository + ); + // This ensures ids are deterministic, and thus equal in renderer and webworker. + this.getInjections(); + } + + let isFirstLine: boolean; + if (!prevState || prevState === StateStackImpl.NULL) { + isFirstLine = true; + const rawDefaultMetadata = + this._basicScopeAttributesProvider.getDefaultAttributes(); + const defaultStyle = this.themeProvider.getDefaults(); + const defaultMetadata = EncodedTokenAttributes.set( + 0, + rawDefaultMetadata.languageId, + rawDefaultMetadata.tokenType, + null, + defaultStyle.fontStyle, + defaultStyle.foregroundId, + defaultStyle.backgroundId + ); + + const rootScopeName = this.getRule(this._rootId).getName( + null, + null + ); + + let scopeList: AttributedScopeStack; + if (rootScopeName) { + scopeList = AttributedScopeStack.createRootAndLookUpScopeName( + rootScopeName, + defaultMetadata, + this + ); + } else { + scopeList = AttributedScopeStack.createRoot( + "unknown", + defaultMetadata + ); + } + + prevState = new StateStackImpl( + null, + this._rootId, + -1, + -1, + false, + null, + scopeList, + scopeList + ); + } else { + isFirstLine = false; + prevState.reset(); + } + + lineText = lineText + "\n"; + const onigLineText = this.createOnigString(lineText); + const lineLength = onigLineText.content.length; + const lineTokens = new LineTokens( + emitBinaryTokens, + lineText, + this._tokenTypeMatchers, + this.balancedBracketSelectors + ); + const lineFonts = new LineFonts(); + const r = _tokenizeString( + this, + onigLineText, + isFirstLine, + 0, + prevState, + lineTokens, + lineFonts, + true, + timeLimit + ); + + disposeOnigString(onigLineText); + + return { + lineLength: lineLength, + lineTokens: lineTokens, + lineFonts: lineFonts, + ruleStack: r.stack, + stoppedEarly: r.stoppedEarly, + }; + } +} + +function initGrammar(grammar: IRawGrammar, base: IRawRule | null | undefined): IRawGrammar { + grammar = clone(grammar); + + grammar.repository = grammar.repository || {}; + grammar.repository.$self = { + $vscodeTextmateLocation: grammar.$vscodeTextmateLocation, + patterns: grammar.patterns, + name: grammar.scopeName + }; + grammar.repository.$base = base || grammar.repository.$self; + return grammar; +} + +export class AttributedScopeStack { + static fromExtension(namesScopeList: AttributedScopeStack | null, contentNameScopesList: AttributedScopeStackFrame[]): AttributedScopeStack | null { + let current = namesScopeList; + let scopeNames = namesScopeList?.scopePath ?? null; + for (const frame of contentNameScopesList) { + scopeNames = ScopeStack.push(scopeNames, frame.scopeNames); + current = new AttributedScopeStack(current, scopeNames!, frame.encodedTokenAttributes, null); + } + return current; + } + + public static createRoot(scopeName: ScopeName, tokenAttributes: EncodedTokenAttributes): AttributedScopeStack { + return new AttributedScopeStack(null, new ScopeStack(null, scopeName), tokenAttributes, null); + } + + public static createRootAndLookUpScopeName(scopeName: ScopeName, tokenAttributes: EncodedTokenAttributes, grammar: Grammar): AttributedScopeStack { + const rawRootMetadata = grammar.getMetadataForScope(scopeName); + const scopePath = new ScopeStack(null, scopeName); + const rootStyle = grammar.themeProvider.themeMatch(scopePath); + + const resolvedTokenAttributes = AttributedScopeStack.mergeAttributes( + tokenAttributes, + rawRootMetadata, + rootStyle + ); + + return new AttributedScopeStack(null, scopePath, resolvedTokenAttributes, rootStyle); + } + + public get scopeName(): ScopeName { return this.scopePath.scopeName; } + + /** + * Invariant: + * ``` + * if (parent && !scopePath.extends(parent.scopePath)) { + * throw new Error(); + * } + * ``` + */ + private constructor( + public readonly parent: AttributedScopeStack | null, + public readonly scopePath: ScopeStack, + public readonly tokenAttributes: EncodedTokenAttributes, + public readonly styleAttributes: StyleAttributes | null + ) { + } + + public toString() { + return this.getScopeNames().join(' '); + } + + public equals(other: AttributedScopeStack): boolean { + return AttributedScopeStack.equals(this, other); + } + + public static equals( + a: AttributedScopeStack | null, + b: AttributedScopeStack | null + ): boolean { + do { + if (a === b) { + return true; + } + + if (!a && !b) { + // End of list reached for both + return true; + } + + if (!a || !b) { + // End of list reached only for one + return false; + } + + if (a.scopeName !== b.scopeName || a.tokenAttributes !== b.tokenAttributes) { + return false; + } + + // Go to previous pair + a = a.parent; + b = b.parent; + } while (true); + } + + private static mergeAttributes( + existingTokenAttributes: EncodedTokenAttributes, + basicScopeAttributes: BasicScopeAttributes, + styleAttributes: StyleAttributes | null + ): EncodedTokenAttributes { + let fontStyle = FontStyle.NotSet; + let foreground = 0; + let background = 0; + + if (styleAttributes !== null) { + fontStyle = styleAttributes.fontStyle; + foreground = styleAttributes.foregroundId; + background = styleAttributes.backgroundId; + } + + return EncodedTokenAttributes.set( + existingTokenAttributes, + basicScopeAttributes.languageId, + basicScopeAttributes.tokenType, + null, + fontStyle, + foreground, + background + ); + } + + public pushAttributed(scopePath: ScopePath | null, grammar: Grammar): AttributedScopeStack { + if (scopePath === null) { + return this; + } + + if (scopePath.indexOf(' ') === -1) { + // This is the common case and much faster + + return AttributedScopeStack._pushAttributed(this, scopePath, grammar); + } + + const scopes = scopePath.split(/ /g); + let result: AttributedScopeStack = this; + for (const scope of scopes) { + result = AttributedScopeStack._pushAttributed(result, scope, grammar); + } + return result; + + } + + private static _pushAttributed( + target: AttributedScopeStack, + scopeName: ScopeName, + grammar: Grammar, + ): AttributedScopeStack { + const rawMetadata = grammar.getMetadataForScope(scopeName); + + const newPath = target.scopePath.push(scopeName); + const scopeThemeMatchResult = + grammar.themeProvider.themeMatch(newPath); + const metadata = AttributedScopeStack.mergeAttributes( + target.tokenAttributes, + rawMetadata, + scopeThemeMatchResult + ); + return new AttributedScopeStack(target, newPath, metadata, scopeThemeMatchResult); + } + + public getScopeNames(): string[] { + return this.scopePath.getSegments(); + } + + public getExtensionIfDefined(base: AttributedScopeStack | null): AttributedScopeStackFrame[] | undefined { + const result: AttributedScopeStackFrame[] = []; + let self: AttributedScopeStack | null = this; + + while (self && self !== base) { + result.push({ + encodedTokenAttributes: self.tokenAttributes, + scopeNames: self.scopePath.getExtensionIfDefined(self.parent?.scopePath ?? null)!, + }); + self = self.parent; + } + return self === base ? result.reverse() : undefined; + } +} + +interface AttributedScopeStackFrame { + encodedTokenAttributes: number; + scopeNames: string[]; +} + +/** + * Represents a "pushed" state on the stack (as a linked list element). + */ +export class StateStackImpl implements StateStack { + _stackElementBrand: void = undefined; + + // TODO remove me + public static NULL = new StateStackImpl( + null, + 0 as any, + 0, + 0, + false, + null, + null, + null + ); + + /** + * The position on the current line where this state was pushed. + * This is relevant only while tokenizing a line, to detect endless loops. + * Its value is meaningless across lines. + */ + private _enterPos: number; + + /** + * The captured anchor position when this stack element was pushed. + * This is relevant only while tokenizing a line, to restore the anchor position when popping. + * Its value is meaningless across lines. + */ + private _anchorPos: number; + + + /** + * The depth of the stack. + */ + public readonly depth: number; + + + /** + * Invariant: + * ``` + * if (contentNameScopesList !== nameScopesList && contentNameScopesList?.parent !== nameScopesList) { + * throw new Error(); + * } + * if (this.parent && !nameScopesList.extends(this.parent.contentNameScopesList)) { + * throw new Error(); + * } + * ``` + */ + constructor( + /** + * The previous state on the stack (or null for the root state). + */ + public readonly parent: StateStackImpl | null, + + /** + * The state (rule) that this element represents. + */ + private readonly ruleId: RuleId, + + enterPos: number, + anchorPos: number, + + /** + * The state has entered and captured \n. This means that the next line should have an anchorPosition of 0. + */ + public readonly beginRuleCapturedEOL: boolean, + + /** + * The "pop" (end) condition for this state in case that it was dynamically generated through captured text. + */ + public readonly endRule: string | null, + + /** + * The list of scopes containing the "name" for this state. + */ + public readonly nameScopesList: AttributedScopeStack | null, + + /** + * The list of scopes containing the "contentName" (besides "name") for this state. + * This list **must** contain as an element `scopeName`. + */ + public readonly contentNameScopesList: AttributedScopeStack | null, + ) { + this.depth = this.parent ? this.parent.depth + 1 : 1; + this._enterPos = enterPos; + this._anchorPos = anchorPos; + } + + public equals(other: StateStackImpl): boolean { + if (other === null) { + return false; + } + return StateStackImpl._equals(this, other); + } + + private static _equals(a: StateStackImpl, b: StateStackImpl): boolean { + if (a === b) { + return true; + } + if (!this._structuralEquals(a, b)) { + return false; + } + return AttributedScopeStack.equals(a.contentNameScopesList, b.contentNameScopesList); + } + + /** + * A structural equals check. Does not take into account `scopes`. + */ + private static _structuralEquals( + a: StateStackImpl | null, + b: StateStackImpl | null + ): boolean { + do { + if (a === b) { + return true; + } + + if (!a && !b) { + // End of list reached for both + return true; + } + + if (!a || !b) { + // End of list reached only for one + return false; + } + + if ( + a.depth !== b.depth || + a.ruleId !== b.ruleId || + a.endRule !== b.endRule + ) { + return false; + } + + // Go to previous pair + a = a.parent; + b = b.parent; + } while (true); + } + + public clone(): StateStackImpl { + return this; + } + + private static _reset(el: StateStackImpl | null): void { + while (el) { + el._enterPos = -1; + el._anchorPos = -1; + el = el.parent; + } + } + + public reset(): void { + StateStackImpl._reset(this); + } + + public pop(): StateStackImpl | null { + return this.parent; + } + + public safePop(): StateStackImpl { + if (this.parent) { + return this.parent; + } + return this; + } + + public push( + ruleId: RuleId, + enterPos: number, + anchorPos: number, + beginRuleCapturedEOL: boolean, + endRule: string | null, + nameScopesList: AttributedScopeStack | null, + contentNameScopesList: AttributedScopeStack | null, + ): StateStackImpl { + return new StateStackImpl( + this, + ruleId, + enterPos, + anchorPos, + beginRuleCapturedEOL, + endRule, + nameScopesList, + contentNameScopesList + ); + } + + public getEnterPos(): number { + return this._enterPos; + } + + public getAnchorPos(): number { + return this._anchorPos; + } + + public getRule(grammar: IRuleRegistry): Rule { + return grammar.getRule(this.ruleId); + } + + public toString(): string { + const r: string[] = []; + this._writeString(r, 0); + return "[" + r.join(",") + "]"; + } + + private _writeString(res: string[], outIndex: number): number { + if (this.parent) { + outIndex = this.parent._writeString(res, outIndex); + } + + res[ + outIndex++ + ] = `(${this.ruleId}, ${this.nameScopesList?.toString()}, ${this.contentNameScopesList?.toString()})`; + + return outIndex; + } + + public withContentNameScopesList( + contentNameScopeStack: AttributedScopeStack + ): StateStackImpl { + if (this.contentNameScopesList === contentNameScopeStack) { + return this; + } + return this.parent!.push( + this.ruleId, + this._enterPos, + this._anchorPos, + this.beginRuleCapturedEOL, + this.endRule, + this.nameScopesList, + contentNameScopeStack + ); + } + + public withEndRule(endRule: string): StateStackImpl { + if (this.endRule === endRule) { + return this; + } + return new StateStackImpl( + this.parent, + this.ruleId, + this._enterPos, + this._anchorPos, + this.beginRuleCapturedEOL, + endRule, + this.nameScopesList, + this.contentNameScopesList + ); + } + + // Used to warn of endless loops + public hasSameRuleAs(other: StateStackImpl): boolean { + let el: StateStackImpl | null = this; + while (el && el._enterPos === other._enterPos) { + if (el.ruleId === other.ruleId) { + return true; + } + el = el.parent; + } + return false; + } + + public toStateStackFrame(): StateStackFrame { + return { + ruleId: ruleIdToNumber(this.ruleId), + beginRuleCapturedEOL: this.beginRuleCapturedEOL, + endRule: this.endRule, + nameScopesList: this.nameScopesList?.getExtensionIfDefined(this.parent?.nameScopesList ?? null)! ?? [], + contentNameScopesList: this.contentNameScopesList?.getExtensionIfDefined(this.nameScopesList)! ?? [], + }; + } + + public static pushFrame(self: StateStackImpl | null, frame: StateStackFrame): StateStackImpl { + const namesScopeList = AttributedScopeStack.fromExtension(self?.nameScopesList ?? null, frame.nameScopesList)!; + return new StateStackImpl( + self, + ruleIdFromNumber(frame.ruleId), + frame.enterPos ?? -1, + frame.anchorPos ?? -1, + frame.beginRuleCapturedEOL, + frame.endRule, + namesScopeList, + AttributedScopeStack.fromExtension(namesScopeList, frame.contentNameScopesList)! + ); + } +} + +export interface StateStackFrame { + ruleId: number; + enterPos?: number; + anchorPos?: number; + beginRuleCapturedEOL: boolean; + endRule: string | null; + nameScopesList: AttributedScopeStackFrame[]; + /** + * on top of nameScopesList + */ + contentNameScopesList: AttributedScopeStackFrame[]; +} + +interface TokenTypeMatcher { + readonly matcher: Matcher; + readonly type: StandardTokenType; +} + +export class BalancedBracketSelectors { + private readonly balancedBracketScopes: Matcher[]; + private readonly unbalancedBracketScopes: Matcher[]; + + private allowAny = false; + + constructor( + balancedBracketScopes: string[], + unbalancedBracketScopes: string[], + ) { + this.balancedBracketScopes = balancedBracketScopes.flatMap((selector) => { + if (selector === '*') { + this.allowAny = true; + return []; + } + return createMatchers(selector, nameMatcher).map((m) => m.matcher); + } + ); + this.unbalancedBracketScopes = unbalancedBracketScopes.flatMap((selector) => + createMatchers(selector, nameMatcher).map((m) => m.matcher) + ); + } + + public get matchesAlways(): boolean { + return this.allowAny && this.unbalancedBracketScopes.length === 0; + } + + public get matchesNever(): boolean { + return this.balancedBracketScopes.length === 0 && !this.allowAny; + } + + public match(scopes: string[]): boolean { + for (const excluder of this.unbalancedBracketScopes) { + if (excluder(scopes)) { + return false; + } + } + + for (const includer of this.balancedBracketScopes) { + if (includer(scopes)) { + return true; + } + } + return this.allowAny; + } +} + +export class LineTokens { + private readonly _emitBinaryTokens: boolean; + /** + * defined only if `DebugFlags.InDebugMode`. + */ + private readonly _lineText: string | null; + /** + * used only if `_emitBinaryTokens` is false. + */ + private readonly _tokens: IToken[]; + /** + * used only if `_emitBinaryTokens` is true. + */ + private readonly _binaryTokens: number[]; + + private _lastTokenEndIndex: number; + + private readonly _tokenTypeOverrides: TokenTypeMatcher[]; + private readonly _mergeConsecutiveTokensWithEqualMetadata: boolean; + + constructor( + emitBinaryTokens: boolean, + lineText: string, + tokenTypeOverrides: TokenTypeMatcher[], + private readonly balancedBracketSelectors: BalancedBracketSelectors | null, + ) { + this._emitBinaryTokens = emitBinaryTokens; + this._tokenTypeOverrides = tokenTypeOverrides; + if (DebugFlags.InDebugMode) { + this._lineText = lineText; + } else { + this._lineText = null; + } + // Don't merge tokens if the line contains RTL characters + this._mergeConsecutiveTokensWithEqualMetadata = !containsRTL(lineText); + this._tokens = []; + this._binaryTokens = []; + this._lastTokenEndIndex = 0; + } + + public produce(stack: StateStackImpl, endIndex: number): void { + this.produceFromScopes(stack.contentNameScopesList, endIndex); + } + + public produceFromScopes( + scopesList: AttributedScopeStack | null, + endIndex: number + ): void { + if (this._lastTokenEndIndex >= endIndex) { + return; + } + + if (this._emitBinaryTokens) { + let metadata = scopesList?.tokenAttributes ?? 0; + let containsBalancedBrackets = false; + if (this.balancedBracketSelectors?.matchesAlways) { + containsBalancedBrackets = true; + } + + if (this._tokenTypeOverrides.length > 0 || (this.balancedBracketSelectors && !this.balancedBracketSelectors.matchesAlways && !this.balancedBracketSelectors.matchesNever)) { + // Only generate scope array when required to improve performance + const scopes = scopesList?.getScopeNames() ?? []; + for (const tokenType of this._tokenTypeOverrides) { + if (tokenType.matcher(scopes)) { + metadata = EncodedTokenAttributes.set( + metadata, + 0, + toOptionalTokenType(tokenType.type), + null, + FontStyle.NotSet, + 0, + 0 + ); + } + } + if (this.balancedBracketSelectors) { + containsBalancedBrackets = this.balancedBracketSelectors.match(scopes); + } + } + + if (containsBalancedBrackets) { + metadata = EncodedTokenAttributes.set( + metadata, + 0, + OptionalStandardTokenType.NotSet, + containsBalancedBrackets, + FontStyle.NotSet, + 0, + 0 + ); + } + + if (this._mergeConsecutiveTokensWithEqualMetadata && this._binaryTokens.length > 0 && this._binaryTokens[this._binaryTokens.length - 1] === metadata) { + // no need to push a token with the same metadata + this._lastTokenEndIndex = endIndex; + return; + } + + if (DebugFlags.InDebugMode) { + const scopes = scopesList?.getScopeNames() ?? []; + console.log(' token: |' + this._lineText!.substring(this._lastTokenEndIndex, endIndex).replace(/\n$/, '\\n') + '|'); + for (let k = 0; k < scopes.length; k++) { + console.log(' * ' + scopes[k]); + } + } + + this._binaryTokens.push(this._lastTokenEndIndex); + this._binaryTokens.push(metadata); + + this._lastTokenEndIndex = endIndex; + return; + } + + const scopes = scopesList?.getScopeNames() ?? []; + + if (DebugFlags.InDebugMode) { + console.log(' token: |' + this._lineText!.substring(this._lastTokenEndIndex, endIndex).replace(/\n$/, '\\n') + '|'); + for (let k = 0; k < scopes.length; k++) { + console.log(' * ' + scopes[k]); + } + } + + this._tokens.push({ + startIndex: this._lastTokenEndIndex, + endIndex: endIndex, + // value: lineText.substring(lastTokenEndIndex, endIndex), + scopes: scopes + }); + + this._lastTokenEndIndex = endIndex; + } + + public getResult(stack: StateStackImpl, lineLength: number): IToken[] { + if (this._tokens.length > 0 && this._tokens[this._tokens.length - 1].startIndex === lineLength - 1) { + // pop produced token for newline + this._tokens.pop(); + } + + if (this._tokens.length === 0) { + this._lastTokenEndIndex = -1; + this.produce(stack, lineLength); + this._tokens[this._tokens.length - 1].startIndex = 0; + } + + return this._tokens; + } + + public getBinaryResult(stack: StateStackImpl, lineLength: number): Uint32Array { + if (this._binaryTokens.length > 0 && this._binaryTokens[this._binaryTokens.length - 2] === lineLength - 1) { + // pop produced token for newline + this._binaryTokens.pop(); + this._binaryTokens.pop(); + } + + if (this._binaryTokens.length === 0) { + this._lastTokenEndIndex = -1; + this.produce(stack, lineLength); + this._binaryTokens[this._binaryTokens.length - 2] = 0; + } + + const result = new Uint32Array(this._binaryTokens.length); + for (let i = 0, len = this._binaryTokens.length; i < len; i++) { + result[i] = this._binaryTokens[i]; + } + + return result; + } +} + +export class FontInfo implements IFontInfo { + + constructor( + public startIndex: number, + public endIndex: number, + public fontFamily: string | null, + public fontSizeMultiplier: number | null, + public lineHeightMultiplier: number | null + ) { } + + optionsEqual(other: IFontInfo): boolean { + return this.fontFamily === other.fontFamily + && this.fontSizeMultiplier === other.fontSizeMultiplier + && this.lineHeightMultiplier === other.lineHeightMultiplier; + } +} + +export let TOTAL_TIME = 0; +export let TOTAL_COUNT = 0; +export let TOTAL_CALLS = 0; + +export let PRODUCE_FROM_SCOPES_TIME = 0; +export let PRODUCE_FROM_SCOPES_COUNT = 0; + +export function resetVariables() { + TOTAL_TIME = 0; + TOTAL_COUNT = 0; + TOTAL_CALLS = 0; +} + +export class LineFonts { + + private readonly _fonts: FontInfo[] = []; + + private _lastIndex: number = 0; + + constructor() { } + + public produce(stack: StateStackImpl, endIndex: number): void { + this.produceFromScopes(stack.contentNameScopesList, endIndex); + } + + public produceFromScopes( + scopesList: AttributedScopeStack | null, + endIndex: number + ): void { + PRODUCE_FROM_SCOPES_COUNT++; + const start = new Date().getTime(); + const fontFamily = this.getFontFamily(scopesList); + const fontSizeMultiplier = this.getFontSize(scopesList); + const lineHeightMultiplier = this.getLineHeight(scopesList); + if (!fontFamily && !fontSizeMultiplier && !lineHeightMultiplier) { + this._lastIndex = endIndex; + return; + } + const font = new FontInfo( + this._lastIndex, + endIndex, + fontFamily, + fontSizeMultiplier, + lineHeightMultiplier + ); + const lastFont = this._fonts[this._fonts.length - 1] + if (lastFont && lastFont.endIndex === this._lastIndex && lastFont.optionsEqual(font)) { + lastFont.endIndex = font.endIndex; + } else { + this._fonts.push(font); + } + this._lastIndex = endIndex; + const end = new Date().getTime(); + PRODUCE_FROM_SCOPES_TIME += (end - start); + } + + public getResult(): IFontInfo[] { + return this._fonts; + } + + private getFontFamily(scopesList: AttributedScopeStack | null): string | null { + TOTAL_COUNT++; + const start = new Date().getTime(); + const attr = this.getAttribute(scopesList, (styleAttributes) => { return styleAttributes.fontFamily; }); + const end = new Date().getTime(); + TOTAL_TIME += (end - start); + return attr; + } + + private getFontSize(scopesList: AttributedScopeStack | null): number | null { + TOTAL_COUNT++; + const start = new Date().getTime(); + const attr = this.getAttribute(scopesList, (styleAttributes) => { return styleAttributes.fontSize; }); + const end = new Date().getTime(); + TOTAL_TIME += (end - start); + return attr; + } + + private getLineHeight(scopesList: AttributedScopeStack | null): number | null { + TOTAL_COUNT++; + const start = new Date().getTime(); + const attr = this.getAttribute(scopesList, (styleAttributes) => { return styleAttributes.lineHeight; }); + const end = new Date().getTime(); + TOTAL_TIME += (end - start); + return attr; + } + + private getAttribute(scopesList: AttributedScopeStack | null, getAttr: (styleAttributes: StyleAttributes) => any | null): any | null { + TOTAL_CALLS++; + if (!scopesList) { + return null; + } + const styleAttributes = scopesList.styleAttributes; + if (!styleAttributes) { + return null; + } + const attribute = getAttr(styleAttributes); + if (attribute) { + return attribute; + } + return this.getAttribute(scopesList.parent, getAttr); + } +}