Support terminal lookbehind (#1356)

Author: msujew

packages/langium/src/grammar/generated/grammar.ts

@@ -3430,6 +3430,14 @@ export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar
                 {
                   "$type": "Keyword",
                   "value": "?!"
+                },
+                {
+                  "$type": "Keyword",
+                  "value": "?<="
+                },
+                {
+                  "$type": "Keyword",
+                  "value": "?<!"
                 }
               ]
             },

packages/langium/src/grammar/langium-grammar.langium

@@ -198,7 +198,7 @@ TerminalTokenElement infers AbstractElement:
     CharacterRange | TerminalRuleCall | ParenthesizedTerminalElement | NegatedToken | UntilToken | RegexToken | Wildcard;
 
 ParenthesizedTerminalElement infers AbstractElement:
-    '(' (lookahead=('?='|'?!'))? TerminalAlternatives ')';
+    '(' (lookahead=('?='|'?!'|'?<='|'?<!'))? TerminalAlternatives ')';
 
 TerminalRuleCall infers AbstractElement:
     {infer TerminalRuleCall} rule=[TerminalRule:ID];

packages/langium/src/languages/generated/ast.ts

@@ -72,7 +72,7 @@ export function isValueLiteral(item: unknown): item is ValueLiteral {
 export interface AbstractElement extends AstNode {
     readonly $type: 'AbstractElement' | 'Action' | 'Alternatives' | 'Assignment' | 'CharacterRange' | 'CrossReference' | 'EndOfFile' | 'Group' | 'Keyword' | 'NegatedToken' | 'RegexToken' | 'RuleCall' | 'TerminalAlternatives' | 'TerminalGroup' | 'TerminalRuleCall' | 'UnorderedGroup' | 'UntilToken' | 'Wildcard';
     cardinality?: '*' | '+' | '?';
-    lookahead?: '?!' | '?=';
+    lookahead?: '?!' | '?<!' | '?<=' | '?=';
 }
 
 export const AbstractElement = 'AbstractElement';

packages/langium/src/parser/token-builder.ts

@@ -49,7 +49,7 @@ export class DefaultTokenBuilder implements TokenBuilder {
 
     protected buildTerminalToken(terminal: TerminalRule): TokenType {
         const regex = terminalRegex(terminal);
-        const pattern = regex.flags.includes('u') ? this.regexPatternFunction(regex) : regex;
+        const pattern = this.requiresCustomPattern(regex) ? this.regexPatternFunction(regex) : regex;
         const tokenType: TokenType = {
             name: terminal.name,
             PATTERN: pattern,
@@ -62,6 +62,18 @@ export class DefaultTokenBuilder implements TokenBuilder {
         return tokenType;
     }
 
+    protected requiresCustomPattern(regex: RegExp): boolean {
+        if (regex.flags.includes('u')) {
+            // Unicode regexes are not supported by Chevrotain.
+            return true;
+        } else if (regex.source.includes('?<=') || regex.source.includes('?<!')) {
+            // Negative and positive lookbehind are not supported by Chevrotain yet.
+            return true;
+        } else {
+            return false;
+        }
+    }
+
     protected regexPatternFunction(regex: RegExp): CustomPatternMatcherFunc {
         const stickyRegex = new RegExp(regex, regex.flags + 'y');
         return (text, offset) => {

packages/langium/test/grammar/grammar-util.test.ts

@@ -170,6 +170,18 @@ describe('TerminalRule to regex', () => {
         expect(regex).toEqual(/(a(?!b))/);
     });
 
+    test('Should create negative lookbehind group', async () => {
+        const terminal = await getTerminal("terminal X: 'a' (?<!'b');");
+        const regex = terminalRegex(terminal);
+        expect(regex).toEqual(/(a(?<!b))/);
+    });
+
+    test('Should create positive lookbehind group', async () => {
+        const terminal = await getTerminal("terminal X: 'a' (?<='b');");
+        const regex = terminalRegex(terminal);
+        expect(regex).toEqual(/(a(?<=b))/);
+    });
+
     test('Should create terminal reference in terminal definition', async () => {
         const terminal = await getTerminal(`
         terminal X: Y Y;

packages/langium/test/parser/langium-parser-builder.test.ts

@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 import type { TokenType, TokenVocabulary } from 'chevrotain';
-import type { AstNode, CstNode, GenericAstNode, Grammar, GrammarAST, LangiumParser, TokenBuilderOptions } from 'langium';
+import type { AstNode, CstNode, GenericAstNode, Grammar, GrammarAST, LangiumParser, ParseResult, TokenBuilderOptions } from 'langium';
 import { EmptyFileSystem, DefaultTokenBuilder } from 'langium';
 import { describe, expect, test, onTestFailed, beforeEach } from 'vitest';
 import { createLangiumGrammarServices, createServicesForGrammar } from 'langium/grammar';
@@ -852,7 +852,31 @@ describe('Unassigned data type rules', () => {
         expect(parseResult.lexerErrors).toHaveLength(0);
         expect(parseResult.parserErrors).toHaveLength(0);
     });
+});
+
+describe('Parsing with lookbehind tokens', () => {
+    test('Parser Success / Failure with positive lookbehind', async () => {
+        await testLookbehind(true, 'AB', 'CB');
+    });
+
+    test('Parser Success / Failure with negative lookbehind', async () => {
+        await testLookbehind(false, 'CB', 'AB');
+    });
 
+    async function testLookbehind(positive: boolean, success: string, failure: string): Promise<void> {
+        const parser = await parserFromGrammar(`
+            grammar test
+            entry Main: ('A' | 'C') b=B;
+            terminal B: (?<${positive ? '=' : '!'}'A')'B';
+            hidden terminal WS: /\\s+/;
+            `
+        );
+        const validResult = parser.parse(success) as ParseResult<GenericAstNode>;
+        expect(validResult.value.b).toEqual('B');
+        const invalidResult = parser.parse(failure);
+        expect(invalidResult.lexerErrors).toHaveLength(1);
+        expect(invalidResult.parserErrors).toHaveLength(1);
+    }
 });
 
 async function parserFromGrammar(grammar: string): Promise<LangiumParser> {