Support comments in closed files

Author: msujew

packages/langium/src/documentation/comment-provider.ts

@@ -28,9 +28,12 @@ export class DefaultCommentProvider implements CommentProvider {
         this.grammarConfig = () => services.parser.GrammarConfig;
     }
     getComment(node: AstNode): string | undefined {
-        if(isAstNodeWithComment(node)) {
+        if (isAstNodeWithComment(node)) {
             return node.$comment;
+        } else if (node.$segments && 'comment' in node.$segments) {
+            return node.$segments.comment;
+        } else {
+            return findCommentNode(node.$cstNode, this.grammarConfig().multilineCommentRules)?.text;
         }
-        return findCommentNode(node.$cstNode, this.grammarConfig().multilineCommentRules)?.text;
     }
 }

packages/langium/src/parser/cst-node-builder.ts

@@ -37,8 +37,8 @@ export class CstNodeBuilder {
         return compositeNode;
     }
 
-    buildLeafNode(token: IToken, feature: AbstractElement): LeafCstNode {
-        const leafNode = new LeafCstNodeImpl(token.startOffset, token.image.length, tokenToRange(token), token.tokenType, false);
+    buildLeafNode(token: IToken, feature?: AbstractElement): LeafCstNode {
+        const leafNode = new LeafCstNodeImpl(token.startOffset, token.image.length, tokenToRange(token), token.tokenType, !feature);
         leafNode.grammarSource = feature;
         leafNode.root = this.rootNode;
         this.current.content.push(leafNode);
@@ -107,7 +107,7 @@ export abstract class AbstractCstNode implements CstNode {
     abstract get range(): Range;
 
     container?: CompositeCstNode;
-    grammarSource: AbstractElement;
+    grammarSource?: AbstractElement;
     root: RootCstNode;
     private _astNode?: AstNode;
 
@@ -117,7 +117,7 @@ export abstract class AbstractCstNode implements CstNode {
     }
 
     /** @deprecated use `grammarSource` instead. */
-    get feature(): AbstractElement {
+    get feature(): AbstractElement | undefined {
         return this.grammarSource;
     }
 

packages/langium/src/parser/langium-parser.ts

@@ -10,7 +10,7 @@ import type { AbstractElement, Action, Assignment, ParserRule } from '../languag
 import type { Linker } from '../references/linker.js';
 import type { LangiumCoreServices } from '../services.js';
 import type { AstNode, AstReflection, CompositeCstNode, CstNode } from '../syntax-tree.js';
-import type { Lexer } from './lexer.js';
+import type { Lexer, LexerResult } from './lexer.js';
 import type { IParserConfig } from './parser-config.js';
 import type { ValueConverter } from './value-converter.js';
 import { defaultParserErrorProvider, EmbeddedActionsParser, LLkLookaheadStrategy } from 'chevrotain';
@@ -21,6 +21,7 @@ import { assignMandatoryProperties, getContainerOfType, linkContentToContainer }
 import { CstNodeBuilder } from './cst-node-builder.js';
 import type { LexingReport } from './token-builder.js';
 import { toDocumentSegment } from '../utils/cst-utils.js';
+import type { CommentProvider } from '../documentation/comment-provider.js';
 
 export type ParseResult<T = AstNode> = {
     value: T,
@@ -123,6 +124,7 @@ const withRuleSuffix = (name: string): string => name.endsWith(ruleSuffix) ? nam
 export abstract class AbstractLangiumParser implements BaseParser {
 
     protected readonly lexer: Lexer;
+    protected readonly commentProvider: CommentProvider;
     protected readonly wrapper: ChevrotainWrapper;
     protected _unorderedGroups: Map<string, boolean[]> = new Map<string, boolean[]>();
 
@@ -138,6 +140,7 @@ export abstract class AbstractLangiumParser implements BaseParser {
             skipValidations: production,
             errorMessageProvider: services.parser.ParserErrorMessageProvider
         });
+        this.commentProvider = services.documentation.CommentProvider;
     }
 
     alternatives(idx: number, choices: Array<IOrAlt<any>>): void {
@@ -197,6 +200,7 @@ export class LangiumParser extends AbstractLangiumParser {
     private readonly converter: ValueConverter;
     private readonly astReflection: AstReflection;
     private readonly nodeBuilder = new CstNodeBuilder();
+    private lexerResult?: LexerResult;
     private stack: any[] = [];
     private assignmentMap = new Map<AbstractElement, AssignmentElement | undefined>();
     private currentMode: CstParserMode = CstParserMode.Retain;
@@ -236,17 +240,15 @@ export class LangiumParser extends AbstractLangiumParser {
     parse<T extends AstNode = AstNode>(input: string, options: ParserOptions = {}): ParseResult<T> {
         this.currentMode = options.cst ?? CstParserMode.Retain;
         this.nodeBuilder.buildRootNode(input);
-        const lexerResult = this.lexer.tokenize(input);
+        const lexerResult = this.lexerResult = this.lexer.tokenize(input);
         this.wrapper.input = lexerResult.tokens;
         const ruleMethod = options.rule ? this.allRules.get(options.rule) : this.mainRule;
         if (!ruleMethod) {
             throw new Error(options.rule ? `No rule found with name '${options.rule}'` : 'No main rule available.');
         }
         const result = ruleMethod.call(this.wrapper, {});
-        if (this.currentMode === CstParserMode.Retain) {
-            this.nodeBuilder.addHiddenTokens(lexerResult.hidden);
-        }
         this.unorderedGroups.clear();
+        this.lexerResult = undefined;
         return {
             value: result,
             lexerErrors: lexerResult.errors,
@@ -277,9 +279,32 @@ export class LangiumParser extends AbstractLangiumParser {
         };
     }
 
+    private appendHiddenTokens(tokens: IToken[]): void {
+        for (const token of tokens) {
+            this.nodeBuilder.buildLeafNode(token);
+        }
+    }
+
+    private getHiddenTokens(token: IToken): IToken[] {
+        const hiddenTokens = this.lexerResult!.hidden;
+        if (!hiddenTokens.length) {
+            return [];
+        }
+        const offset = token.startOffset;
+        for (let i = 0; i < hiddenTokens.length; i++) {
+            const token = hiddenTokens[i];
+            if (token.startOffset > offset) {
+                return hiddenTokens.splice(0, i);
+            }
+        }
+        return hiddenTokens.splice(0, hiddenTokens.length);
+    }
+
     consume(idx: number, tokenType: TokenType, feature: AbstractElement): void {
         const token = this.wrapper.wrapConsume(idx, tokenType);
         if (!this.isRecording() && this.isValidToken(token)) {
+            const hiddenTokens = this.getHiddenTokens(token);
+            this.appendHiddenTokens(hiddenTokens);
             const leafNode = this.nodeBuilder.buildLeafNode(token, feature);
             const { assignment, isCrossRef } = this.getAssignment(feature);
             const current = this.current;
@@ -374,9 +399,12 @@ export class LangiumParser extends AbstractLangiumParser {
         } else {
             assignMandatoryProperties(this.astReflection, obj);
         }
+        obj.$cstNode = cstNode;
+        delete obj.$segments.comment;
+        obj.$segments.comment = this.commentProvider.getComment(obj);
         obj.$segments.full = toDocumentSegment(cstNode);
-        if (this.currentMode === CstParserMode.Retain) {
-            obj.$cstNode = cstNode;
+        if (this.currentMode === CstParserMode.Discard) {
+            obj.$cstNode = undefined;
         }
         return [obj, cstNode];
     }

packages/langium/src/serializer/hydrator.ts

@@ -296,10 +296,13 @@ export class DefaultHydrator implements Hydrator {
         return this.lexer.definition[name];
     }
 
-    protected getGrammarElementId(node: AbstractElement): number | undefined {
+    protected getGrammarElementId(node: AbstractElement | undefined): number | undefined {
         if (this.grammarElementIdMap.size === 0) {
             this.createGrammarElementIdMap();
         }
+        if (!node) {
+            return undefined;
+        }
         return this.grammarElementIdMap.get(node);
     }
 

packages/langium/src/syntax-tree.ts

@@ -45,6 +45,7 @@ export type Properties<N extends AstNode> = SpecificNodeProperties<N> extends ne
 
 export interface AstNodeSegments {
     readonly full: DocumentSegment;
+    readonly comment?: string;
     readonly properties: Record<string, DocumentSegment[]>;
 }
 
@@ -240,9 +241,9 @@ export interface CstNode extends DocumentSegment {
     /** The root CST node */
     readonly root: RootCstNode;
     /** The grammar element from which this node was parsed */
-    readonly grammarSource: AbstractElement;
+    readonly grammarSource?: AbstractElement;
     /** @deprecated use `grammarSource` instead. */
-    readonly feature: AbstractElement;
+    readonly feature?: AbstractElement;
     /** The AST node created from this CST node */
     readonly astNode: AstNode;
     /** @deprecated use `astNode` instead. */

packages/langium/src/utils/cst-utils.ts

@@ -9,7 +9,7 @@ import type { Range } from 'vscode-languageserver-types';
 import type { CstNode, CompositeCstNode, LeafCstNode, AstNode, Mutable, Reference } from '../syntax-tree.js';
 import type { DocumentSegment } from '../workspace/documents.js';
 import type { Stream, TreeStream } from './stream.js';
-import { isCompositeCstNode, isLeafCstNode, isRootCstNode } from '../syntax-tree.js';
+import { isCompositeCstNode, isLeafCstNode } from '../syntax-tree.js';
 import { TreeStreamImpl } from './stream.js';
 import { streamAst, streamReferences } from './ast-utils.js';
 
@@ -135,20 +135,24 @@ export function findDeclarationNodeAtOffset(cstNode: CstNode | undefined, offset
 }
 
 export function findCommentNode(cstNode: CstNode | undefined, commentNames: string[]): CstNode | undefined {
-    if (cstNode) {
-        const previous = getPreviousNode(cstNode, true);
-        if (previous && isCommentNode(previous, commentNames)) {
-            return previous;
+    if (isCompositeCstNode(cstNode)) {
+        const firstChild = cstNode.content[0];
+        if (isCompositeCstNode(firstChild)) {
+            return findCommentNode(firstChild, commentNames);
+        }
+        // Find the first non-hidden child
+        let index = -1;
+        for (const child of cstNode.content) {
+            if (!child.hidden) {
+                break;
+            }
+            index++;
         }
-        if (isRootCstNode(cstNode)) {
-            // Go from the first non-hidden node through all nodes in reverse order
-            // We do this to find the comment node which directly precedes the root node
-            const endIndex = cstNode.content.findIndex(e => !e.hidden);
-            for (let i = endIndex - 1; i >= 0; i--) {
-                const child = cstNode.content[i];
-                if (isCommentNode(child, commentNames)) {
-                    return child;
-                }
+        // Search backwards for a comment node
+        for (let i = index; i >= 0; i--) {
+            const child = cstNode.content[i];
+            if (isCommentNode(child, commentNames)) {
+                return child;
             }
         }
     }