Copy JSDoc comments to generated AST (#1835)

Author: aabounegm

examples/statemachine/src/language-server/generated/ast.ts

@@ -41,6 +41,7 @@ export function isCommand(item: unknown): item is Command {
     return reflection.isInstance(item, Command);
 }
 
+/** An event is the trigger for a transition */
 export interface Event extends langium.AstNode {
     readonly $container: Statemachine;
     readonly $type: 'Event';
@@ -53,11 +54,13 @@ export function isEvent(item: unknown): item is Event {
     return reflection.isInstance(item, Event);
 }
 
+/** A description of the status of a system */
 export interface State extends langium.AstNode {
     readonly $container: Statemachine;
     readonly $type: 'State';
     actions: Array<langium.Reference<Command>>;
     name: string;
+    /** The transitions to other states that can take place from the current one */
     transitions: Array<Transition>;
 }
 
@@ -67,12 +70,17 @@ export function isState(item: unknown): item is State {
     return reflection.isInstance(item, State);
 }
 
+/** A textual represntation of a state machine */
 export interface Statemachine extends langium.AstNode {
     readonly $type: 'Statemachine';
     commands: Array<Command>;
+    /** The list of recognized event names */
     events: Array<Event>;
+    /** The starting state for the machine */
     init: langium.Reference<State>;
+    /** The name of the machine */
     name: string;
+    /** Definitions of available states */
     states: Array<State>;
 }
 
@@ -82,10 +90,13 @@ export function isStatemachine(item: unknown): item is Statemachine {
     return reflection.isInstance(item, Statemachine);
 }
 
+/** A change from one state to another */
 export interface Transition extends langium.AstNode {
     readonly $container: State;
     readonly $type: 'Transition';
+    /** The event triggering the transition */
     event: langium.Reference<Event>;
+    /** The target state */
     state: langium.Reference<State>;
 }
 

examples/statemachine/src/language-server/generated/grammar.ts

@@ -33,7 +33,8 @@ export const StatemachineGrammar = (): Grammar => loadedStatemachineGrammar ?? (
                 "$ref": "#/rules@6"
               },
               "arguments": []
-            }
+            },
+            "$comment": "/** The name of the machine */"
           },
           {
             "$type": "Group",
@@ -53,7 +54,8 @@ export const StatemachineGrammar = (): Grammar => loadedStatemachineGrammar ?? (
                   },
                   "arguments": []
                 },
-                "cardinality": "+"
+                "cardinality": "+",
+                "$comment": "/** The list of recognized event names */"
               }
             ],
             "cardinality": "?"
@@ -95,7 +97,8 @@ export const StatemachineGrammar = (): Grammar => loadedStatemachineGrammar ?? (
                 "$ref": "#/rules@3"
               },
               "deprecatedSyntax": false
-            }
+            },
+            "$comment": "/** The starting state for the machine */"
           },
           {
             "$type": "Assignment",
@@ -108,15 +111,17 @@ export const StatemachineGrammar = (): Grammar => loadedStatemachineGrammar ?? (
               },
               "arguments": []
             },
-            "cardinality": "*"
+            "cardinality": "*",
+            "$comment": "/** Definitions of available states */"
           }
         ]
       },
       "definesHiddenTokens": false,
       "fragment": false,
       "hiddenTokens": [],
       "parameters": [],
-      "wildcard": false
+      "wildcard": false,
+      "$comment": "/** A textual represntation of a state machine */"
     },
     {
       "$type": "ParserRule",
@@ -138,7 +143,8 @@ export const StatemachineGrammar = (): Grammar => loadedStatemachineGrammar ?? (
       "fragment": false,
       "hiddenTokens": [],
       "parameters": [],
-      "wildcard": false
+      "wildcard": false,
+      "$comment": "/** An event is the trigger for a transition */"
     },
     {
       "$type": "ParserRule",
@@ -226,7 +232,8 @@ export const StatemachineGrammar = (): Grammar => loadedStatemachineGrammar ?? (
               },
               "arguments": []
             },
-            "cardinality": "*"
+            "cardinality": "*",
+            "$comment": "/** The transitions to other states that can take place from the current one */"
           },
           {
             "$type": "Keyword",
@@ -239,7 +246,8 @@ export const StatemachineGrammar = (): Grammar => loadedStatemachineGrammar ?? (
       "fragment": false,
       "hiddenTokens": [],
       "parameters": [],
-      "wildcard": false
+      "wildcard": false,
+      "$comment": "/** A description of the status of a system */"
     },
     {
       "$type": "ParserRule",
@@ -257,7 +265,8 @@ export const StatemachineGrammar = (): Grammar => loadedStatemachineGrammar ?? (
                 "$ref": "#/rules@1"
               },
               "deprecatedSyntax": false
-            }
+            },
+            "$comment": "/** The event triggering the transition */"
           },
           {
             "$type": "Keyword",
@@ -273,16 +282,19 @@ export const StatemachineGrammar = (): Grammar => loadedStatemachineGrammar ?? (
                 "$ref": "#/rules@3"
               },
               "deprecatedSyntax": false
-            }
+            },
+            "$comment": "/** The target state */"
           }
-        ]
+        ],
+        "$comment": "/** The event triggering the transition */"
       },
       "definesHiddenTokens": false,
       "entry": false,
       "fragment": false,
       "hiddenTokens": [],
       "parameters": [],
-      "wildcard": false
+      "wildcard": false,
+      "$comment": "/** A change from one state to another */"
     },
     {
       "$type": "TerminalRule",

examples/statemachine/src/language-server/statemachine.langium

@@ -1,26 +1,41 @@
 grammar Statemachine
 
+/** A textual represntation of a state machine */
 entry Statemachine:
-    'statemachine' name=ID
-    ('events' events+=Event+)?
+    'statemachine'
+    /** The name of the machine */
+    name=ID
+    ('events'
+        /** The list of recognized event names */
+        events+=Event+)?
     ('commands'    commands+=Command+)?
-    'initialState' init=[State]
+    'initialState'
+    /** The starting state for the machine */
+    init=[State]
+    /** Definitions of available states */
     states+=State*;
 
+/** An event is the trigger for a transition */
 Event:
     name=ID;
 
 Command:
     name=ID;
 
+/** A description of the status of a system */
 State:
     'state' name=ID
         ('actions' '{' actions+=[Command]+ '}')?
+        /** The transitions to other states that can take place from the current one */
         transitions+=Transition*
     'end';
 
+/** A change from one state to another */
 Transition:
-    event=[Event] '=>' state=[State];
+    /** The event triggering the transition */
+    event=[Event] '=>'
+    /** The target state */
+    state=[State];
 
 hidden terminal WS: /\s+/;
 terminal ID: /[_a-zA-Z][\w_]*/;

packages/langium-cli/src/generator/ast-generator.ts

@@ -13,7 +13,7 @@ import { generatedHeader } from './node-util.js';
 import { collectKeywords, collectTerminalRegexps } from './langium-util.js';
 
 export function generateAst(services: LangiumCoreServices, grammars: Grammar[], config: LangiumConfig): string {
-    const astTypes = collectAst(grammars, services.shared.workspace.LangiumDocuments);
+    const astTypes = collectAst(grammars, services);
     const importFrom = config.langiumInternal ? `../../syntax-tree${config.importExtension}` : 'langium';
     /* eslint-disable @typescript-eslint/indent */
     const fileNode = expandToNode`

packages/langium-cli/src/generator/types-generator.ts

@@ -9,7 +9,7 @@ import { collectAst, LangiumGrammarGrammar } from 'langium/grammar';
 import { collectKeywords } from './langium-util.js';
 
 export function generateTypesFile(services: LangiumCoreServices, grammars: Grammar[]): string {
-    const { unions, interfaces } = collectAst(grammars, services.shared.workspace.LangiumDocuments);
+    const { unions, interfaces } = collectAst(grammars, services);
     const reservedWords = new Set(collectKeywords(LangiumGrammarGrammar()));
 
     const fileNode = joinToNode([

packages/langium/src/grammar/ast-reflection-interpreter.ts

@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 import type { AstReflection, ReferenceInfo, TypeProperty, TypeMetaData } from '../syntax-tree.js';
-import type { LangiumDocuments } from '../workspace/documents.js';
+import type { LangiumCoreServices } from '../index.js';
 import type { Grammar } from '../languages/generated/ast.js';
 import type { AstTypes, Property } from './type-system/type-collector/types.js';
 import { AbstractAstReflection } from '../syntax-tree.js';
@@ -15,11 +15,11 @@ import { collectAst } from './type-system/ast-collector.js';
 import { collectTypeHierarchy, findReferenceTypes, isAstType, mergeTypesAndInterfaces } from './type-system/types-util.js';
 
 export function interpretAstReflection(astTypes: AstTypes): AstReflection;
-export function interpretAstReflection(grammar: Grammar, documents?: LangiumDocuments): AstReflection;
-export function interpretAstReflection(grammarOrTypes: Grammar | AstTypes, documents?: LangiumDocuments): AstReflection {
+export function interpretAstReflection(grammar: Grammar, services?: LangiumCoreServices): AstReflection;
+export function interpretAstReflection(grammarOrTypes: Grammar | AstTypes, services?: LangiumCoreServices): AstReflection {
     let collectedTypes: AstTypes;
     if (isGrammar(grammarOrTypes)) {
-        collectedTypes = collectAst(grammarOrTypes, documents);
+        collectedTypes = collectAst(grammarOrTypes, services);
     } else {
         collectedTypes = grammarOrTypes;
     }

packages/langium/src/grammar/type-system/ast-collector.ts

@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 import type { Grammar } from '../../languages/generated/ast.js';
-import type { LangiumDocuments } from '../../workspace/documents.js';
+import type { LangiumCoreServices } from '../../index.js';
 import type { AstTypes, InterfaceType, PropertyType, TypeOption, UnionType } from './type-collector/types.js';
 import type { ValidationAstTypes } from './type-collector/all-types.js';
 import type { PlainAstTypes, PlainInterface, PlainUnion } from './type-collector/plain-types.js';
@@ -18,10 +18,10 @@ import { plainToTypes } from './type-collector/plain-types.js';
  * Collects all types for the generated AST. The types collector entry point.
  *
  * @param grammars All grammars involved in the type generation process
- * @param documents Additional documents so that imports can be resolved as necessary
+ * @param services Langium core services to resolve imports as needed, and to pass along JSDoc comments to the generated AST
  */
-export function collectAst(grammars: Grammar | Grammar[], documents?: LangiumDocuments): AstTypes {
-    const { inferred, declared } = collectTypeResources(grammars, documents);
+export function collectAst(grammars: Grammar | Grammar[], services?: LangiumCoreServices): AstTypes {
+    const { inferred, declared } = collectTypeResources(grammars, services);
     return createAstTypes(inferred, declared);
 }
 
@@ -30,10 +30,10 @@ export function collectAst(grammars: Grammar | Grammar[], documents?: LangiumDoc
  * The validation process requires us to compare our inferred types with our declared types.
  *
  * @param grammars All grammars involved in the validation process
- * @param documents Additional documents so that imports can be resolved as necessary
+ * @param services Langium core services to resolve imports as needed, and to pass along JSDoc comments to the generated AST
  */
-export function collectValidationAst(grammars: Grammar | Grammar[], documents?: LangiumDocuments): ValidationAstTypes {
-    const { inferred, declared, astResources } = collectTypeResources(grammars, documents);
+export function collectValidationAst(grammars: Grammar | Grammar[], services?: LangiumCoreServices): ValidationAstTypes {
+    const { inferred, declared, astResources } = collectTypeResources(grammars, services);
 
     return {
         astResources,

packages/langium/src/grammar/type-system/type-collector/all-types.ts

@@ -6,7 +6,7 @@
 
 import type { ParserRule, Interface, Type, Grammar } from '../../../languages/generated/ast.js';
 import type { URI } from '../../../utils/uri-utils.js';
-import type { LangiumDocuments } from '../../../workspace/documents.js';
+import type { LangiumCoreServices } from '../../../index.js';
 import type { PlainAstTypes } from './plain-types.js';
 import type { AstTypes } from './types.js';
 import { collectInferredTypes } from './inferred-types.js';
@@ -35,10 +35,10 @@ export interface ValidationAstTypes {
     astResources: AstResources
 }
 
-export function collectTypeResources(grammars: Grammar | Grammar[], documents?: LangiumDocuments): TypeResources {
-    const astResources = collectAllAstResources(grammars, documents);
-    const declared = collectDeclaredTypes(astResources.interfaces, astResources.types);
-    const inferred = collectInferredTypes(astResources.parserRules, astResources.datatypeRules, declared);
+export function collectTypeResources(grammars: Grammar | Grammar[], services?: LangiumCoreServices): TypeResources {
+    const astResources = collectAllAstResources(grammars, undefined, undefined, services);
+    const declared = collectDeclaredTypes(astResources.interfaces, astResources.types, services);
+    const inferred = collectInferredTypes(astResources.parserRules, astResources.datatypeRules, declared, services);
 
     return {
         astResources,
@@ -49,8 +49,8 @@ export function collectTypeResources(grammars: Grammar | Grammar[], documents?:
 
 ///////////////////////////////////////////////////////////////////////////////
 
-export function collectAllAstResources(grammars: Grammar | Grammar[], documents?: LangiumDocuments, visited: Set<URI> = new Set(),
-    astResources: AstResources = { parserRules: [], datatypeRules: [], interfaces: [], types: [] }): AstResources {
+export function collectAllAstResources(grammars: Grammar | Grammar[], visited: Set<URI> = new Set(),
+    astResources: AstResources = { parserRules: [], datatypeRules: [], interfaces: [], types: [] }, services?: LangiumCoreServices): AstResources {
 
     if (!Array.isArray(grammars)) grammars = [grammars];
     for (const grammar of grammars) {
@@ -71,9 +71,10 @@ export function collectAllAstResources(grammars: Grammar | Grammar[], documents?
         grammar.interfaces.forEach(e => astResources.interfaces.push(e));
         grammar.types.forEach(e => astResources.types.push(e));
 
+        const documents = services?.shared.workspace.LangiumDocuments;
         if (documents) {
             const importedGrammars = grammar.imports.map(e => resolveImport(documents, e)).filter((e): e is Grammar => e !== undefined);
-            collectAllAstResources(importedGrammars, documents, visited, astResources);
+            collectAllAstResources(importedGrammars, visited, astResources, services);
         }
     }
     return astResources;

packages/langium/src/grammar/type-system/type-collector/declared-types.ts

@@ -5,13 +5,15 @@
  ******************************************************************************/
 
 import type { Interface, Type, TypeDefinition, ValueLiteral } from '../../../languages/generated/ast.js';
+import type { LangiumCoreServices } from '../../../index.js';
 import { isArrayLiteral, isBooleanLiteral } from '../../../languages/generated/ast.js';
 import type { PlainAstTypes, PlainInterface, PlainProperty, PlainPropertyDefaultValue, PlainPropertyType, PlainUnion } from './plain-types.js';
 import { isArrayType, isReferenceType, isUnionType, isSimpleType } from '../../../languages/generated/ast.js';
 import { getTypeNameWithoutError, isPrimitiveGrammarType } from '../../internal-grammar-util.js';
 import { getTypeName } from '../../../utils/grammar-utils.js';
 
-export function collectDeclaredTypes(interfaces: Interface[], unions: Type[]): PlainAstTypes {
+export function collectDeclaredTypes(interfaces: Interface[], unions: Type[], services?: LangiumCoreServices): PlainAstTypes {
+    const commentProvider = services?.documentation.CommentProvider;
     const declaredTypes: PlainAstTypes = { unions: [], interfaces: [] };
 
     // add interfaces
@@ -22,7 +24,8 @@ export function collectDeclaredTypes(interfaces: Interface[], unions: Type[]): P
                 name: attribute.name,
                 optional: attribute.isOptional,
                 astNodes: new Set([attribute]),
-                type: typeDefinitionToPropertyType(attribute.type)
+                type: typeDefinitionToPropertyType(attribute.type),
+                comment: commentProvider?.getComment(attribute)
             };
             if (attribute.defaultValue) {
                 property.defaultValue = toPropertyDefaultValue(attribute.defaultValue);
@@ -41,7 +44,8 @@ export function collectDeclaredTypes(interfaces: Interface[], unions: Type[]): P
             abstract: false,
             properties: properties,
             superTypes: superTypes,
-            subTypes: new Set()
+            subTypes: new Set(),
+            comment: commentProvider?.getComment(type),
         };
         declaredTypes.interfaces.push(interfaceType);
     }
@@ -53,7 +57,8 @@ export function collectDeclaredTypes(interfaces: Interface[], unions: Type[]): P
             declared: true,
             type: typeDefinitionToPropertyType(union.type),
             superTypes: new Set(),
-            subTypes: new Set()
+            subTypes: new Set(),
+            comment: commentProvider?.getComment(union),
         };
         declaredTypes.unions.push(unionType);
     }