Support language and interpreter in the web (#3)

msujew · msujew · commit bc8d6c52d0f4 · 2024-07-07T15:32:34.000+02:00
diff --git a/langium-config.json b/langium-config.json
@@ -6,6 +6,9 @@
         "fileExtensions": [".lox"],
         "textMate": {
             "out": "syntaxes/lox.tmLanguage.json"
+        },
+        "monarch": {
+            "out": "syntaxes/lox.monarch.ts"
         }
     }],
     "out": "src/language-server/generated"
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -10,26 +10,37 @@
         "Programming Languages"
     ],
     "contributes": {
-        "languages": [{
-            "id": "lox",
-            "aliases": ["Lox", "lox"],
-            "extensions": [".lox"],
-            "configuration": "./language-configuration.json"
-        }],
-        "grammars": [{
-            "language": "lox",
-            "scopeName": "source.lox",
-            "path": "./syntaxes/lox.tmLanguage.json"
-        }],
-        "notebooks": [{
-            "type": "lox-notebook",
-            "displayName": "Lox Notebook",
-            "selector": [
-                {
-                    "filenamePattern": "*.loxnb"
-                }
-            ]
-        }]
+        "languages": [
+            {
+                "id": "lox",
+                "aliases": [
+                    "Lox",
+                    "lox"
+                ],
+                "extensions": [
+                    ".lox"
+                ],
+                "configuration": "./language-configuration.json"
+            }
+        ],
+        "grammars": [
+            {
+                "language": "lox",
+                "scopeName": "source.lox",
+                "path": "./syntaxes/lox.tmLanguage.json"
+            }
+        ],
+        "notebooks": [
+            {
+                "type": "lox-notebook",
+                "displayName": "Lox Notebook",
+                "selector": [
+                    {
+                        "filenamePattern": "*.loxnb"
+                    }
+                ]
+            }
+        ]
     },
     "activationEvents": [
         "onLanguage:lox"
@@ -49,18 +60,16 @@
         "build": "tsc -b tsconfig.json",
         "watch": "tsc -b tsconfig.json --watch",
         "lint": "eslint src --ext ts",
+        "clean": "shx rm -rf out node_modules",
         "langium:generate": "langium generate",
         "langium:watch": "langium generate --watch"
     },
     "dependencies": {
-        "chevrotain": "^9.1.0",
         "colors": "^1.4.0",
         "commander": "^8.0.0",
-        "langium": "1.0.1",
+        "langium": "~1.2.1",
         "uuid": "^9.0.0",
-        "vscode-languageclient": "8.0.2",
-        "vscode-languageserver": "8.0.2",
-        "vscode-uri": "^3.0.2"
+        "vscode-languageclient": "^8.1.0"
     },
     "devDependencies": {
         "@types/node": "^14.17.3",
@@ -69,7 +78,8 @@
         "@typescript-eslint/eslint-plugin": "^4.14.1",
         "@typescript-eslint/parser": "^4.14.1",
         "eslint": "^7.19.0",
-        "langium-cli": "1.0.0",
+        "langium-cli": "~1.2.1",
+        "shx": "^0.3.4",
         "typescript": "^4.6.2"
     }
 }
diff --git a/src/interpreter/runner.ts b/src/interpreter/runner.ts
@@ -3,14 +3,18 @@ import { BinaryExpression, Expression, isBinaryExpression, isBooleanExpression,
 import { createLoxServices } from "../language-server/lox-module";
 import { v4 } from 'uuid';
 import { URI } from "vscode-uri";
-import { CancellationToken } from "vscode-languageclient";
+import { CancellationToken, CancellationTokenSource } from "vscode-languageserver";
 
 export interface InterpreterContext {
-    log: (value: unknown) => MaybePromise<void>
+    log: (value: unknown) => MaybePromise<void>,
+    onStart?: () => void,
 }
 
 const services = createLoxServices(EmptyFileSystem);
 
+// after 5 seconds, the interpreter will be interrupted and call onTimeout
+const TIMEOUT_MS = 1000 * 5;
+
 export async function runInterpreter(program: string, context: InterpreterContext): Promise<void> {
     const buildResult = await buildDocument(program);
     try {
@@ -25,7 +29,10 @@ type ReturnFunction = (value: unknown) => void;
 
 interface RunnerContext {
     variables: Variables,
-    log: (value: unknown) => MaybePromise<void>
+    cancellationToken: CancellationToken,
+    timeout: NodeJS.Timeout,
+    log: (value: unknown) => MaybePromise<void>,
+    onStart?: () => void,
 }
 
 class Variables {
@@ -88,17 +95,38 @@ async function buildDocument(program: string): Promise<BuildResult> {
     }
 }
 
-async function runProgram(program: LoxProgram, outerContext: InterpreterContext): Promise<void> {
+export async function runProgram(program: LoxProgram, outerContext: InterpreterContext): Promise<void> {
+    const cancellationTokenSource = new CancellationTokenSource();
+    const cancellationToken = cancellationTokenSource.token;
+    
+    const timeout = setTimeout(async () => {
+        cancellationTokenSource.cancel();  
+    }, TIMEOUT_MS);
+    
     const context: RunnerContext = {
         variables: new Variables(),
-        log: outerContext.log
+        cancellationToken,
+        timeout,
+        log: outerContext.log,
+        onStart: outerContext.onStart,
     };
-    context.variables.enter();
+
     let end = false;
+    
+    context.variables.enter();
+    if (context.onStart) {
+        context.onStart();
+    }
+
     for (const statement of program.elements) {
+        await interruptAndCheck(context.cancellationToken);
+
         if (!isClass(statement) && !isFunctionDeclaration(statement)) {
             await runLoxElement(statement, context, () => { end = true });
         }
+        else if (isClass(statement)) {
+            throw new AstNodeError(statement, 'Classes are currently unsupported');
+        }
         if (end) {
             break;
         }
@@ -107,6 +135,8 @@ async function runProgram(program: LoxProgram, outerContext: InterpreterContext)
 }
 
 async function runLoxElement(element: LoxElement, context: RunnerContext, returnFn: ReturnFunction): Promise<void> {
+    await interruptAndCheck(context.cancellationToken);
+
     if (isExpressionBlock(element)) {
         await interruptAndCheck(CancellationToken.None);
         context.variables.enter();
@@ -164,6 +194,9 @@ async function runLoxElement(element: LoxElement, context: RunnerContext, return
 }
 
 async function runExpression(expression: Expression, context: RunnerContext): Promise<unknown> {
+    await interruptAndCheck(context.cancellationToken);
+
+
     if (isBinaryExpression(expression)) {
         const { left, right, operator } = expression;
         const rightValue = await runExpression(right, context);
@@ -244,6 +277,8 @@ async function setExpressionValue(left: Expression, right: unknown, context: Run
 }
 
 async function runMemberCall(memberCall: MemberCall, context: RunnerContext): Promise<unknown> {
+    await interruptAndCheck(context.cancellationToken);
+
     let previous: unknown = undefined;
     if (memberCall.previous) {
         previous = await runExpression(memberCall.previous, context);
@@ -255,7 +290,7 @@ async function runMemberCall(memberCall: MemberCall, context: RunnerContext): Pr
     } else if (isVariableDeclaration(ref) || isParameter(ref)) {
         value = context.variables.get(memberCall, ref.name);
     } else if (isClass(ref)) {
-        throw new AstNodeError(memberCall, 'Classes are current unsupported');
+        throw new AstNodeError(memberCall, 'Classes are currently unsupported');
     } else {
         value = previous;
     }
diff --git a/src/language-server/lox-validator.ts b/src/language-server/lox-validator.ts
@@ -1,5 +1,5 @@
 import { AstNode, streamAllContents, ValidationAcceptor, ValidationChecks, ValidationRegistry } from 'langium';
-import { BinaryExpression, ExpressionBlock, FunctionDeclaration, isReturnStatement, LoxAstType, MethodMember, TypeReference, UnaryExpression, VariableDeclaration } from './generated/ast';
+import { BinaryExpression, Class, ExpressionBlock, FunctionDeclaration, isReturnStatement, LoxAstType, MethodMember, TypeReference, UnaryExpression, VariableDeclaration } from './generated/ast';
 import type { LoxServices } from './lox-module';
 import { isAssignable } from './type-system/assignment';
 import { isVoidType, TypeDescription, typeToString } from './type-system/descriptions';
@@ -18,6 +18,7 @@ export class LoxValidationRegistry extends ValidationRegistry {
             UnaryExpression: validator.checkUnaryOperationAllowed,
             VariableDeclaration: validator.checkVariableDeclaration,
             MethodMember: validator.checkMethodReturnType,
+            Class: validator.checkClassDeclaration,
             FunctionDeclaration: validator.checkFunctionReturnType
         };
         this.register(checks, validator);
@@ -37,6 +38,14 @@ export class LoxValidator {
         this.checkFunctionReturnTypeInternal(method.body, method.returnType, accept);
     }
 
+    // TODO: implement classes 
+    checkClassDeclaration(declaration: Class, accept: ValidationAcceptor): void {
+        accept('error', 'Classes are currently unsupported.', {
+            node: declaration,
+            property: 'name'
+        });
+    }
+
     private checkFunctionReturnTypeInternal(body: ExpressionBlock, returnType: TypeReference, accept: ValidationAcceptor): void {
         const map = this.getTypeCache();
         const returnStatements = streamAllContents(body).filter(isReturnStatement).toArray();
diff --git a/src/language-server/main-browser.ts b/src/language-server/main-browser.ts
@@ -0,0 +1,66 @@
+/******************************************************************************
+ * Copyright 2022 TypeFox GmbH
+ * This program and the accompanying materials are made available under the
+ * terms of the MIT License, which is available in the project root.
+ ******************************************************************************/
+
+import { startLanguageServer, EmptyFileSystem, DocumentState, LangiumDocument, OperationCancelled } from 'langium';
+import { BrowserMessageReader, BrowserMessageWriter, Diagnostic, NotificationType, createConnection } from 'vscode-languageserver/browser';
+import { runInterpreter } from '../interpreter/runner';
+import { createLoxServices } from './lox-module';
+
+declare const self: DedicatedWorkerGlobalScope;
+
+/* browser specific setup code */
+const messageReader = new BrowserMessageReader(self);
+const messageWriter = new BrowserMessageWriter(self);
+
+const connection = createConnection(messageReader, messageWriter);
+
+// Inject the shared services and language-specific services
+const { shared } = createLoxServices({ connection, ...EmptyFileSystem });
+
+// Start the language server with the shared services
+startLanguageServer(shared);
+
+// Send a notification with the serialized AST after every document change
+type DocumentChange = { uri: string, content: string, diagnostics: Diagnostic[] };
+const documentChangeNotification = new NotificationType<DocumentChange>('browser/DocumentChange');
+
+
+shared.workspace.DocumentBuilder.onBuildPhase(DocumentState.Validated, async documents => {
+    for (const document of documents) {
+        if (document.diagnostics === undefined || document.diagnostics.filter((i) => i.severity === 1).length === 0) {
+            try {
+                await runInterpreter(document.textDocument.getText(), {
+                    log: (message) => {
+                        sendMessage(document, "output", message);
+                    },
+                    onStart: () => {
+                        sendMessage(document, "notification", "startInterpreter");
+                    },
+                });
+            } catch (e: any) {
+                if (e === OperationCancelled) {
+                    sendMessage(document, "error", "Interpreter timed out");
+                } else {
+                    sendMessage(document, "error", e.message);
+                }
+            } finally {
+                sendMessage(document, "notification", "endInterpreter");
+            }
+
+        }
+        else {
+            sendMessage(document, "error", document.diagnostics)
+        }
+    }
+});
+
+function sendMessage(document: LangiumDocument, type: string, content: unknown): void {
+    connection.sendNotification(documentChangeNotification, {
+        uri: document.uri.toString(),
+        content: JSON.stringify({ type, content }),
+        diagnostics: document.diagnostics ?? []
+    });
+}
diff --git a/syntaxes/lox.monarch.ts b/syntaxes/lox.monarch.ts
@@ -0,0 +1,30 @@
+// Monarch syntax highlighting for the lox language.
+export default {
+    keywords: [
+        'and','boolean','class','else','false','for','fun','if','nil','number','or','print','return','string','super','this','true','var','void','while'
+    ],
+    operators: [
+        '!','!=','*','+',',','-','.','/',':',';','<','<=','=','==','=>','>','>='
+    ],
+    symbols:  /!|!=|\(|\)|\*|\+|,|-|\.|/|:|;|<|<=|=|==|=>|>|>=|\{|\}/,
+
+    tokenizer: {
+        initial: [
+            { regex: /[_a-zA-Z][\w_]*/, action: { cases: { '@keywords': {"token":"keyword"}, '@default': {"token":"ID"} }} },
+            { regex: /[0-9]+(\.[0-9]+)?/, action: {"token":"number"} },
+            { regex: /"[^"]*"/, action: {"token":"string"} },
+            { include: '@whitespace' },
+            { regex: /@symbols/, action: { cases: { '@operators': {"token":"operator"}, '@default': {"token":""} }} },
+        ],
+        whitespace: [
+            { regex: /\s+/, action: {"token":"white"} },
+            { regex: /\/\*/, action: {"token":"comment","next":"@comment"} },
+            { regex: /\/\/[^\n\r]*/, action: {"token":"comment"} },
+        ],
+        comment: [
+            { regex: /[^\/\*]+/, action: {"token":"comment"} },
+            { regex: /\*\//, action: {"token":"comment","next":"@pop"} },
+            { regex: /[\/\*]/, action: {"token":"comment"} },
+        ],
+    }
+};
diff --git a/syntaxes/lox.tmLanguage.json b/syntaxes/lox.tmLanguage.json
@@ -15,7 +15,12 @@
     {
       "name": "string.quoted.double.lox",
       "begin": "\"",
-      "end": "\""
+      "end": "\"",
+      "patterns": [
+        {
+          "include": "#string-character-escape"
+        }
+      ]
     }
   ],
   "repository": {
@@ -47,6 +52,10 @@
           "name": "comment.line.lox"
         }
       ]
+    },
+    "string-character-escape": {
+      "name": "constant.character.escape.lox",
+      "match": "\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|u\\{[0-9A-Fa-f]+\\}|[0-2][0-7]{0,2}|3[0-6][0-7]?|37[0-7]?|[4-7][0-7]?|.|$)"
     }
   }
-}
+}
diff --git a/tsconfig.json b/tsconfig.json
@@ -2,7 +2,7 @@
   "compilerOptions": {
     "target": "ES6",
     "module": "commonjs",
-    "lib": ["ESNext"],
+    "lib": ["ESNext", "WebWorker"],
     "sourceMap": true,
     "outDir": "out",
     "strict": true,

Original file line number	Diff line number	Diff line change
`@@ -15,7 +15,12 @@`
`15`	`15`	`{`
`16`	`16`	`"name": "string.quoted.double.lox",`
`17`	`17`	`"begin": "\"",`
`18`		`- "end": "\""`
	`18`	`+ "end": "\"",`
	`19`	`+ "patterns": [`
	`20`	`+ {`
	`21`	`+ "include": "#string-character-escape"`
	`22`	`+ }`
	`23`	`+ ]`
`19`	`24`	`}`
`20`	`25`	`],`
`21`	`26`	`"repository": {`
`@@ -47,6 +52,10 @@`
`47`	`52`	`"name": "comment.line.lox"`
`48`	`53`	`}`
`49`	`54`	`]`
	`55`	`+ },`
	`56`	`+ "string-character-escape": {`
	`57`	`+ "name": "constant.character.escape.lox",`
	`58`	`+ "match": "\\\\(x[0-9A-Fa-f]{2}\|u[0-9A-Fa-f]{4}\|u\\{[0-9A-Fa-f]+\\}\|[0-2][0-7]{0,2}\|3[0-6][0-7]?\|37[0-7]?\|[4-7][0-7]?\|.\|$)"`
`50`	`59`	`}`
`51`	`60`	`}`
`52`		`-}`
	`61`	`+}`