Fix infinite tokenizer loop (#341)

* Major code cleanup

* Fix infinite loop in tokenizer

* Fix rgb color not accepting `.0` float format

Author: duckdoom4

.eslintrc.json

@@ -22,6 +22,7 @@
     "require-await": "warn",
     "camelcase": "error",
     "@typescript-eslint/no-var-requires": "warn",
-    "no-param-reassign": "warn" // Disable reassign, since this basically means you override the reference from the caller function with a new local version. (It doesn't do what you expect)
+    "no-param-reassign": "warn", // Disable reassign, since this basically means you override the reference from the caller function with a new local version. (It doesn't do what you expect)
+    "@typescript-eslint/no-namespace": "off"
   }
 }

.vscode/launch.json

@@ -15,7 +15,8 @@
         "${workspaceFolder}/examples" // open examples directory
       ],
       "outFiles": ["${workspaceFolder}/dist/**/*.js"],
-      "preLaunchTask": "npm: watch"
+      "preLaunchTask": "npm: watch",
+      "skipFiles": ["<node_internals>/**", "**/extensions/git*/**", "**/node_modules/prettier/**", "**/node/extensionHostProcess.js"]
     },
     {
       "name": "Launch Extension (Release)",
@@ -27,7 +28,8 @@
         "${workspaceFolder}/examples" // open examples directory
       ],
       "outFiles": ["${workspaceFolder}/dist/**/*.js"],
-      "preLaunchTask": "npm: watch-release"
+      "preLaunchTask": "npm: watch-release",
+      "skipFiles": ["<node_internals>/**", "**/extensions/git*/**", "**/node_modules/prettier/**"]
     },
     {
       "name": "Extension Tests",
@@ -36,7 +38,8 @@
       "runtimeExecutable": "${execPath}",
       "args": ["--extensionDevelopmentPath=${workspaceFolder}", "--extensionTestsPath=${workspaceFolder}/out/test"],
       "outFiles": ["${workspaceFolder}/out/test/**/*.js"],
-      "preLaunchTask": "npm: test-compile"
+      "preLaunchTask": "npm: test-compile",
+      "skipFiles": ["<node_internals>/**", "**/extensions/git*/**", "**/node_modules/prettier/**"]
     }
   ]
 }

TODO.md

@@ -3,7 +3,6 @@ Renpy Features List
 - Support launching the project through VSCode
 
 - Bugs to fix:
-  * Fix error on restart
   * Show color editor in tags (also check https://www.renpy.org/doc/html/color_class.html)
   * % can be escaped in strings
   * if line contains unclosed ( [ or { line is continued (see https://www.renpy.org/doc/html/language_basics.html#logical-lines)

package.json

@@ -2,7 +2,7 @@
   "name": "languague-renpy",
   "displayName": "Ren'Py Language",
   "description": "Adds rich support for the Ren'Py programming language to Visual Studio Code.",
-  "version": "2.3.3",
+  "version": "2.3.4",
   "publisher": "LuqueDaniel",
   "license": "MIT",
   "homepage": "https://github.com/LuqueDaniel/vscode-language-renpy",

src/color.ts

@@ -1,30 +1,44 @@
 // Color conversion methods for Color provider
-import { CancellationToken, Color, ColorInformation, ColorPresentation, DocumentColorProvider, Range, TextDocument, TextEdit } from "vscode";
+import { CancellationToken, Color, ColorInformation, ColorPresentation, ProviderResult, Range, TextDocument, TextEdit, languages } from "vscode";
 import { ValueEqualsSet } from "./utilities/hashset";
-import { tokenizeDocument } from "./tokenizer/tokenizer";
+import { Tokenizer } from "./tokenizer/tokenizer";
 import { LiteralTokenType } from "./tokenizer/renpy-tokens";
 import { TextMateRule, injectCustomTextmateTokens } from "./decorator";
-/*import { tokenizeDocument } from "./tokenizer/tokenizer";
-import { injectCustomTextmateTokens, TextMateRule } from "./decorator";
-import { LiteralTokenType } from "./tokenizer/renpy-tokens";
-import { ValueEqualsSet } from "./utilities/hashset";*/
 
-export class RenpyColorProvider implements DocumentColorProvider {
-    public provideDocumentColors(document: TextDocument, token: CancellationToken): Thenable<ColorInformation[]> {
-        return getColorInformation(document);
-    }
-    public provideColorPresentations(color: Color, context: { document: TextDocument; range: Range }, token: CancellationToken): Thenable<ColorPresentation[]> {
-        return getColorPresentations(color, context.document, context.range);
-    }
-}
+export type DocumentColorContext = {
+    document: TextDocument;
+    range: Range;
+};
+
+export const colorProvider = languages.registerColorProvider("renpy", {
+    provideDocumentColors(document: TextDocument, token: CancellationToken): ProviderResult<ColorInformation[]> {
+        if (token.isCancellationRequested) {
+            return;
+        }
+
+        return new Promise((resolve) => {
+            resolve(getColorInformation(document));
+        });
+    },
+
+    provideColorPresentations(color: Color, context: DocumentColorContext, token: CancellationToken): ProviderResult<ColorPresentation[]> {
+        if (token.isCancellationRequested) {
+            return;
+        }
+
+        return new Promise((resolve) => {
+            resolve(getColorPresentations(color, context));
+        });
+    },
+});
 
 /**
  * Finds all colors in the given document and returns their ranges and color
  * @param document - the TextDocument to search
- * @returns - Thenable<ColorInformation[]> - an array that provides a range and color for each match
+ * @returns - ColorInformation[] - an array that provides a range and color for each match
  */
-export function getColorInformation(document: TextDocument): Thenable<ColorInformation[]> {
-    injectCustomColorStyles(document);
+export async function getColorInformation(document: TextDocument) {
+    await injectCustomColorStyles(document);
 
     // find all colors in the document
     const colors: ColorInformation[] = [];
@@ -67,7 +81,7 @@ export function getColorInformation(document: TextDocument): Thenable<ColorInfor
             }
         }
     }
-    return Promise.resolve(colors);
+    return colors;
 }
 
 /**
@@ -77,12 +91,12 @@ export function getColorInformation(document: TextDocument): Thenable<ColorInfor
  * @param range - The Range of the color match
  * @returns - ColorPresentation to replace the color in the document with the new chosen color
  */
-export function getColorPresentations(color: Color, document: TextDocument, range: Range): Thenable<ColorPresentation[]> {
+export function getColorPresentations(color: Color, context: DocumentColorContext): ColorPresentation[] {
     // user hovered/tapped the color block/return the color they picked
     const colors: ColorPresentation[] = [];
-    const line = document.lineAt(range.start.line).text;
-    const text = line.substring(range.start.character, range.end.character);
-    const oldRange = new Range(range.start.line, range.start.character, range.start.line, range.start.character + text.length);
+    const range = context.range;
+    const text = context.document.getText(range);
+    const oldRange = new Range(range.start, range.end);
 
     const colR = Math.round(color.red * 255);
     const colG = Math.round(color.green * 255);
@@ -112,12 +126,12 @@ export function getColorPresentations(color: Color, document: TextDocument, rang
         colors.push(rgbColorPres);
     }
 
-    return Promise.resolve(colors);
+    return colors;
 }
 
-export function injectCustomColorStyles(document: TextDocument) {
+export async function injectCustomColorStyles(document: TextDocument) {
     // Disabled until filter is added to the tree class
-    const documentTokens = tokenizeDocument(document);
+    const documentTokens = await Tokenizer.tokenizeDocument(document);
     // TODO: Should probably make sure this constant is actually part of a tag, but for now this is fine.
     const colorTags = documentTokens.filter((x) => x.token?.tokenType === LiteralTokenType.Color);
     const colorRules = new ValueEqualsSet<TextMateRule>();
@@ -221,32 +235,55 @@ export function convertHtmlToColor(htmlHex: string): Color | null {
  */
 export function convertRenpyColorToColor(renpy: string): Color | null {
     try {
-        const colorTuple = renpy.replace("Color(", "").replace("color", "").replace("=", "").replace(" ", "").replace("(", "[").replace(")", "]");
-        const result = JSON.parse(colorTuple);
-        if (result.length === 3) {
-            return new Color(parseInt(result[0], 16) / 255, parseInt(result[1], 16) / 255, parseInt(result[2], 16) / 255, 1.0);
-        } else if (result.length === 4) {
-            return new Color(parseInt(result[0], 16) / 255, parseInt(result[1], 16) / 255, parseInt(result[2], 16) / 255, parseInt(result[3], 16) / 255);
+        const colorTuple = renpy
+            .replaceAll(" ", "")
+            .replace(/[Cc]olor=?\(/g, "")
+            .replace(")", "");
+
+        const result = colorTuple.split(",");
+        if (result.length < 3) {
+            return null;
         }
-        return null;
+
+        const r = parseInt(result[0], 16) / 255;
+        const g = parseInt(result[1], 16) / 255;
+        const b = parseInt(result[2], 16) / 255;
+        const a = result.length === 4 ? parseInt(result[3], 16) / 255 : 1.0;
+        return new Color(r, g, b, a);
     } catch (error) {
         return null;
     }
 }
 
+/**
+ * Returns a float value based on the given Ren'Py float string value
+ * @remarks Values starting with a dot (e.g., `.5`) are forced to be parsed as `0.5` due to javascript's `parseFloat` behavior.
+ * @param value The renpy float value to parse
+ * @returns The parsed float value
+ */
+function parseRenpyFloat(value: string): number {
+    if (value.startsWith(".")) {
+        return parseFloat("0" + value);
+    }
+    return parseFloat(value);
+}
+
 /**
  * Returns a Color provider object based on the given Ren'Py rgb tuple
- * @remarks
- * The rgb tuple values should be numeric values between 0.0 and 1.0 (e.g., `rgb=(1.0, 0.0, 0.0)`)
+ * @remarks The rgb tuple values should be numeric values between 0.0 and 1.0 (e.g., `rgb=(1.0, 0.0, 0.0)`).
+ * Values starting with a dot (e.g., `.5`) are forced to be parsed as `0.5` due to javascript's `parseFloat` behavior.
  * @param renpyColor - Renpy `rgb` color tuple (e.g., `rgb=(r, g, b)`)
  * @returns The `Color` provider object
  */
 export function convertRgbColorToColor(renpyColor: string): Color | null {
     try {
-        const colorTuple = renpyColor.replace("rgb", "").replace("=", "").replace(" ", "").replace("(", "[").replace(")", "]");
-        const result = JSON.parse(colorTuple);
+        const colorTuple = renpyColor
+            .replaceAll(" ", "")
+            .replace(/rgb=\(/g, "")
+            .replace(")", "");
+        const result = colorTuple.split(",");
         if (result.length === 3) {
-            return new Color(parseFloat(result[0]), parseFloat(result[1]), parseFloat(result[2]), 1.0);
+            return new Color(parseRenpyFloat(result[0]), parseRenpyFloat(result[1]), parseRenpyFloat(result[2]), 1.0);
         }
         return null;
     } catch (error) {

src/completion.ts

@@ -1,10 +1,30 @@
 // Completion Provider
-import { TextDocument, Position, CompletionContext, CompletionItem, CompletionTriggerKind, CompletionItemKind, workspace } from "vscode";
+import { TextDocument, Position, CompletionContext, CompletionItem, CompletionTriggerKind, CompletionItemKind, workspace, languages, CancellationToken, ProviderResult } from "vscode";
 import { Displayable } from "./displayable";
 import { getDefinitionFromFile } from "./hover";
 import { getCurrentContext } from "./navigation";
 import { NavigationData } from "./navigation-data";
 
+export const completionProvider = languages.registerCompletionItemProvider(
+    "renpy",
+    {
+        provideCompletionItems(document: TextDocument, position: Position, token: CancellationToken, context: CompletionContext): ProviderResult<CompletionItem[]> {
+            if (token.isCancellationRequested) {
+                return;
+            }
+
+            return new Promise((resolve) => {
+                resolve(getCompletionList(document, position, context));
+            });
+        },
+    },
+    ".",
+    " ",
+    "@",
+    "-",
+    "("
+);
+
 /**
  * Returns an array of auto-complete items related to the keyword at the given document/position
  * @param document - The current TextDocument

src/configuration.ts

@@ -0,0 +1,56 @@
+import { ConfigurationTarget, ExtensionContext, WorkspaceConfiguration, workspace } from "vscode";
+
+export class Configuration {
+    public static initialize(context: ExtensionContext) {
+        // hide rpyc files if the setting is enabled
+        const config = workspace.getConfiguration("renpy");
+        if (config?.excludeCompiledFilesFromWorkspace) {
+            this.excludeCompiledFilesConfig();
+        }
+
+        // Listen to configuration changes
+        context.subscriptions.push(
+            workspace.onDidChangeConfiguration((e) => {
+                if (e.affectsConfiguration("renpy.excludeCompiledFilesFromWorkspace")) {
+                    if (workspace.getConfiguration("renpy").get("excludeCompiledFilesFromWorkspace")) {
+                        this.excludeCompiledFilesConfig();
+                    }
+                }
+            })
+        );
+    }
+
+    public static isAutoSaveDisabled(): boolean {
+        const config = workspace.getConfiguration("files");
+        const autoSave = config.get<string>("autoSave");
+        return autoSave === "off";
+    }
+
+    public static compileOnDocumentSave(): boolean {
+        const config = workspace.getConfiguration("renpy");
+        return config.get<boolean>("compileOnDocumentSave") === true;
+    }
+
+    public static shouldWatchFoldersForChanges(): boolean {
+        const config = workspace.getConfiguration("renpy");
+        return config.get<boolean>("watchFoldersForChanges") === true;
+    }
+
+    public static getRenpyExecutablePath(): string {
+        const config = workspace.getConfiguration("renpy");
+        return config.get<string>("renpyExecutableLocation") || "";
+    }
+
+    private static excludeCompiledFilesConfig() {
+        const renpyExclude = ["**/*.rpyc", "**/*.rpa", "**/*.rpymc", "**/cache/"];
+        const config = workspace.getConfiguration("files");
+        const workspaceExclude = config.inspect<WorkspaceConfiguration>("exclude");
+        const exclude = { ...workspaceExclude?.workspaceValue };
+        renpyExclude.forEach((element) => {
+            if (!(element in exclude)) {
+                Object.assign(exclude, { [element]: true });
+            }
+        });
+        config.update("exclude", exclude, ConfigurationTarget.Workspace);
+    }
+}

src/definition.ts

@@ -1,16 +1,26 @@
-// Definition Provider
-"use strict";
-
-import { Definition, Location, Position, TextDocument, Uri } from "vscode";
+// Provider for Go To Definition
+import { CancellationToken, Definition, Location, Position, ProviderResult, TextDocument, Uri, languages } from "vscode";
 import { getKeywordPrefix } from "./extension";
 import { rangeAsString } from "./navigation";
 import { NavigationData } from "./navigation-data";
 import { getFileWithPath, stripWorkspaceFromFile } from "./workspace";
 
+export const definitionProvider = languages.registerDefinitionProvider("renpy", {
+    provideDefinition(document: TextDocument, position: Position, token: CancellationToken): ProviderResult<Definition> {
+        if (token.isCancellationRequested) {
+            return;
+        }
+
+        return new Promise((resolve) => {
+            resolve(getDefinition(document, position));
+        });
+    },
+});
+
 export function getDefinition(document: TextDocument, position: Position): Definition | undefined {
     const range = document.getWordRangeAtPosition(position);
     if (!range) {
-        return;
+        return undefined;
     }
 
     // check if this range is a semantic token

src/diagnostics.ts

@@ -1,6 +1,4 @@
 // Diagnostics (warnings and errors)
-"use strict";
-
 import { Diagnostic, DiagnosticCollection, DiagnosticSeverity, ExtensionContext, Range, TextDocument, window, workspace } from "vscode";
 import { NavigationData } from "./navigation-data";
 import { extractFilename } from "./workspace";

src/displayable.ts

@@ -1,5 +1,4 @@
 // Displayable Class
-"use strict";
 
 export class Displayable {
     name: string;