Add symbols provider (#54)

PhilippeChab · web-flow · commit d2c49614bdd0 · 2023-05-23T09:36:40.000-04:00
* Add symbols provider

* Fix comment

* Complete symbols provider

* Update REAME.md and CHANGELOG

* Fix minor issue with completion items provider

* Keep variable complex tokens inside function ones instead of keeping a parent id

* Tweak
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -84,4 +84,10 @@ I think we can consider the extension stable and out of beta. A big thank you to
 
 ## [2.0.2]
 
-- Add back the initial indexing. Its removal caused performance issues. 
+- Indexing added back. Its removal caused performances issues.
+
+## [2.1.0]
+
+- New provider: [Document Symbols](https://code.visualstudio.com/api/language-extensions/programmatic-language-features#show-all-symbol-definitions-within-a-document).
+- Fixed a small issue with the `CompletionItemsProvider`.
+- Goto will now _really_ work with definitions from `nwscript.nss` if the file is in your project.
diff --git a/README.md b/README.md
@@ -15,6 +15,7 @@ NWScript: EE Language Server is a Visual Studio Code extension for the NWScript
 - Range formatting
 - Signature help
 - Diagnostics
+- Document Symbols
 
 ## Dependencies
 
diff --git a/package.json b/package.json
@@ -8,7 +8,7 @@
     "url": "https://github.com/PhilippeChab/nwscript-ee-language-server"
   },
   "license": "MIT",
-  "version": "2.0.2",
+  "version": "2.1.0",
   "author": {
     "name": "Philippe Chabot"
   },
diff --git a/server/src/Documents/Document.ts b/server/src/Documents/Document.ts
@@ -3,8 +3,8 @@ import { STATIC_PREFIX } from "./DocumentsCollection";
 import type { ComplexToken, StructComplexToken } from "../Tokenizer/types";
 import type DocumentsCollection from "./DocumentsCollection";
 
-export type OwnedComplexTokens = { owner: string | null; tokens: ComplexToken[] };
-export type OwnedStructComplexTokens = { owner: string | null; tokens: StructComplexToken[] };
+export type OwnedComplexTokens = { owner?: string; tokens: ComplexToken[] };
+export type OwnedStructComplexTokens = { owner?: string; tokens: StructComplexToken[] };
 
 export default class Document {
   constructor(
@@ -42,13 +42,7 @@ export default class Document {
   }
 
   public getGlobalComplexTokensWithRef(computedChildren: string[] = []): OwnedComplexTokens[] {
-    const localStandardLibDefinitions = this.collection.get("nwscript");
-    return [
-      { owner: this.base ? null : this.uri, tokens: this.complexTokens },
-      ...(localStandardLibDefinitions
-        ? [{ owner: localStandardLibDefinitions.uri, tokens: localStandardLibDefinitions.complexTokens }]
-        : []),
-    ].concat(
+    return [{ owner: this.base ? undefined : this.uri, tokens: this.complexTokens } as OwnedComplexTokens].concat(
       this.children.flatMap((child) => {
         // Cycling children or/and duplicates
         if (computedChildren.includes(child)) {
@@ -92,7 +86,7 @@ export default class Document {
   }
 
   public getGlobalStructComplexTokensWithRef(computedChildren: string[] = []): OwnedStructComplexTokens[] {
-    return [{ owner: this.base ? null : this.uri, tokens: this.structComplexTokens }].concat(
+    return [{ owner: this.base ? undefined : this.uri, tokens: this.structComplexTokens } as OwnedStructComplexTokens].concat(
       this.children.flatMap((child) => {
         // Cycling children or/and duplicates
         if (computedChildren.includes(child)) {
diff --git a/server/src/Providers/Builders/SymbolBuilder.ts b/server/src/Providers/Builders/SymbolBuilder.ts
@@ -0,0 +1,99 @@
+import { DocumentSymbol, SymbolKind } from "vscode-languageserver";
+
+import type {
+  ComplexToken,
+  ConstantComplexToken,
+  FunctionComplexToken,
+  FunctionParamComplexToken,
+  StructComplexToken,
+  StructPropertyComplexToken,
+  VariableComplexToken,
+} from "../../Tokenizer/types";
+import Builder from "./Builder";
+
+export default class SymbolBuilder extends Builder {
+  public static buildItem(token: ComplexToken): DocumentSymbol {
+    if (this.isConstantToken(token)) {
+      return this.buildConstantItem(token);
+    } else if (this.isVariableToken(token)) {
+      return this.buildVariableItem(token);
+    } else if (this.isFunctionParameterToken(token)) {
+      return this.buildFunctionParamItem(token);
+    } else if (this.isFunctionToken(token)) {
+      return this.buildFunctionItem(token);
+    } else if (this.isStructPropertyToken(token)) {
+      return this.buildStructPropertyItem(token);
+    } else if (this.isStructToken(token)) {
+      return this.buildStructItem(token);
+    } else {
+      throw new Error("Invalid complex token. Cannot build symbol.");
+    }
+  }
+
+  private static buildConstantItem(token: ConstantComplexToken) {
+    return DocumentSymbol.create(
+      token.identifier,
+      undefined,
+      SymbolKind.Constant,
+      { start: token.position, end: token.position },
+      { start: token.position, end: token.position },
+    );
+  }
+
+  private static buildVariableItem(token: VariableComplexToken) {
+    return DocumentSymbol.create(
+      token.identifier,
+      undefined,
+      SymbolKind.Variable,
+      { start: token.position, end: token.position },
+      { start: token.position, end: token.position },
+    );
+  }
+
+  private static buildFunctionParamItem(token: FunctionParamComplexToken) {
+    return DocumentSymbol.create(
+      token.identifier,
+      undefined,
+      SymbolKind.TypeParameter,
+      { start: token.position, end: token.position },
+      { start: token.position, end: token.position },
+    );
+  }
+
+  private static buildFunctionItem(token: FunctionComplexToken) {
+    const paramSymbols = token.params.map((child) => SymbolBuilder.buildItem(child)) || [];
+    const variableSymbols = token.variables?.map((child) => SymbolBuilder.buildItem(child)) || [];
+
+    return DocumentSymbol.create(
+      token.identifier,
+      undefined,
+      SymbolKind.Function,
+      { start: token.position, end: token.position },
+      { start: token.position, end: token.position },
+      paramSymbols.concat(variableSymbols),
+    );
+  }
+
+  private static buildStructPropertyItem(token: StructPropertyComplexToken) {
+    return DocumentSymbol.create(
+      token.identifier,
+      undefined,
+      SymbolKind.Property,
+      { start: token.position, end: token.position },
+      { start: token.position, end: token.position },
+    );
+  }
+
+  private static buildStructItem(token: StructComplexToken) {
+    const symbols = token.properties?.map((child) => SymbolBuilder.buildItem(child));
+
+    return DocumentSymbol.create(
+      token.identifier,
+      undefined,
+      SymbolKind.Struct,
+      { start: token.position, end: token.position },
+      { start: token.position, end: token.position },
+      symbols,
+    );
+  }
+}
diff --git a/server/src/Providers/Builders/index.ts b/server/src/Providers/Builders/index.ts
@@ -1,5 +1,6 @@
 import CompletionItemBuilder from "./CompletionItemBuilder";
 import HoverContentBuilder from "./HoverContentBuilder";
 import SignatureHelpBuilder from "./SignatureHelpBuilder";
+import SymbolBuilder from "./SymbolBuilder";
 
-export { CompletionItemBuilder, HoverContentBuilder, SignatureHelpBuilder };
+export { CompletionItemBuilder, HoverContentBuilder, SignatureHelpBuilder, SymbolBuilder };
diff --git a/server/src/Providers/CompletionItemsProvider.ts b/server/src/Providers/CompletionItemsProvider.ts
@@ -37,7 +37,7 @@ export default class CompletionItemsProvider extends Provider {
               const structVariableIdentifier = this.server.tokenizer?.findLineIdentiferFromPositionAt(
                 liveDocument.getText(),
                 position,
-                -2,
+                -1,
               );
 
               const structIdentifer = localScope.functionVariablesComplexTokens.find(
@@ -53,7 +53,7 @@ export default class CompletionItemsProvider extends Provider {
             }
 
             if (
-              this.server.tokenizer?.findLineIdentiferFromPositionAt(liveDocument.getText(), position, -3) ===
+              this.server.tokenizer?.findLineIdentiferFromPositionAt(liveDocument.getText(), position, -2) ===
               LanguageTypes.struct
             ) {
               return document.getGlobalStructComplexTokens().map((token) => CompletionItemBuilder.buildItem(token));
diff --git a/server/src/Providers/GotoDefinitionProvider.ts b/server/src/Providers/GotoDefinitionProvider.ts
@@ -74,7 +74,13 @@ export default class GotoDefinitionProvider extends Provider {
           }
 
           if (!token && (tokenType === CompletionItemKind.Constant || tokenType === CompletionItemKind.Function)) {
+            const localStandardLibDefinitions = this.server.documentsCollection.get("nwscript");
             const tokensWithRef = document.getGlobalComplexTokensWithRef();
+
+            if (localStandardLibDefinitions) {
+              tokensWithRef.push({ owner: localStandardLibDefinitions?.uri, tokens: localStandardLibDefinitions?.complexTokens });
+            }
+
             for (let i = 0; i < tokensWithRef.length; i++) {
               ref = tokensWithRef[i];
 
diff --git a/server/src/Providers/SymbolsProvider.ts b/server/src/Providers/SymbolsProvider.ts
@@ -0,0 +1,42 @@
+import { CompletionItemKind, DocumentSymbolParams } from "vscode-languageserver";
+
+import type { ServerManager } from "../ServerManager";
+import { SymbolBuilder } from "./Builders";
+import { TokenizedScope } from "../Tokenizer/Tokenizer";
+import Provider from "./Provider";
+
+export default class SymbolsProvider extends Provider {
+  constructor(server: ServerManager) {
+    super(server);
+
+    this.server.connection.onDocumentSymbol((params) => this.exceptionsWrapper(this.providerHandler(params)));
+  }
+
+  private providerHandler(params: DocumentSymbolParams) {
+    return () => {
+      const {
+        textDocument: { uri },
+      } = params;
+
+      const liveDocument = this.server.liveDocumentsManager.get(uri);
+      const document = this.server.documentsCollection.getFromUri(uri);
+
+      const constantSymbols =
+        document?.complexTokens
+          .filter((token) => token.tokenType === CompletionItemKind.Constant)
+          .map((token) => SymbolBuilder.buildItem(token)) || [];
+
+      const structSymbols = document?.structComplexTokens.map((token) => SymbolBuilder.buildItem(token)) || [];
+
+      if (liveDocument) {
+        const localScope = this.server.tokenizer?.tokenizeContent(liveDocument.getText(), TokenizedScope.local);
+
+        if (localScope) {
+          return constantSymbols.concat(
+            structSymbols.concat(localScope.functionsComplexTokens.map((token) => SymbolBuilder.buildItem(token))),
+          );
+        }
+      }
+    };
+  }
+}
diff --git a/server/src/Providers/index.ts b/server/src/Providers/index.ts
@@ -7,6 +7,7 @@ import SignatureHelpProvider from "./SignatureHelpProvider";
 import DocumentFormatingProvider from "./DocumentFormattingProvider";
 import DocumentRangeFormattingProvider from "./DocumentRangeFormattingProvider";
 import DiagnosticsProvider from "./DiagnosticsProvider";
+import SymbolsProvider from "./SymbolsProvider";
 
 export enum TriggerCharacters {
   dot = ".",
@@ -24,4 +25,5 @@ export {
   DocumentFormatingProvider,
   DocumentRangeFormattingProvider,
   DiagnosticsProvider,
+  SymbolsProvider,
 };
diff --git a/server/src/ServerManager/CapabilitiesHandler.ts b/server/src/ServerManager/CapabilitiesHandler.ts
@@ -19,6 +19,7 @@ export default class CapabilitiesHandler {
       },
       documentFormattingProvider: true,
       documentRangeFormattingProvider: true,
+      documentSymbolProvider: true,
       definitionProvider: true,
       hoverProvider: true,
       completionProvider: {
diff --git a/server/src/ServerManager/ServerManager.ts b/server/src/ServerManager/ServerManager.ts
@@ -1,5 +1,6 @@
 import { cpus } from "os";
 import { join } from "path";
+import { readFileSync } from "fs";
 import { pathToFileURL } from "url";
 import * as clustering from "cluster";
 import type { Connection, InitializeParams } from "vscode-languageserver";
@@ -13,10 +14,12 @@ import {
   GotoDefinitionProvider,
   HoverContentProvider,
   SignatureHelpProvider,
+  SymbolsProvider,
   WorkspaceProvider,
 } from "../Providers";
 import { DocumentsCollection, LiveDocumentsManager } from "../Documents";
 import { Tokenizer } from "../Tokenizer";
+import { TokenizedScope } from "../Tokenizer/Tokenizer";
 import { WorkspaceFilesSystem } from "../WorkspaceFilesSystem";
 import { Logger } from "../Logger";
 import { defaultServerConfiguration } from "./Config";
@@ -82,6 +85,7 @@ export default class ServerManger {
 
     let filesIndexedCount = 0;
     const filesPath = this.workspaceFilesSystem.getAllFilesPath();
+    const nwscriptPath = filesPath.find((path) => path.includes("nwscript.nss"));
     const progressReporter = await this.connection.window.createWorkDoneProgress();
     const filesCount = filesPath.length;
     this.logger.info(`Indexing files ...`);
@@ -105,6 +109,12 @@ export default class ServerManger {
         this.logger.info(`Indexed ${filesIndexedCount} files.`);
         this.configLoaded = true;
         this.diagnosticsProvider?.processDocumentsWaitingForPublish();
+
+        if (nwscriptPath) {
+          const fileContent = readFileSync(nwscriptPath).toString();
+          const globalScope = this.tokenizer?.tokenizeContent(fileContent, TokenizedScope.global)!;
+          this.documentsCollection?.createDocument(pathToFileURL(nwscriptPath).href, globalScope);
+        }
       }
     });
   }
@@ -118,6 +128,7 @@ export default class ServerManger {
     SignatureHelpProvider.register(this);
     DocumentFormatingProvider.register(this);
     DocumentRangeFormattingProvider.register(this);
+    SymbolsProvider.register(this);
     this.diagnosticsProvider = DiagnosticsProvider.register(this) as DiagnosticsProvider;
   }
 
diff --git a/server/src/Tokenizer/Tokenizer.ts b/server/src/Tokenizer/Tokenizer.ts
diff --git a/server/src/Tokenizer/types.ts b/server/src/Tokenizer/types.ts
diff --git a/server/test/static/localScopeTokens.json b/server/test/static/localScopeTokens.json
diff --git a/server/test/static/localScopeTokensWithContext.json b/server/test/static/localScopeTokensWithContext.json
diff --git a/server/test/static/localScopeTokensWithoutContext.json b/server/test/static/localScopeTokensWithoutContext.json
diff --git a/server/test/tokenization_test.ts b/server/test/tokenization_test.ts
