Skip to content

Commit 24ae1bf

Browse files
PhilippeChabPhilippeChab
authored
Clean up tokenizer methods (#64)
* Clean up tokenizer methods * Small fix * Remove unused imports and variables
1 parent aedf961 commit 24ae1bf

File tree

5 files changed

+120
-257
lines changed

5 files changed

+120
-257
lines changed

‎server/src/Providers/CompletionItemsProvider.ts

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { CompletionParams } from "vscode-languageserver";
22

33
import type { ServerManager } from "../ServerManager";
44
import { CompletionItemBuilder } from "./Builders";
5-
import { LocalScopeTokenizationResult, TokenizedScope } from "../Tokenizer/Tokenizer";
5+
import { LocalScopeTokenizationResult } from "../Tokenizer/Tokenizer";
66
import { TriggerCharacters } from ".";
77
import { Document } from "../Documents";
88
import { LanguageTypes } from "../Tokenizer/constants";
@@ -27,11 +27,12 @@ export default class CompletionItemsProvider extends Provider {
2727
const document = this.server.documentsCollection.getFromUri(uri);
2828
if (!liveDocument || !document) return;
2929

30-
const localScope = this.server.tokenizer.tokenizeContent(liveDocument.getText(), TokenizedScope.local, 0, position.line);
30+
const [lines, rawTokenizedContent] = this.server.tokenizer.tokenizeContentToRaw(liveDocument.getText());
31+
const localScope = this.server.tokenizer.tokenizeContentFromRaw(lines, rawTokenizedContent, 0, position.line);
3132

3233
if (params.context?.triggerCharacter === TriggerCharacters.dot) {
33-
const structVariableIdentifier = this.server.tokenizer.findLineIdentiferFromPositionAt(liveDocument.getText(), position, -1);
34-
const structIdentifer = localScope.functionVariablesComplexTokens.find((token) => token.identifier === structVariableIdentifier)?.valueType;
34+
const { rawContent } = this.server.tokenizer.getActionTargetAtPosition(lines, rawTokenizedContent, position, -1);
35+
const structIdentifer = localScope.functionVariablesComplexTokens.find((token) => token.identifier === rawContent)?.valueType;
3536

3637
return document
3738
.getGlobalStructComplexTokens()
@@ -41,7 +42,7 @@ export default class CompletionItemsProvider extends Provider {
4142
});
4243
}
4344

44-
if (this.server.tokenizer.findLineIdentiferFromPositionAt(liveDocument.getText(), position, -2) === LanguageTypes.struct) {
45+
if (this.server.tokenizer.getActionTargetAtPosition(lines, rawTokenizedContent, position, -2).rawContent === LanguageTypes.struct) {
4546
return document.getGlobalStructComplexTokens().map((token) => CompletionItemBuilder.buildItem(token));
4647
}
4748

‎server/src/Providers/GotoDefinitionProvider.ts

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import { TextDocument } from "vscode-languageserver-textdocument";
44
import type { OwnedComplexTokens, OwnedStructComplexTokens } from "../Documents/Document";
55
import type { ServerManager } from "../ServerManager";
66
import type { ComplexToken } from "../Tokenizer/types";
7-
import { TokenizedScope } from "../Tokenizer/Tokenizer";
87
import { Document } from "../Documents";
98
import Provider from "./Provider";
109

@@ -55,13 +54,14 @@ export default class GotoDefinitionProvider extends Provider {
5554
let token: ComplexToken | undefined;
5655
let ref: OwnedComplexTokens | OwnedStructComplexTokens | undefined;
5756

58-
const { tokenType, structVariableIdentifier, identifier } = this.server.tokenizer.findActionTargetAtPosition(liveDocument.getText(), position);
59-
const localScope = this.server.tokenizer.tokenizeContent(liveDocument.getText(), TokenizedScope.local, 0, position.line);
57+
const [lines, rawTokenizedContent] = this.server.tokenizer.tokenizeContentToRaw(liveDocument.getText());
58+
const localScope = this.server.tokenizer.tokenizeContentFromRaw(lines, rawTokenizedContent, 0, position.line);
59+
const { tokenType, lookBehindRawContent, rawContent } = this.server.tokenizer.getActionTargetAtPosition(lines, rawTokenizedContent, position);
6060

6161
switch (tokenType) {
6262
case CompletionItemKind.Function:
6363
case CompletionItemKind.Constant:
64-
token = localScope.functionsComplexTokens.find((candidate) => candidate.identifier === identifier);
64+
token = localScope.functionsComplexTokens.find((candidate) => candidate.identifier === rawContent);
6565
if (token) break;
6666

6767
const localStandardLibDefinitions = this.server.documentsCollection.get("nwscript");
@@ -74,7 +74,7 @@ export default class GotoDefinitionProvider extends Provider {
7474
loop: for (let i = 0; i < tokensWithRef.length; i++) {
7575
ref = tokensWithRef[i];
7676

77-
token = ref?.tokens.find((candidate) => candidate.identifier === identifier);
77+
token = ref?.tokens.find((candidate) => candidate.identifier === rawContent);
7878
if (token) {
7979
break loop;
8080
}
@@ -85,28 +85,28 @@ export default class GotoDefinitionProvider extends Provider {
8585
loop: for (let i = 0; i < tokensWithRef.length; i++) {
8686
ref = tokensWithRef[i];
8787

88-
token = ref?.tokens.find((candidate) => candidate.identifier === identifier);
88+
token = ref?.tokens.find((candidate) => candidate.identifier === rawContent);
8989
if (token) {
9090
break loop;
9191
}
9292
}
9393
break;
9494
case CompletionItemKind.Property:
95-
const structIdentifer = localScope.functionVariablesComplexTokens.find((candidate) => candidate.identifier === structVariableIdentifier)?.valueType;
95+
const structIdentifer = localScope.functionVariablesComplexTokens.find((candidate) => candidate.identifier === lookBehindRawContent)?.valueType;
9696

9797
tokensWithRef = document.getGlobalStructComplexTokensWithRef();
9898
loop: for (let i = 0; i < tokensWithRef.length; i++) {
9999
ref = tokensWithRef[i];
100100

101-
token = (ref as OwnedStructComplexTokens).tokens.find((candidate) => candidate.identifier === structIdentifer)?.properties.find((property) => property.identifier === identifier);
101+
token = (ref as OwnedStructComplexTokens).tokens.find((candidate) => candidate.identifier === structIdentifer)?.properties.find((property) => property.identifier === rawContent);
102102

103103
if (token) {
104104
break loop;
105105
}
106106
}
107107
break;
108108
default:
109-
token = localScope.functionVariablesComplexTokens.find((candidate) => candidate.identifier === identifier);
109+
token = localScope.functionVariablesComplexTokens.find((candidate) => candidate.identifier === rawContent);
110110
}
111111

112112
return [token, ref];

‎server/src/Providers/HoverContentProvider.ts

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ import { TextDocument } from "vscode-languageserver-textdocument";
33

44
import type { ServerManager } from "../ServerManager";
55
import type { ComplexToken } from "../Tokenizer/types";
6-
import { TokenizedScope } from "../Tokenizer/Tokenizer";
76
import { HoverContentBuilder } from "./Builders";
87
import { Document } from "../Documents";
98
import Provider from "./Provider";
@@ -40,36 +39,37 @@ export default class HoverContentProvider extends Provider {
4039
let tokens;
4140
let token: ComplexToken | undefined;
4241

43-
const { tokenType, structVariableIdentifier, identifier } = this.server.tokenizer.findActionTargetAtPosition(liveDocument.getText(), position);
44-
const localScope = this.server.tokenizer.tokenizeContent(liveDocument.getText(), TokenizedScope.local, 0, position.line);
42+
const [lines, rawTokenizedContent] = this.server.tokenizer.tokenizeContentToRaw(liveDocument.getText());
43+
const localScope = this.server.tokenizer.tokenizeContentFromRaw(lines, rawTokenizedContent, 0, position.line);
44+
const { tokenType, lookBehindRawContent, rawContent } = this.server.tokenizer.getActionTargetAtPosition(lines, rawTokenizedContent, position);
4545

4646
switch (tokenType) {
4747
case CompletionItemKind.Function:
4848
case CompletionItemKind.Constant:
49-
token = localScope.functionsComplexTokens.find((candidate) => candidate.identifier === identifier);
49+
token = localScope.functionsComplexTokens.find((candidate) => candidate.identifier === rawContent);
5050
if (token) break;
5151

5252
tokens = document.getGlobalComplexTokens();
53-
token = tokens.find((candidate) => candidate.identifier === identifier);
53+
token = tokens.find((candidate) => candidate.identifier === rawContent);
5454
if (token) break;
5555

5656
tokens = this.server.documentsCollection.standardLibComplexTokens;
57-
token = tokens.find((candidate) => candidate.identifier === identifier);
57+
token = tokens.find((candidate) => candidate.identifier === rawContent);
5858
break;
5959
case CompletionItemKind.Struct:
6060
tokens = document.getGlobalStructComplexTokens();
61-
token = tokens.find((candidate) => candidate.identifier === identifier);
61+
token = tokens.find((candidate) => candidate.identifier === rawContent);
6262
break;
6363
case CompletionItemKind.Property:
64-
const structIdentifer = localScope?.functionVariablesComplexTokens.find((candidate) => candidate.identifier === structVariableIdentifier)?.valueType;
64+
const structIdentifer = localScope?.functionVariablesComplexTokens.find((candidate) => candidate.identifier === lookBehindRawContent)?.valueType;
6565

6666
token = document
6767
.getGlobalStructComplexTokens()
6868
.find((candidate) => candidate.identifier === structIdentifer)
69-
?.properties.find((property) => property.identifier === identifier);
69+
?.properties.find((property) => property.identifier === rawContent);
7070
break;
7171
default:
72-
token = localScope.functionVariablesComplexTokens.find((candidate) => candidate.identifier === identifier);
72+
token = localScope.functionVariablesComplexTokens.find((candidate) => candidate.identifier === rawContent);
7373
}
7474

7575
return token;

‎server/src/Providers/SignatureHelpProvider.ts

Lines changed: 10 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { SignatureHelpParams } from "vscode-languageserver/node";
22

33
import type { ServerManager } from "../ServerManager";
4-
import type { ComplexToken, FunctionComplexToken } from "../Tokenizer/types";
4+
import type { FunctionComplexToken } from "../Tokenizer/types";
55
import { LanguageScopes } from "../Tokenizer/constants";
66
import { TokenizedScope } from "../Tokenizer/Tokenizer";
77
import { SignatureHelpBuilder } from "./Builders";
@@ -19,44 +19,26 @@ export default class SignatureHelpProvider extends Provider {
1919
const {
2020
textDocument: { uri },
2121
position,
22-
context,
2322
} = params;
2423

2524
const liveDocument = this.server.liveDocumentsManager.get(uri);
2625
const document = this.server.documentsCollection.getFromUri(uri);
2726
if (!liveDocument || !document) return;
2827

29-
const tokenizedResult = this.server.tokenizer.isInLanguageScope(liveDocument.getText(), position, LanguageScopes.functionCall);
30-
if (!tokenizedResult) return;
28+
const [lines, rawTokenizedContent] = this.server.tokenizer.tokenizeContentToRaw(liveDocument.getText());
29+
const line = lines[position.line];
30+
const tokensArray = rawTokenizedContent[position.line];
3131

32-
const functionIdentifier = this.server.tokenizer.findIdentiferFromPositionForLanguageScopes(tokenizedResult.line, tokenizedResult.tokensArray, position, [
33-
LanguageScopes.functionCall,
34-
LanguageScopes.functionIdentifier,
35-
]);
36-
const activeParameter = this.server.tokenizer.getLanguageScopeOccurencesFromPositionWithDelimiter(
37-
tokenizedResult.tokensArray,
38-
position,
39-
LanguageScopes.separatorStatement,
40-
LanguageScopes.leftArgumentsRoundBracket,
41-
);
42-
43-
if (context?.isRetrigger && context.activeSignatureHelp) {
44-
const { activeSignature, signatures } = context?.activeSignatureHelp;
32+
if (!tokensArray || !this.server.tokenizer.isInScope(tokensArray, position, LanguageScopes.functionCall)) return;
4533

46-
if (activeSignature && functionIdentifier === this.server.tokenizer.findFirstIdentiferForLanguageScope(signatures[activeSignature].label, LanguageScopes.functionIdentifier)) {
47-
return {
48-
signatures,
49-
activeSignature,
50-
activeParameter: activeParameter || null,
51-
};
52-
}
53-
}
34+
const rawContent = this.server.tokenizer.getLookBehindScopesRawContent(line, tokensArray, position, [LanguageScopes.functionCall, LanguageScopes.functionIdentifier]);
35+
const activeParameter = this.server.tokenizer.getLookBehindScopeOccurences(tokensArray, position, LanguageScopes.separatorStatement, LanguageScopes.leftArgumentsRoundBracket);
5436

5537
const localScope = this.server.tokenizer.tokenizeContent(liveDocument.getText(), TokenizedScope.local, 0, position.line);
5638
const functionComplexToken =
57-
localScope.functionsComplexTokens.find((token) => token.identifier === functionIdentifier) ||
58-
document.getGlobalComplexTokens().find((token) => token.identifier === functionIdentifier) ||
59-
this.getStandardLibComplexTokens().find((token) => token.identifier === functionIdentifier);
39+
localScope.functionsComplexTokens.find((token) => token.identifier === rawContent) ||
40+
document.getGlobalComplexTokens().find((token) => token.identifier === rawContent) ||
41+
this.getStandardLibComplexTokens().find((token) => token.identifier === rawContent);
6042

6143
if (functionComplexToken) {
6244
return SignatureHelpBuilder.buildFunctionItem(functionComplexToken as FunctionComplexToken, activeParameter);

0 commit comments

Comments
 (0)