implement semantic tokens

Author: d-biehl

package.json

@@ -39,13 +39,81 @@
     "onDebugResolve:robotframework",
     "onLanguage:robotframework"
   ],
-  "preview": true,
   "galleryBanner": {
     "theme": "dark",
-    "color": "#000000"
+    "color": "#111111"
   },
   "main": "./out/extension.js",
   "contributes": {
+    "configurationDefaults": {
+      "[robotframework]": {
+        "editor.semanticHighlighting.enabled": true
+      }
+    },
+    "semanticTokenScopes": [
+      {
+        "language": "robotframework",
+        "scopes": {
+          "header": [
+            "keyword.other.header.robotframework"
+          ],
+          "headerComment": [
+            "keyword.other.header.comment.robotframework"
+          ],
+          "headerKeyword": [
+            "keyword.other.header.keyword.robotframework"
+          ],
+          "headerSetting": [
+            "keyword.other.header.setting.robotframework"
+          ],
+          "headerVariable": [
+            "keyword.other.header.variable.robotframework"
+          ],
+          "headerTestcase": [
+            "keyword.other.header.testcase.robotframework"
+          ],
+          "setting": [
+            "keyword.control.settings.robotframework"
+          ],
+          "settingImport": [
+            "keyword.control.import.robotframework"
+          ],
+          "testcaseName": [
+            "entity.name.function.testcase.name.robotframework"
+          ],
+          "keywordName": [
+            "entity.name.function.keyword.name.robotframework"
+          ],
+          "controlFlow": [
+            "keyword.control.flow.robotframework"
+          ],
+          "forSeparator": [
+            "keyword.operator.for.robotframework"
+          ],
+          "argument": [
+            "string.unquoted.argument.robotframework"
+          ],
+          "variable": [
+            "variable.other.readwrite.robotframework"
+          ],
+          "keywordCall": [
+            "meta.function-call.keyword.robotframework"
+          ],
+          "nameCall": [
+            "meta.function-call.name.robotframework"
+          ],
+          "continuation": [
+            "punctuation.separator.continuation.robotframework"
+          ],
+          "separator": [
+            "punctuation.separator.robotframework"
+          ],
+          "terminator": [
+            "punctuation.terminator.robotframework"
+          ]
+        }
+      }
+    ],
     "languages": [
       {
         "id": "robotframework",
@@ -418,7 +486,7 @@
   "devDependencies": {
     "@types/glob": "^7.1.4",
     "@types/mocha": "^9.0.0",
-    "@types/node": "^16.9.4",
+    "@types/node": "^16.9.6",
     "@types/vscode": "^1.60.0",
     "@typescript-eslint/eslint-plugin": "^4.31.2",
     "@typescript-eslint/parser": "^4.31.2",
@@ -434,13 +502,13 @@
     "eslint-plugin-promise": "^5.1.0",
     "eslint-plugin-react": "^7.26.0",
     "eslint-plugin-react-hooks": "^4.2.0",
-    "glob": "^7.1.7",
+    "glob": "^7.2.0",
     "mocha": "^9.1.1",
     "prettier": "^2.4.1",
     "replace-in-file": "^6.2.0",
     "ts-loader": "^9.2.6",
     "typescript": "^4.4.3",
-    "vsce": "^1.99.0",
+    "vsce": "^1.100.0",
     "vscode-debugadapter-testsupport": "^1.49.0",
     "vscode-dts": "^0.3.1",
     "vscode-test": "^1.6.1",

robotcode/language_server/common/parts/semantic_tokens.py

@@ -70,10 +70,10 @@ def extend_capabilities(self, capabilities: ServerCapabilities) -> None:
                     token_types=[e.value for e in self.token_types],
                     token_modifiers=[e.value for e in self.token_modifiers],
                 ),
-                full=SemanticTokensOptionsFull(delta=len(self.collect_full_delta) > 0)
+                full=SemanticTokensOptionsFull(delta=True if len(self.collect_full_delta) > 0 else False)
                 if len(self.collect_full)
-                else None,
-                range=SemanticTokensOptionsRange() if len(self.collect_range) else None,
+                else False,
+                range=SemanticTokensOptionsRange() if len(self.collect_range) else False,
             )
 
     @rpc_method(name="textDocument/semanticTokens/full", param_type=SemanticTokensParams)

robotcode/language_server/common/types.py

@@ -1422,6 +1422,7 @@ class SemanticTokensRangeParams(WorkDoneProgressParams, PartialResultParams):
 
 
 class SemanticTokenTypes(Enum):
+    NAMESPACE = "namespace"
     TYPE = "type"
     CLASS = "class"
     ENUM = "enum"

robotcode/language_server/robotframework/diagnostics/namespace.py

@@ -33,8 +33,7 @@
     Position,
     Range,
 )
-from ..utils.ast import Token as AstToken
-from ..utils.ast import is_non_variable_token, range_from_token_or_node
+from ..utils.ast import Token, is_non_variable_token, range_from_token_or_node
 from ..utils.async_ast import AsyncVisitor
 from .imports_manager import ImportsManager
 from .library_doc import (
@@ -61,13 +60,6 @@ class ImportError(DiagnosticsError):
     pass
 
 
-@dataclass
-class Token:
-    line_no: int
-    col_offset: int
-    end_col_offset: int
-
-
 @dataclass
 class Import:
     name: Optional[str]
@@ -81,11 +73,11 @@ class Import:
     def range(self) -> Range:
         return Range(
             start=Position(
-                line=self.name_token.line_no - 1 if self.name_token is not None else self.line_no - 1,
+                line=self.name_token.lineno - 1 if self.name_token is not None else self.line_no - 1,
                 character=self.name_token.col_offset if self.name_token is not None else self.col_offset,
             ),
             end=Position(
-                line=self.name_token.line_no - 1 if self.name_token is not None else self.end_line_no - 1,
+                line=self.name_token.lineno - 1 if self.name_token is not None else self.end_line_no - 1,
                 character=self.name_token.end_col_offset if self.name_token is not None else self.end_col_offset,
             ),
         )
@@ -171,9 +163,7 @@ async def visit_LibraryImport(self, node: ast.AST) -> None:  # noqa: N802
         self._results.append(
             LibraryImport(
                 name=n.name,
-                name_token=Token(line_no=name.lineno, col_offset=name.col_offset, end_col_offset=name.end_col_offset)
-                if name is not None
-                else None,
+                name_token=name if name is not None else None,
                 args=n.args,
                 alias=n.alias,
                 line_no=node.lineno,
@@ -194,9 +184,7 @@ async def visit_ResourceImport(self, node: ast.AST) -> None:  # noqa: N802
         self._results.append(
             ResourceImport(
                 name=n.name,
-                name_token=Token(line_no=name.lineno, col_offset=name.col_offset, end_col_offset=name.end_col_offset)
-                if name is not None
-                else None,
+                name_token=name if name is not None else None,
                 line_no=node.lineno,
                 col_offset=node.col_offset,
                 end_line_no=node.end_lineno if node.end_lineno is not None else -1,
@@ -217,9 +205,7 @@ async def visit_VariablesImport(self, node: ast.AST) -> None:  # noqa: N802
         self._results.append(
             VariablesImport(
                 name=n.name,
-                name_token=Token(line_no=name.lineno, col_offset=name.col_offset, end_col_offset=name.end_col_offset)
-                if name is not None
-                else None,
+                name_token=name if name is not None else None,
                 args=n.args,
                 line_no=node.lineno,
                 col_offset=node.col_offset,
@@ -244,8 +230,8 @@ async def _analyze_keyword_call(
         self,
         keyword: Optional[str],
         value: ast.AST,
-        keyword_token: AstToken,
-        argument_tokens: List[AstToken],
+        keyword_token: Token,
+        argument_tokens: List[Token],
         analyse_run_keywords: bool = True,
     ) -> Optional[KeywordDoc]:
         result: Optional[KeywordDoc] = None
@@ -351,8 +337,8 @@ async def _analyze_keyword_call(
         return result
 
     async def _analyse_run_keyword(
-        self, keyword_doc: Optional[KeywordDoc], node: ast.AST, argument_tokens: List[AstToken]
-    ) -> List[AstToken]:
+        self, keyword_doc: Optional[KeywordDoc], node: ast.AST, argument_tokens: List[Token]
+    ) -> List[Token]:
 
         if keyword_doc is None or not keyword_doc.is_any_run_keyword():
             return argument_tokens
@@ -445,11 +431,13 @@ async def visit_Fixture(self, node: ast.AST) -> None:  # noqa: N802
         from robot.parsing.model.statements import Fixture
 
         value = cast(Fixture, node)
-        keyword_token = cast(AstToken, value.get_token(RobotToken.NAME))
+        keyword_token = cast(Token, value.get_token(RobotToken.NAME))
+
+        # TODO: calculate possible variables in NAME
 
-        if keyword_token is not None:
+        if keyword_token is not None and is_non_variable_token(keyword_token):
             await self._analyze_keyword_call(
-                value.name, value, keyword_token, [cast(AstToken, e) for e in value.get_tokens(RobotToken.ARGUMENT)]
+                value.name, value, keyword_token, [cast(Token, e) for e in value.get_tokens(RobotToken.ARGUMENT)]
             )
 
         await self.generic_visit(node)
@@ -459,9 +447,11 @@ async def visit_TestTemplate(self, node: ast.AST) -> None:  # noqa: N802
         from robot.parsing.model.statements import TestTemplate
 
         value = cast(TestTemplate, node)
-        keyword_token = cast(AstToken, value.get_token(RobotToken.NAME))
+        keyword_token = cast(Token, value.get_token(RobotToken.NAME))
 
-        if keyword_token is not None:
+        # TODO: calculate possible variables in NAME
+
+        if keyword_token is not None and is_non_variable_token(keyword_token):
             await self._analyze_keyword_call(value.value, value, keyword_token, [])
 
         await self.generic_visit(node)
@@ -471,9 +461,11 @@ async def visit_Template(self, node: ast.AST) -> None:  # noqa: N802
         from robot.parsing.model.statements import Template
 
         value = cast(Template, node)
-        keyword_token = cast(AstToken, value.get_token(RobotToken.NAME))
+        keyword_token = cast(Token, value.get_token(RobotToken.NAME))
+
+        # TODO: calculate possible variables in NAME
 
-        if keyword_token is not None:
+        if keyword_token is not None and is_non_variable_token(keyword_token):
             await self._analyze_keyword_call(value.value, value, keyword_token, [])
 
         await self.generic_visit(node)
@@ -497,7 +489,7 @@ async def visit_KeywordCall(self, node: ast.AST) -> None:  # noqa: N802
             )
         else:
             await self._analyze_keyword_call(
-                value.keyword, value, keyword_token, [cast(AstToken, e) for e in value.get_tokens(RobotToken.ARGUMENT)]
+                value.keyword, value, keyword_token, [cast(Token, e) for e in value.get_tokens(RobotToken.ARGUMENT)]
             )
 
         if not self.current_testcase_or_keyword_name: