implement highligth escape sequences, implement semantic highlight collect_range

d-biehl · d-biehl · commit d7e89ed6f951 · 2021-09-24T15:55:13.000+02:00
diff --git a/package.json b/package.json
@@ -110,6 +110,15 @@
           ],
           "terminator": [
             "punctuation.terminator.robotframework"
+          ],
+          "variableBegin": [
+            "punctuation.definition.variable.begin.robotframework"
+          ],
+          "variableEnd": [
+            "punctuation.definition.variable.end.robotframework"
+          ],
+          "escape": [
+            "constant.character.escape.robotframework"
           ]
         }
       }
@@ -128,7 +137,7 @@
         "configuration": "./language-configuration.json"
       }
     ],
-    "grammars": [
+    "#grammars": [
       {
         "language": "robotframework",
         "scopeName": "source.robotframework",
diff --git a/robotcode/language_server/common/parts/semantic_tokens.py b/robotcode/language_server/common/parts/semantic_tokens.py
@@ -20,7 +20,6 @@
     SemanticTokensLegend,
     SemanticTokensOptions,
     SemanticTokensOptionsFull,
-    SemanticTokensOptionsRange,
     SemanticTokensParams,
     SemanticTokensPartialResult,
     SemanticTokensRangeParams,
@@ -70,10 +69,12 @@ def extend_capabilities(self, capabilities: ServerCapabilities) -> None:
                     token_types=[e.value for e in self.token_types],
                     token_modifiers=[e.value for e in self.token_modifiers],
                 ),
-                full=SemanticTokensOptionsFull(delta=True if len(self.collect_full_delta) > 0 else False)
+                full=SemanticTokensOptionsFull(delta=True if len(self.collect_full_delta) else None)
+                if len(self.collect_full) and len(self.collect_full_delta)
+                else True
                 if len(self.collect_full)
-                else False,
-                range=SemanticTokensOptionsRange() if len(self.collect_range) else False,
+                else None,
+                range=True if len(self.collect_range) else None,
             )
 
     @rpc_method(name="textDocument/semanticTokens/full", param_type=SemanticTokensParams)
diff --git a/robotcode/language_server/robotframework/parts/semantic_tokens.py b/robotcode/language_server/robotframework/parts/semantic_tokens.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import operator
+import re
 from enum import Enum
 from functools import reduce
 from typing import (
@@ -20,12 +21,15 @@
 from ...common.language import language_id
 from ...common.text_document import TextDocument
 from ...common.types import (
+    Range,
     SemanticTokenModifiers,
     SemanticTokens,
+    SemanticTokensDelta,
+    SemanticTokensDeltaPartialResult,
     SemanticTokensPartialResult,
     SemanticTokenTypes,
 )
-from ..utils.ast import Token
+from ..utils.ast import Token, token_in_range
 
 if TYPE_CHECKING:
     from ..protocol import RobotLanguageServerProtocol
@@ -54,6 +58,9 @@ class RobotSemTokenTypes(Enum):
     SEPARATOR = "separator"
     TERMINATOR = "terminator"
     FOR_SEPARATOR = "forSeparator"
+    VARIABLE_BEGIN = "variableBegin"
+    VARIABLE_END = "variableEnd"
+    ESCAPE = "escape"
 
 
 class SemTokenInfo(NamedTuple):
@@ -64,9 +71,20 @@ class SemTokenInfo(NamedTuple):
     sem_modifiers: Optional[Set[Enum]] = None
 
     @classmethod
-    def from_token(cls, token: Token, sem_token_type: Enum, sem_modifiers: Optional[Set[Enum]] = None) -> SemTokenInfo:
+    def from_token(
+        cls,
+        token: Token,
+        sem_token_type: Enum,
+        sem_modifiers: Optional[Set[Enum]] = None,
+        col_offset: Optional[int] = None,
+        length: Optional[int] = None,
+    ) -> SemTokenInfo:
         return cls(
-            token.lineno, token.col_offset, token.end_col_offset - token.col_offset, sem_token_type, sem_modifiers
+            token.lineno,
+            col_offset if col_offset is not None else token.col_offset,
+            length if length is not None else token.end_col_offset - token.col_offset,
+            sem_token_type,
+            sem_modifiers,
         )
 
 
@@ -77,6 +95,8 @@ def __init__(self, parent: RobotLanguageServerProtocol) -> None:
         super().__init__(parent)
         parent.semantic_tokens.token_types += [e for e in RobotSemTokenTypes]
         parent.semantic_tokens.collect_full.add(self.collect_full)
+        parent.semantic_tokens.collect_range.add(self.collect_range)
+        parent.semantic_tokens.collect_full_delta.add(self.collect_full_delta)
 
     @classmethod
     def generate_mapping(cls) -> Dict[str, Tuple[Enum, Optional[Set[Enum]]]]:
@@ -139,6 +159,45 @@ def mapping(cls) -> Dict[str, Tuple[Enum, Optional[Set[Enum]]]]:
             cls.__mapping = cls.generate_mapping()
         return cls.__mapping
 
+    ESCAPE_REGEX = re.compile(
+        r"(?P<t>[^\\]+)|(?P<x>\\([^xuU]|x[0-f]{2}|u[0-f]{4}|U[0-f]{8}){0,1})", re.MULTILINE | re.DOTALL
+    )
+
+    @classmethod
+    async def generate_sem_sub_tokens(
+        cls, token: Token, col_offset: Optional[int] = None, length: Optional[int] = None
+    ) -> AsyncGenerator[SemTokenInfo, None]:
+        from robot.parsing.lexer.tokens import Token as RobotToken
+
+        sem_info = cls.mapping().get(token.type, None) if token.type is not None else None
+
+        if sem_info is not None:
+            if token.type == RobotToken.VARIABLE:
+                if col_offset is None:
+                    col_offset = token.col_offset
+                if length is None:
+                    length = token.end_col_offset - token.col_offset
+
+                yield SemTokenInfo(token.lineno, col_offset, 2, RobotSemTokenTypes.VARIABLE_BEGIN)
+                yield SemTokenInfo.from_token(token, sem_info[0], sem_info[1], col_offset + 2, length - 3)
+                yield SemTokenInfo(token.lineno, col_offset + length - 1, 1, RobotSemTokenTypes.VARIABLE_END)
+            if token.type == RobotToken.ARGUMENT and "\\" in token.value:
+                if col_offset is None:
+                    col_offset = token.col_offset
+                if length is None:
+                    length = token.end_col_offset - token.col_offset
+
+                for g in cls.ESCAPE_REGEX.finditer(token.value):
+                    yield SemTokenInfo.from_token(
+                        token,
+                        sem_info[0] if g.group("x") is None or g.end() - g.start() == 1 else RobotSemTokenTypes.ESCAPE,
+                        sem_info[1],
+                        col_offset + g.start(),
+                        g.end() - g.start(),
+                    )
+            else:
+                yield SemTokenInfo.from_token(token, sem_info[0], sem_info[1], col_offset, length)
+
     @classmethod
     async def generate_sem_tokens(cls, token: Token) -> AsyncGenerator[SemTokenInfo, None]:
         from robot.parsing.lexer.tokens import Token as RobotToken
@@ -148,44 +207,46 @@ async def generate_sem_tokens(cls, token: Token) -> AsyncGenerator[SemTokenInfo,
             try:
                 for sub_token in token.tokenize_variables():
                     last_sub_token = sub_token
-                    if sub_token.type is not None:
-                        sem_info = cls.mapping().get(sub_token.type, None)
-                        if sem_info is not None:
-                            yield SemTokenInfo.from_token(sub_token, sem_info[0], sem_info[1])
+                    async for e in cls.generate_sem_sub_tokens(sub_token):
+                        yield e
             except BaseException:
                 pass
-            if last_sub_token == token and token.type is not None:
-                sem_info = cls.mapping().get(token.type, None)
-                if sem_info is not None:
-                    yield SemTokenInfo.from_token(token, sem_info[0], sem_info[1])
+            if last_sub_token == token:
+                async for e in cls.generate_sem_sub_tokens(last_sub_token):
+                    yield e
             elif last_sub_token is not None and last_sub_token.end_col_offset < token.end_col_offset:
-                if token.type is not None:
-                    sem_info = cls.mapping().get(token.type, None)
-                    if sem_info is not None:
-                        yield SemTokenInfo(
-                            token.lineno,
-                            last_sub_token.end_col_offset,
-                            token.end_col_offset - last_sub_token.end_col_offset - last_sub_token.col_offset,
-                            sem_info[0],
-                            sem_info[1],
-                        )
-
-        elif token.type is not None:
-            sem_info = cls.mapping().get(token.type, None)
-            if sem_info is not None:
-                yield SemTokenInfo.from_token(token, sem_info[0], sem_info[1])
+                async for e in cls.generate_sem_sub_tokens(
+                    token,
+                    last_sub_token.end_col_offset,
+                    token.end_col_offset - last_sub_token.end_col_offset - last_sub_token.col_offset,
+                ):
+                    yield e
 
-    @language_id("robotframework")
-    async def collect_full(
-        self, sender: Any, document: TextDocument, **kwargs: Any
+        else:
+            async for e in cls.generate_sem_sub_tokens(token):
+                yield e
+
+    async def collect(
+        self, document: TextDocument, range: Optional[Range]
     ) -> Union[SemanticTokens, SemanticTokensPartialResult, None]:
 
         data = []
         last_line = 0
         last_col = 0
 
         tokens = await self.parent.documents_cache.get_tokens(document)
+
+        start = True
         for robot_token in tokens:
+            if range is not None:
+                if start and not token_in_range(robot_token, range):
+                    continue
+                else:
+                    start = False
+
+                if not start and not token_in_range(robot_token, range):
+                    break
+
             async for token in self.generate_sem_tokens(robot_token):
                 current_line = token.lineno - 1
 
@@ -215,3 +276,21 @@ async def collect_full(
                 )
 
         return SemanticTokens(data=data)
+
+    @language_id("robotframework")
+    async def collect_full(
+        self, sender: Any, document: TextDocument, **kwargs: Any
+    ) -> Union[SemanticTokens, SemanticTokensPartialResult, None]:
+        return await self.collect(document, None)
+
+    @language_id("robotframework")
+    async def collect_range(
+        self, sender: Any, document: TextDocument, range: Range, **kwargs: Any
+    ) -> Union[SemanticTokens, SemanticTokensPartialResult, None]:
+        return await self.collect(document, range)
+
+    @language_id("robotframework")
+    async def collect_full_delta(
+        self, sender: Any, document: TextDocument, previous_result_id: str, **kwargs: Any
+    ) -> Union[SemanticTokens, SemanticTokensDelta, SemanticTokensDeltaPartialResult, None]:
+        return None
diff --git a/robotcode/language_server/robotframework/utils/ast.py b/robotcode/language_server/robotframework/utils/ast.py
@@ -85,6 +85,16 @@ def range_from_token(token: Token) -> Range:
     )
 
 
+def token_in_range(token: Token, range: Range) -> bool:
+    token_range = range_from_token(token)
+    return token_range.start.is_in_range(range) or token_range.end.is_in_range(range)
+
+
+def node_in_range(node: ast.AST, range: Range) -> bool:
+    node_range = range_from_node(node)
+    return node_range.start.is_in_range(range) or node_range.end.is_in_range(range)
+
+
 def range_from_token_or_node(node: ast.AST, token: Optional[Token]) -> Range:
     if token is not None:
         return range_from_token(token)
diff --git a/robotcode/utils/async_event.py b/robotcode/utils/async_event.py
@@ -222,27 +222,13 @@ def _get_name_prefix(descriptor: AsyncEventDescriptorBase[Any, Any, Any]) -> str
     return f"{descriptor._owner.__qualname__}.{descriptor._owner_name}"
 
 
-class async_tasking_event_iterator(  # noqa: N801
-    AsyncEventDescriptorBase[_TCallable, Any, AsyncTaskingEventIterator[_TCallable, Any]]
-):
-    def __init__(self, _func: _TCallable) -> None:
-        super().__init__(
-            _func, AsyncTaskingEventIterator[_TCallable, Any], task_name_prefix=lambda: _get_name_prefix(self)
-        )
-
-
 class AsyncTaskingEvent(AsyncTaskingEventResultIteratorBase[_TCallable, _TResult]):
     async def __call__(self, *args: Any, **kwargs: Any) -> List[Union[_TResult, BaseException]]:
         return [a async for a in self._notify(*args, **kwargs)]
 
 
-class async_tasking_event(AsyncEventDescriptorBase[_TCallable, Any, AsyncTaskingEvent[_TCallable, Any]]):  # noqa: N801
-    def __init__(self, _func: _TCallable) -> None:
-        super().__init__(_func, AsyncTaskingEvent[_TCallable, Any], task_name_prefix=lambda: _get_name_prefix(self))
-
-
 class AsyncThreadingEventResultIteratorBase(AsyncEventResultIteratorBase[_TCallable, _TResult]):
-    __executor: Optional[ThreadPoolExecutor]
+    __executor: Optional[ThreadPoolExecutor] = None
 
     def __init__(self, *, thread_name_prefix: Optional[str] = None) -> None:
         super().__init__()
@@ -308,13 +294,13 @@ def _done(f: asyncio.Future[_TResult]) -> None:
                     result_callback(None, e)
 
         if executor is None:
-            if self.__executor is None:
-                self.__executor = ThreadPoolExecutor(
+            if AsyncThreadingEventResultIteratorBase.__executor is None:
+                AsyncThreadingEventResultIteratorBase.__executor = ThreadPoolExecutor(
                     thread_name_prefix=self.__thread_name_prefix()
                     if callable(self.__thread_name_prefix)
                     else self.__thread_name_prefix
                 )
-            executor = self.__executor
+            executor = AsyncThreadingEventResultIteratorBase.__executor
 
         awaitables: List[asyncio.Future[_TResult]] = []
         for method in filter(
@@ -370,3 +356,21 @@ class async_threading_event(  # noqa: N801
 ):
     def __init__(self, _func: _TCallable) -> None:
         super().__init__(_func, AsyncThreadingEvent[_TCallable, Any], thread_name_prefix=lambda: _get_name_prefix(self))
+
+
+# async_tasking_event_iterator = async_threading_event_iterator
+# async_tasking_event = async_threading_event
+
+
+class async_tasking_event_iterator(  # noqa: N801
+    AsyncEventDescriptorBase[_TCallable, Any, AsyncTaskingEventIterator[_TCallable, Any]]
+):
+    def __init__(self, _func: _TCallable) -> None:
+        super().__init__(
+            _func, AsyncTaskingEventIterator[_TCallable, Any], task_name_prefix=lambda: _get_name_prefix(self)
+        )
+
+
+class async_tasking_event(AsyncEventDescriptorBase[_TCallable, Any, AsyncTaskingEvent[_TCallable, Any]]):  # noqa: N801
+    def __init__(self, _func: _TCallable) -> None:
+        super().__init__(_func, AsyncTaskingEvent[_TCallable, Any], task_name_prefix=lambda: _get_name_prefix(self))

Original file line number	Diff line number	Diff line change
`@@ -110,6 +110,15 @@`
`110`	`110`	`],`
`111`	`111`	`"terminator": [`
`112`	`112`	`"punctuation.terminator.robotframework"`
	`113`	`+ ],`
	`114`	`+ "variableBegin": [`
	`115`	`+ "punctuation.definition.variable.begin.robotframework"`
	`116`	`+ ],`
	`117`	`+ "variableEnd": [`
	`118`	`+ "punctuation.definition.variable.end.robotframework"`
	`119`	`+ ],`
	`120`	`+ "escape": [`
	`121`	`+ "constant.character.escape.robotframework"`
`113`	`122`	`]`
`114`	`123`	`}`
`115`	`124`	`}`
`@@ -128,7 +137,7 @@`
`128`	`137`	`"configuration": "./language-configuration.json"`
`129`	`138`	`}`
`130`	`139`	`],`
`131`		`- "grammars": [`
	`140`	`+ "#grammars": [`
`132`	`141`	`{`
`133`	`142`	`"language": "robotframework",`
`134`	`143`	`"scopeName": "source.robotframework",`