correctly find keyword with '.' in name, highlight variables in inner keywords, some renaming

Author: d-biehl

robotcode/language_server/robotframework/parts/references.py

@@ -39,10 +39,10 @@
     get_nodes_at_position,
     get_tokens_at_position,
     is_not_variable_token,
+    iter_over_keyword_names_and_owners,
     range_from_token,
     range_from_token_or_node,
     tokenize_variables,
-    yield_owner_and_kw_names,
 )
 from ..utils.async_ast import iter_nodes
 
@@ -356,7 +356,7 @@ async def get_keyword_references_from_tokens(
             kw_matcher = KeywordMatcher(kw_doc.name)
             kw_name = unescape(kw_token.value) if unescape_kw_token else kw_token.value
 
-            for lib, name in yield_owner_and_kw_names(kw_name):
+            for lib, name in iter_over_keyword_names_and_owners(kw_name):
                 if lib is not None:
                     lib_matcher = KeywordMatcher(lib)
                     if (

robotcode/language_server/robotframework/parts/semantic_tokens.py

@@ -48,9 +48,9 @@
     Token,
     is_not_variable_token,
     iter_nodes,
+    iter_over_keyword_names_and_owners,
     token_in_range,
     tokenize_variables,
-    yield_owner_and_kw_names,
 )
 
 if TYPE_CHECKING:
@@ -265,7 +265,7 @@ async def generate_sem_sub_tokens(
                 kw_namespace: Optional[str] = None
                 kw: str = token.value
 
-                for lib, name in yield_owner_and_kw_names(token.value):
+                for lib, name in iter_over_keyword_names_and_owners(token.value):
                     if lib is not None:
                         lib_matcher = KeywordMatcher(lib)
                         if (
@@ -319,10 +319,13 @@ async def generate_sem_tokens(
     ) -> AsyncGenerator[SemTokenInfo, None]:
         from robot.parsing.lexer.tokens import Token as RobotToken
 
-        if token.type in {*RobotToken.ALLOW_VARIABLES, RobotToken.KEYWORD}:
+        if token.type in {*RobotToken.ALLOW_VARIABLES, RobotToken.KEYWORD, ROBOT_KEYWORD_INNER}:
 
             for sub_token in tokenize_variables(
-                token, ignore_errors=True, identifiers="$" if token.type == RobotToken.KEYWORD_NAME else "$@&%"
+                token,
+                ignore_errors=True,
+                identifiers="$" if token.type == RobotToken.KEYWORD_NAME else "$@&%",
+                extra_types={ROBOT_KEYWORD_INNER},
             ):
                 async for e in self.generate_sem_sub_tokens(namespace, builtin_library_doc, sub_token, node):
                     yield e
@@ -535,7 +538,7 @@ async def get_tokens() -> AsyncGenerator[Tuple[Token, ast.AST], None]:
                         )
                         kw: Optional[str] = None
 
-                        for _, name in yield_owner_and_kw_names(kw_token.value):
+                        for _, name in iter_over_keyword_names_and_owners(kw_token.value):
                             if name is not None:
                                 matcher = KeywordMatcher(name)
                                 if matcher in ALL_RUN_KEYWORDS_MATCHERS:

robotcode/language_server/robotframework/utils/ast.py

@@ -9,6 +9,7 @@
     List,
     Optional,
     Protocol,
+    Set,
     Tuple,
     runtime_checkable,
 )
@@ -195,12 +196,17 @@ def _tokenize_no_variables(token: Token) -> Generator[Token, None, None]:
 
 
 def tokenize_variables(
-    token: Token, identifiers: str = "$@&%", ignore_errors: bool = False
+    token: Token, identifiers: str = "$@&%", ignore_errors: bool = False, *, extra_types: Optional[Set[str]] = None
 ) -> Generator[Token, Any, Any]:
     from robot.api.parsing import Token as RobotToken
     from robot.variables import VariableIterator
 
-    if token.type not in {*RobotToken.ALLOW_VARIABLES, RobotToken.KEYWORD, RobotToken.ASSIGN}:
+    if token.type not in {
+        *RobotToken.ALLOW_VARIABLES,
+        RobotToken.KEYWORD,
+        RobotToken.ASSIGN,
+        *(extra_types if extra_types is not None else set()),
+    }:
         return _tokenize_no_variables(token)
     variables = VariableIterator(token.value, identifiers=identifiers, ignore_errors=ignore_errors)
     if not variables:
@@ -224,7 +230,9 @@ def _tokenize_variables(token: Token, variables: Any) -> Generator[Token, Any, A
         yield RobotToken(token.type, remaining, lineno, col_offset)
 
 
-def yield_owner_and_kw_names(full_name: str) -> Iterator[Tuple[Optional[str], ...]]:
+def iter_over_keyword_names_and_owners(full_name: str) -> Iterator[Tuple[Optional[str], ...]]:
+    yield None, full_name
+
     tokens = full_name.split(".")
     if len(tokens) == 1:
         yield None, tokens[0]