feat(vscode): python syntax highlightning in IF and WHILE statements

implements a part of #230

Author: d-biehl

packages/language_server/src/robotcode/language_server/robotframework/parts/semantic_tokens.py

@@ -403,13 +403,14 @@ def generate_sem_sub_tokens(
                     col_offset = token.col_offset
 
                 for g in cls.ESCAPE_REGEX.finditer(token.value):
-                    yield SemTokenInfo.from_token(
-                        token,
-                        sem_type if g.group("x") is None else RobotSemTokenTypes.ESCAPE,
-                        sem_mod,
-                        col_offset + g.start(),
-                        g.end() - g.start(),
-                    )
+                    if g.group("x") is not None:
+                        yield SemTokenInfo.from_token(
+                            token,
+                            sem_type if g.group("x") is None else RobotSemTokenTypes.ESCAPE,
+                            sem_mod,
+                            col_offset + g.start(),
+                            g.end() - g.start(),
+                        )
             elif token.type in [Token.KEYWORD, ROBOT_KEYWORD_INNER] or (
                 token.type == Token.NAME and isinstance(node, (Fixture, Template, TestTemplate))
             ):
@@ -614,7 +615,8 @@ def generate_sem_sub_tokens(
                     1,
                 )
             else:
-                yield SemTokenInfo.from_token(token, sem_type, sem_mod, col_offset, length)
+                if token.type != Token.ARGUMENT:
+                    yield SemTokenInfo.from_token(token, sem_type, sem_mod, col_offset, length)
 
     def generate_sem_tokens(
         self,

syntaxes/robotframework.tmLanguage.json

@@ -111,6 +111,48 @@
 				{ "include": "#line_continuation" }
 			]
 		},
+		"control_flow_expression_statements": {
+			"name": "meta.keyword-call.content.robotframework",
+			"contentName": "string.unquoted.argument.robotframework",
+			"begin": "^(?!(?: {2,}| ?\\t ?)+(?:(?=[$\\[@&%]|\\.)))(?: {2,}| ?\\t ?)+(IF|ELSE IF|WHILE)(( {2,}| ?\\t)( *)(.*?))?(?= {2,}| ?\\t ?| ?$)",
+			"end": "^(?!\\s*\\.\\.\\.)",
+			"beginCaptures": {
+				"1": { "name": "keyword.control.flow.robotframework" },
+				"5": {
+					"name": "meta.python-expression.robotframework",
+					"patterns": [ { "include": "#expression" } ]
+				}
+			},
+			"patterns": [
+				{ "include": "#comment" },
+				{ "include": "#variables" },
+				{ "include": "#line_continuation" }
+			]
+		},
+		"control_flow_statements": {
+			"name": "meta.keyword-call.content.robotframework",
+			"contentName": "string.unquoted.argument.robotframework",
+			"begin": "^(?!(?: {2,}| ?\\t ?)+(?:(?=[$\\[@&%]|\\.)))(?: {2,}| ?\\t ?)+(FOR|END|ELSE|TRY|EXCEPT|FINALLY)(?= {2,}| ?\\t ?| ?$)",
+			"end": "^(?!\\s*\\.\\.\\.)",
+			"beginCaptures": { "1": { "name": "keyword.control.flow.robotframework" } },
+			"patterns": [
+				{ "include": "#comment" },
+				{ "include": "#variables" },
+				{ "include": "#line_continuation" }
+			]
+		},
+		"other_statements": {
+			"name": "meta.keyword-call.content.robotframework",
+			"contentName": "string.unquoted.argument.robotframework",
+			"begin": "^(?!(?: {2,}| ?\\t ?)+(?:(?=[$\\[@&%]|\\.)))(?: {2,}| ?\\t ?)+(VAR)(?= {2,}| ?\\t ?| ?$)",
+			"end": "^(?!\\s*\\.\\.\\.)",
+			"beginCaptures": { "1": { "name": "keyword.robotframework" } },
+			"patterns": [
+				{ "include": "#comment" },
+				{ "include": "#variables" },
+				{ "include": "#line_continuation" }
+			]
+		},
 		"returning_keyword_call": {
 			"name": "meta.returning-keyword-call.robotframework",
 			"contentName": "meta.returning.keyword.content.robotframework",
@@ -2278,6 +2320,9 @@
 		{ "include": "#variable_setting" },
 		{ "include": "#setting" },
 		{ "include": "#testcase_settings" },
+		{ "include": "#control_flow_expression_statements" },
+		{ "include": "#control_flow_statements" },
+		{ "include": "#other_statements" },
 		{ "include": "#keyword_call" },
 		{ "include": "#returning_keyword_call" }
 	],

tests/robotcode/language_server/robotframework/parts/_regtest_outputs/rf41/test_semantic_tokens.test[bddstyle.robot].out

@@ -245,21 +245,6 @@ result: !SemanticTokens
   - 2
   - 39
   - 0
-  - 0
-  - 6
-  - 1
-  - 36
-  - 0
-  - 0
-  - 9
-  - 1
-  - 36
-  - 0
-  - 0
-  - 9
-  - 1
-  - 36
-  - 0
   - 1
   - 0
   - 10
@@ -270,21 +255,6 @@ result: !SemanticTokens
   - 2
   - 39
   - 0
-  - 0
-  - 6
-  - 1
-  - 36
-  - 0
-  - 0
-  - 9
-  - 1
-  - 36
-  - 0
-  - 0
-  - 9
-  - 1
-  - 36
-  - 0
   - 1
   - 0
   - 10
@@ -295,19 +265,4 @@ result: !SemanticTokens
   - 2
   - 39
   - 0
-  - 0
-  - 6
-  - 1
-  - 36
-  - 0
-  - 0
-  - 9
-  - 1
-  - 36
-  - 0
-  - 0
-  - 9
-  - 2
-  - 36
-  - 0
   result_id: null