Produce tokens for test-file.ps1

Author: vors

tests/__init__.py

@@ -43,10 +43,62 @@ def getTokenHeader(self):
         return ['scope_name', 'str', 'start', 'end']
 
     def getTokens(self):
-        selectors = ["comment", "constant", "entity", "interpolated", "keyword", "meta", "punctuation", "source", "storage", "string", "support", "variable"]
+        selectors = [
+            "comment.block",
+            "comment.documentation.embedded",
+            "comment.line.number-sign",
+            "constant.character.escape",
+            "constant.language",
+            "constant.numeric.hexadecimal",
+            "constant.numeric.scientific",
+            "constant.string.documentation",
+            "entity.name",
+            "entity.name.function.invocation",
+            "entity.other",
+            "entity.other.attribute-name",
+            "entity.other.attribute.parameter",
+            "interpolated.complex.source",
+            "interpolated.simple.source",
+            "keyword.control",
+            "keyword.operator.assignment",
+            "keyword.operator.bitwise",
+            "keyword.operator.comparison",
+            "keyword.operator.documentation",
+            "keyword.operator.logical",
+            "keyword.operator.math",
+            "keyword.operator.other",
+            "keyword.operator.range",
+            "keyword.operator.redirection",
+            "keyword.operator.string-format",
+            "keyword.operator.unary",
+            "keyword.other",
+            "keyword.other.statement-separator",
+            "meta",
+            "meta.group.array-expression",
+            "meta.group.complex.subexpression",
+            "meta.scriptblock",
+            "punctuation.end.definition.comment.block",
+            "punctuation.start.definition.comment.block",
+            "source",
+            "storage",
+            "storage.modifier.scope",
+            "string.quoted.double",
+            "string.quoted.double.heredoc",
+            "string.quoted.single",
+            "string.quoted.single.heredoc",
+            "support.constant",
+            "support.constant.variable",
+            "support.function",
+            "support.variable.automatic",
+            "support.variable.drive",
+            "variable.language",
+            "variable.other",
+            "variable.other.normal",
+            "variable.other.readwrite",
+        ]
         tokens = []
         for selector in selectors:
             regions = self.view.find_by_selector(selector)
             for region in regions:
-                tokens += [{ 'scope_name': selector, 'str': 'foo', 'start': region.a, 'end': region.b }]
+                tokens += [{ 'scope_name': selector, 'str': self.view.substr(region), 'start': region.a, 'end': region.b }]
         return tokens

tests/syntax_def/test_token_gen.py

@@ -1,16 +1,19 @@
 import os
 import sublime
-import csv
+import json
 
 from PowerShell.tests import PowerShellSyntaxTokenTest
 
 class Test_TokenGenerator(PowerShellSyntaxTokenTest):
     def testGetTokens(self):
 
-        self.append("""$foo = @'
-hello
-'@
-""")
+        test_path = os.path.join(sublime.packages_path(), 'PowerShell', 'tests', 'samples', 'test-file.ps1')
+
+        with open(test_path) as f:
+            content = f.readlines()
+            for line in content:
+                self.append(line)
+
         tokens = self.getTokens()
 
         outputdir = os.path.join(sublime.packages_path(), 'User', 'UnitTesting', "tokens")
@@ -21,9 +24,6 @@ def testGetTokens(self):
         if os.path.exists(outfile):
             os.remove(outfile)
         with open(outfile, 'w') as f:
-            header = self.getTokenHeader()
-            writer = csv.DictWriter(f, header, delimiter=',')
-            writer.writeheader()
-            writer.writerows(tokens)
+            f.write(json.dumps(tokens, indent=4, separators=(',', ': ')))
+            f.write(test_path)
 
-