add harness to produce tokens output

Author: vors

tests/__init__.py

@@ -32,6 +32,10 @@ def getScopeNameAtRowCol(self, row, col):
     def getFinestScopeNameAtRowCol(self, row, col):
         return self.getScopeNameAtRowCol(row, col).split()[-1]
 
+class SyntaxTokenTest(SyntaxTest):
+    def getTokens(self):
+        return self.view.find_all(".")
+
 
 class PowerShellSyntaxTest(SyntaxTest):
     def setUp(self):

tests/syntax_def/test_token_gen.py

@@ -0,0 +1,24 @@
+import os
+import sublime
+from PowerShell.tests import SyntaxTokenTest
+
+class Test_TokenGenerator(SyntaxTokenTest):
+    def testGetTokens(self):
+
+        self.append("""$foo = @'
+hello
+'@
+""")
+        tokens = self.getTokens()
+        
+        outputdir = os.path.join(sublime.packages_path(), 'User', 'UnitTesting', "tokens")
+        if not os.path.isdir(outputdir):
+            os.makedirs(outputdir)
+        outfile = os.path.join(outputdir, "PowerShell_tokens")
+
+        if os.path.exists(outfile):
+            os.remove(outfile)
+        with open(outfile, 'w') as f:
+            for r in tokens:
+                f.write('hi there\n') 
+