Merge pull request #72 from vors/syntax

vors · vors · commit f3610998e675 · 2015-01-21T15:13:31.000-08:00
Fix bugs for milestone 2.0+
diff --git a/Support/PowershellSyntax.tmLanguage b/Support/PowershellSyntax.tmLanguage
@@ -244,10 +244,29 @@
 		</dict>
 		<dict>
 			<key>match</key>
-			<string>(?&lt;!\w)((?i:begin|break|catch|class|continue|data|define|do|dynamicparam|else|elseif|end|exit|filter|finally|for|foreach(?!-object)|from|if|in|inlinescript|parallel|param|process|return|switch|throw|trap|try|until|using|var|where(?!=-object)|while|workflow)|%|\?)(?!\w)</string>
+			<string>(?&lt;!\w)((?i:begin|break|catch|continue|data|define|do|dynamicparam|else|elseif|end|exit|filter|finally|for|foreach(?!-object)|from|if|in|inlinescript|parallel|param|process|return|switch|throw|trap|try|until|using|var|where(?!=-object)|while)|%|\?)(?!\w)</string>
 			<key>name</key>
 			<string>keyword.control.powershell</string>
 		</dict>
+		<dict>
+			<key>captures</key>
+			<dict>
+				<key>1</key>
+				<dict>
+					<key>name</key>
+					<string>storage.type.powershell</string>
+				</dict>
+				<key>2</key>
+				<dict>
+					<key>name</key>
+					<string>entity.name.function</string>
+				</dict>
+			</dict>
+			<key>comment</key>
+			<string>capture should be entity.name.type, but it doesn't provide a good color in the default schema.</string>
+			<key>match</key>
+			<string>(?&lt;!\w)((?i:class)|%|\?)(?:\s)+((?:\p{L}|\d|_|-|)+)\b</string>
+		</dict>
 		<dict>
 			<key>match</key>
 			<string>(?&lt;!\w)-(?i:is(?:not)?|as)\b</string>
@@ -331,7 +350,7 @@
 				</dict>
 			</dict>
 			<key>end</key>
-			<string>(?&lt;=\))\]</string>
+			<string>\]</string>
 			<key>endCaptures</key>
 			<dict>
 				<key>0</key>
@@ -501,7 +520,7 @@
 		<key>function</key>
 		<dict>
 			<key>begin</key>
-			<string>(?i:function|configuration)\s+((?:\p{L}|\d|_|-)+)</string>
+			<string>((?i:function|configuration|workflow))\s+((?:\p{L}|\d|_|-|\.)+)</string>
 			<key>beginCaptures</key>
 			<dict>
 				<key>0</key>
@@ -692,17 +711,19 @@
 				<key>0</key>
 				<dict>
 					<key>name</key>
-					<string>entity.name.type</string>
+					<string>entity.other.attribute-name</string>
 				</dict>
 			</dict>
+			<key>comment</key>
+			<string>name should be entity.name.type but default schema doesn't have a good color for it</string>
 			<key>end</key>
 			<string>\]</string>
 			<key>endCaptures</key>
 			<dict>
 				<key>0</key>
 				<dict>
 					<key>name</key>
-					<string>entity.name.type</string>
+					<string>entity.other.attribute-name</string>
 				</dict>
 			</dict>
 			<key>patterns</key>
@@ -711,7 +732,7 @@
 					<key>match</key>
 					<string>(\p{L}|\.|``\d+)+?</string>
 					<key>name</key>
-					<string>entity.name.type</string>
+					<string>entity.other.attribute-name</string>
 				</dict>
 				<dict>
 					<key>include</key>
@@ -777,7 +798,7 @@
 						<key>2</key>
 						<dict>
 							<key>name</key>
-							<string>support.variable.automatic.powershell</string>
+							<string>support.constant.automatic.powershell</string>
 						</dict>
 						<key>3</key>
 						<dict>
@@ -786,7 +807,7 @@
 						</dict>
 					</dict>
 					<key>comment</key>
-					<string>Automatic variables are not constants, but they are read-only...</string>
+					<string>Automatic variables are not constants, but they are read-only. In monokai (default) color schema support.variable doesn't have color, so we use constant.</string>
 					<key>match</key>
 					<string>(\$)(?i:(\$|\^|\?|_|Args|ConsoleFileName|Event|EventArgs|EventSubscriber|ForEach|Input|LastExitCode|Matches|MyInvocation|NestedPromptLevel|Profile|PSBoundParameters|PsCmdlet|PsCulture|PSDebugContext|PSItem|PSCommandPath|PSScriptRoot|PsUICulture|Pwd|Sender|SourceArgs|SourceEventArgs|StackTrace|Switch|This))((?:\.(?:\p{L}|\d|_)+)*\b)?\b</string>
 				</dict>
diff --git a/appveyor.yml b/appveyor.yml
@@ -1,11 +1,12 @@
 install:
   - ps: write-host -ForegroundColor Yellow "installing sublime text 3"
   - ps: start-filedownload "http://c758482.r82.cf2.rackcdn.com/Sublime%20Text%20Build%203059%20x64.zip"
-  - ps: write-host -ForegroundColor Yellow "installing Sublime%20Text%20Build%203059%20x64.zip"
   - ps: 7z.exe x "Sublime%20Text%20Build%203059%20x64.zip" -o"C:\st" > $null
+  - ps: write-host -ForegroundColor Yellow "copying PowerShell package"
   - ps: mkdir "C:\st\Data\Packages" -force > $null
   - ps: mkdir "C:\st\Data\Packages\PowerShell" -force > $null
   - ps: cp -Recurse * "C:\st\Data\Packages\PowerShell" -Force
+  - ps: write-host -ForegroundColor Yellow "installing UnitTesting package"
   - ps: git clone -q --branch=master https://github.com/randy3k/UnitTesting.git "C:\st\Data\Packages\UnitTesting"
   - ps: write-host -ForegroundColor Yellow "installing pester"
   - ps: cinst pester
diff --git a/tests/pester/Syntax.Tests.ps1 b/tests/pester/Syntax.Tests.ps1
@@ -17,24 +17,28 @@ Describe "Syntax highlighting" {
         }
 
         # splitted in two lines, because of a bug in Sort-Object
-        $stScopes = cat -Raw $scopesFile | ConvertFrom-Json; $stScopes = $stScopes | sort -Property @('startOffset', 'endOffset')
+        $stScopes =  Get-SublimeScopesFromFile $scopesFile
         # tokens are already sorted
         $psScopes = Get-TokensFromFile $testFile | Convert-TokenToScope 
 
         It "split tokens across the scopes 0 times" {
             $stIndex = 0
             $errorCounter = 0
 
-            $psScopes | %{
-                while ($stScopes[$stIndex].endOffset -le $_.startOffset) {
+            foreach ($psScope in $psScopes) {
+                while ($stScopes[$stIndex].endOffset -le $psScope.startOffset) {
                     $stIndex++
                     if ($stIndex -ge $stScopes.Length) {
                         break
                     }
                 }
 
+                if ($stIndex -ge $stScopes.Length) {
+                    # we are done with sublime scopes
+                    break
+                }
+
                 $stScope = $stScopes[$stIndex]
-                $psScope = $_
 
                 #Write-Host "PowerShell scope $psScope"
                 #Write-Host "SublimeText scope $stScope"
@@ -49,6 +53,12 @@ Describe "Syntax highlighting" {
                             $ignore = $true
                         }
 
+                        if (@('7>') -contains $psScope.Text) {
+                            # this is fine. It's more like a powershell parsing problem.
+                            $ignore = $true
+                        }
+
+
                         # TODO: These are bugs
                         if (@('Number', 'Redirection') -contains $psScope.Kind) {
                             $ignore = $true    
@@ -61,8 +71,7 @@ Describe "Syntax highlighting" {
                     }
                 }
             }
-            # TODO: These are bugs, make it 0
-            $errorCounter | Should be @(0..2)
+            $errorCounter | Should be 0
         }
 
         It "produces same tokens for lower case" {
diff --git a/tests/pester/SyntaxHelper.psm1 b/tests/pester/SyntaxHelper.psm1
@@ -18,7 +18,6 @@ function Get-SublimeScopesFromFile
         [string] $filePath
     )
 
-    # splitted in two lines, because of a bug in Sort-Object
     $scopes = cat -Raw $filePath | ConvertFrom-Json
     $scopes = $scopes | sort -Property @('startOffset', 'endOffset')
     return $scopes
@@ -134,8 +133,21 @@ function Test-ScopesEqual
     if ($leftScope.startOffset -ne $rightScope.startOffset) {return $false} 
     if ($leftScope.endOffset -ne $rightScope.endOffset) {return $false}
     if ($leftScope.Text -ne $rightScope.Text) {return $false}
-    #TODO: this is week, need to verify that Kind is the same
-    #if ($leftScope.Kind.Split('.')[0] -ne $rightScope.Kind.Split('.')[0]) {return $false}
+    if ($leftScope.Kind -ne $rightScope.Kind) 
+    {
+        # TODO: what's that??
+        $bugsExclude = @('entity.name.function*', 'keyword.operator*', 'storage.type*', 'entity.other.attribute-name*', '*constant*')
+        foreach ($bug in $bugsExclude) 
+        {
+            if (($leftScope.Kind -like $bug) -and ($rightScope.Kind -like $bug)) 
+            { 
+                Write-Warning "[Fix me in SyntaxHelper.psm1] Ignore mistmatch in left scope $leftScope and right scope $rightScope"
+                return $true
+            }
+        }
+
+        return $false
+    }
     return $true
 }
 
diff --git a/tests/py/__init__.py b/tests/py/__init__.py
@@ -55,57 +55,63 @@ def getTestFilePath(self):
 
     def getTokens(self):
         selectors = [
-            "comment.block",
-            "comment.documentation.embedded",
-            "comment.line.number-sign",
-            "constant.character.escape",
-            "constant.language",
-            "constant.numeric.hexadecimal",
-            "constant.numeric.scientific",
-            "constant.string.documentation",
-            "entity.name",
-            "entity.name.function.invocation",
-            "entity.other",
-            "entity.other.attribute-name",
-            "entity.other.attribute.parameter",
-            "interpolated.complex.source",
-            "interpolated.simple.source",
-            "keyword.control",
-            "keyword.operator.assignment",
-            "keyword.operator.bitwise",
-            "keyword.operator.comparison",
-            "keyword.operator.documentation",
-            "keyword.operator.logical",
-            "keyword.operator.math",
-            "keyword.operator.other",
-            "keyword.operator.range",
-            "keyword.operator.redirection",
-            "keyword.operator.string-format",
-            "keyword.operator.unary",
-            "keyword.other",
-            "keyword.other.statement-separator",
-            "meta",
-            "meta.group.array-expression",
-            "meta.group.complex.subexpression",
-            "meta.scriptblock",
-            "punctuation.end.definition.comment.block",
-            "punctuation.start.definition.comment.block",
-            #"source",  # this scope represents the whole file
-            "storage",
-            "storage.modifier.scope",
-            "string.quoted.double",
-            "string.quoted.double.heredoc",
-            "string.quoted.single",
-            "string.quoted.single.heredoc",
-            "support.constant",
-            "support.constant.variable",
-            "support.function",
-            "support.variable.automatic",
-            "support.variable.drive",
-            "variable.language",
-            "variable.other",
-            "variable.other.normal",
-            "variable.other.readwrite",
+            "comment.block.powershell", 
+            "comment.documentation.embedded.powershell", 
+            "comment.line.number-sign.powershell", 
+            "constant.character.escape.powershell", 
+            "constant.language.powershell", 
+            "constant.numeric.hexadecimal.powershell", 
+            "constant.numeric.scientific.powershell", 
+            "constant.string.documentation.powershell", 
+            "entity.name.function", 
+            "entity.name.function ", 
+            "entity.name.function.invocation.powershell", 
+            "entity.name.tag", 
+            "entity.other.attribute-name", 
+            "entity.other.attribute-name ", 
+            "entity.other.attribute-name.powershell", 
+            "entity.other.attribute.parameter.powershell", 
+            "interpolated.complex.source.powershell", 
+            "interpolated.simple.source.powershell", 
+            "keyword.control.powershell", 
+            "keyword.operator.assignment.powershell", 
+            "keyword.operator.bitwise.powershell", 
+            "keyword.operator.comparison.powershell", 
+            "keyword.operator.documentation.powershell", 
+            "keyword.operator.logical.powershell", 
+            "keyword.operator.math.powershell", 
+            "keyword.operator.other.powershell", 
+            "keyword.operator.range.powershell", 
+            "keyword.operator.redirection.powershell", 
+            "keyword.operator.string-format.powershell", 
+            "keyword.operator.unary.powershell", 
+            "keyword.other.powershell", 
+            "keyword.other.statement-separator.powershell", 
+            "meta.function", 
+            "meta.group.array-expression.powershell", 
+            "meta.group.complex.subexpression.powershell", 
+            "meta.scriptblock.powershell", 
+            "PowerShell", 
+            "punctuation.end.definition.comment.block.powershell", 
+            "punctuation.start.definition.comment.block.powershell", 
+            "source.powershell", 
+            "storage.modifier.scope.powershell", 
+            "storage.type", 
+            "storage.type.powershell", 
+            "string.quoted.double.heredoc.powershell", 
+            "string.quoted.double.powershell", 
+            "string.quoted.single.heredoc.powershell", 
+            "string.quoted.single.powershell", 
+            "support.constant.automatic.powershell", 
+            "support.constant.powershell", 
+            "support.constant.variable.powershell", 
+            "support.function.powershell", 
+            "support.variable.automatic.powershell", 
+            "support.variable.drive.powershell", 
+            "variable.language.powershell", 
+            "variable.other.normal.powershell", 
+            "variable.other.powershell", 
+            "variable.other.readwrite.powershell",
         ]
         tokens = []
         for selector in selectors:
diff --git a/tests/py/syntax_def/test_token_gen.py b/tests/py/syntax_def/test_token_gen.py
@@ -18,10 +18,19 @@ def testGetTokens(self):
         self.writeTokensToFile(self.getTokens(), "test-file.ps1.tokens")
 
     def testGetLowerTokens(self):
+        def escaped_lower(i, line):
+            should = not (i > 0 and line[i-1] == '`')
+            if (should):
+                return line[i].lower()
+            else:
+                return line[i]
+
         with open(self.test_path) as f:
             content = f.readlines()
             for line in content:
-                self.append(line.lower())
+                # escaped chars (like `n) are case-sensitive, don't upper them
+                line_low = ''.join([escaped_lower(i, line) for i in range(len(line))])
+                self.append(line_low)
         self.writeTokensToFile(self.getTokens(), "test-file.ps1.lower.tokens")
 
     def testGetUpperTokens(self):
diff --git a/tests/samples/test-file.ps1 b/tests/samples/test-file.ps1
@@ -48,13 +48,29 @@ function Get-MemberSignature {
         # The whole script name is the command, not just the part before the extension
         Some-Script.ps1 -argOne one -argTwo "two"
         &"Some Script.bat" -param $PSVersionTable -arg 43
+
+        $args
+        $input
+        $PSScriptRoot
     }
 }
 
 function echo([string]$text) {
     write-host $text
 }
 
+# declarations should be consistent
+function foo.bar() {}
+Function foo() {}
+
+class A {}
+Class Foo-Bar {}
+
+workflow w1 {}
+Workflow work {}
+
+configuration c {}
+Configuration c {}
 
 # Highlight types
 [int[]][char[]]"hello world"
@@ -427,3 +443,28 @@ Get-things.ps1 -value @args
 
 # array subexpression
 @(This $a is it. | "$(this-is | @($('yeah'| "" )) )")
+
+# TODO: [minor] D:\dev\Find-String should not be treated as cmdlet name
+Import-Module D:\dev\Find-String
+
+
+function Get-EscapedPath
+{
+    param( 
+    [Parameter(
+        Position=0, 
+        Mandatory=$true
+        ValueFromPipeline=$true,
+        ValueFromPipelineByPropertyName=$true)
+    ]
+    [string]$path
+    ) 
+
+    process {
+        if ($path.Contains(' '))
+        {
+            return '"' + $path + '"'
+        }
+        return $path
+    }
+}
