Add upper and lower tests

Author: vors

tests/pester/Syntax.Tests.ps1

@@ -6,18 +6,14 @@ Describe "Syntax highlighting" {
 
     Context "test-file.ps1" {
 
-        $tokensFileName = "test-file.ps1.tokens"
         $tokensFolder = Join-Path $sublimeRoot "Data\Packages\User\UnitTesting\tokens" 
-        $scopesFile = Join-Path $tokensFolder $tokensFileName 
+        $scopesFile = Join-Path $tokensFolder "test-file.ps1.tokens" 
 
         $testFile = Resolve-Path "$PSScriptRoot\..\samples\test-file.ps1"
 
         if (-not (Test-Path $scopesFile)) {
-        Write-Warning @'
-This Pester UT file consumes scopes generated by python unit test Test_TokenGenerator.testGetTokens .
-You need to run python tests first to make it work.
-'@
-        return
+            Write-Warning 'This Pester UT suite consumes scopes generated by python unit test Test_TokenGenerator. You need to run python tests first.'
+            return
         }
 
         # splitted in two lines, because of a bug in Sort-Object
@@ -32,6 +28,9 @@ You need to run python tests first to make it work.
             $psScopes | %{
                 while ($stScopes[$stIndex].endOffset -le $_.startOffset) {
                     $stIndex++
+                    if ($stIndex -ge $stScopes.Length) {
+                        break
+                    }
                 }
 
                 $stScope = $stScopes[$stIndex]
@@ -65,5 +64,23 @@ You need to run python tests first to make it work.
             # TODO: These are bugs, make it 0
             $errorCounter | Should be @(0..4)
         }
+
+        It "produces same tokens for lower case" {
+            $stLowScopes = Get-SublimeScopesFromFile (Join-Path $tokensFolder "test-file.ps1.lower.tokens" )
+            $diff = Compare-Object $stScopes $stLowScopes
+            if ($diff) {
+                $diff | Write-DiffAsWarning
+            }
+            $diff | Should Be $null
+        }
+
+        It "produces same tokens for upper case" {
+            $stUpperScopes = Get-SublimeScopesFromFile (Join-Path $tokensFolder "test-file.ps1.upper.tokens" )
+            $diff = Compare-Object $stScopes $stUpperScopes
+            if ($diff) {
+                $diff | Write-DiffAsWarning
+            }
+            $diff | Should Be $null
+        }
     }
 }

tests/pester/SyntaxHelper.psm1

@@ -13,6 +13,19 @@ function Get-TokensFromFile
     return $tokens
 }
 
+function Get-SublimeScopesFromFile
+{
+    [CmdletBinding()]
+    param(
+        [Parameter(ValueFromPipeline)]
+        [string] $filePath
+    )
+
+    # splitted in two lines, because of a bug in Sort-Object
+    $scopes = cat -Raw $filePath | ConvertFrom-Json
+    $scopes = $scopes | sort -Property @('startOffset', 'endOffset')
+    return $scopes
+}
 
 function Get-TokensFromInput
 {
@@ -29,6 +42,53 @@ function Get-TokensFromInput
     return $tokens
 }
 
+function Select-TokenByOffset
+{
+    [CmdletBinding()]
+    param(
+        [Parameter(ValueFromPipeline)]
+        [System.Management.Automation.Language.Token] $token,
+        [int] $offset
+    )
+
+    process 
+    {
+        if (((Get-TokenStartOffset $token) -le $offset) -and ((Get-TokenEndOffset $token) -gt $offset)) {
+            return $token
+        }
+    }
+}
+
+function Get-TokenStartOffset
+{
+    [CmdletBinding()]
+    param(
+        [Parameter(ValueFromPipeline)]
+        [System.Management.Automation.Language.Token] $token
+    )
+
+    process 
+    {
+        # Adjust offsets for CRLF
+        return $token.Extent.StartOffset - $token.Extent.StartLineNumber + 1
+    }
+}
+
+function Get-TokenEndOffset
+{
+    [CmdletBinding()]
+    param(
+        [Parameter(ValueFromPipeline)]
+        [System.Management.Automation.Language.Token] $token
+    )
+
+    process 
+    {
+        # Adjust offsets for CRLF
+        return $token.Extent.EndOffset - $token.Extent.EndLineNumber + 1
+    }
+}
+
 function Convert-TokenToScope
 {
     [CmdletBinding()]
@@ -46,9 +106,8 @@ function Convert-TokenToScope
 
         $h = @{
             text = $token.Text
-            # Adjust offsets for CRLF
-            startOffset = $token.Extent.StartOffset - $token.Extent.StartLineNumber + 1
-            endOffset = $token.Extent.EndOffset - $token.Extent.EndLineNumber + 1
+            startOffset = Get-TokenStartOffset $token
+            endOffset = Get-TokenEndOffset $token
             kind = $token.Kind
         }
 
@@ -96,4 +155,18 @@ function Test-ScopeDisclosure
     }    
 
     return $false
+}
+
+function Write-DiffAsWarning
+{
+    [CmdletBinding()]
+    param(
+        [Parameter(ValueFromPipeline)]
+        $diff
+    )
+
+    process 
+    {
+        Write-Warning "$($diff.SideIndicator) | $($diff.InputObject)"
+    }
 }

tests/py/syntax_def/test_token_gen.py

@@ -11,11 +11,23 @@ def setUp(self):
         self.test_path = os.path.join(sublime.packages_path(), 'PowerShell', 'tests', 'samples', 'test-file.ps1')
 
     def testGetTokens(self):
-
         with open(self.test_path) as f:
             content = f.readlines()
             for line in content:
                 self.append(line)
-
         self.writeTokensToFile(self.getTokens(), "test-file.ps1.tokens")
+
+    def testGetLowerTokens(self):
+        with open(self.test_path) as f:
+            content = f.readlines()
+            for line in content:
+                self.append(line.lower())
+        self.writeTokensToFile(self.getTokens(), "test-file.ps1.lower.tokens")
+
+    def testGetUpperTokens(self):
+        with open(self.test_path) as f:
+            content = f.readlines()
+            for line in content:
+                self.append(line.upper())
+        self.writeTokensToFile(self.getTokens(), "test-file.ps1.upper.tokens")