Impliment doesn't split tokens across the scopes test

Author: vors

tests/pester/Syntax.Tests.ps1

@@ -1,6 +1,4 @@
 $sublimeRoot = "C:\st"
-$scopesFile = Join-Path $sublimeRoot "Data\Packages\User\UnitTesting\tokens\PowerShell_tokens"
-$testFile = Resolve-Path "$PSScriptRoot\..\samples\test-file.ps1"
 
 Import-Module "$PSScriptRoot\SyntaxHelper.psm1"
 
@@ -16,11 +14,31 @@ You need to run python tests first to make it work.
 
     Context "test-file.ps1" {
 
-        $scopes = cat -Raw $scopesFile | ConvertFrom-Json
-        $tokens = Get-TokensFromFile $testFile
+        $scopesFile = Join-Path $sublimeRoot "Data\Packages\User\UnitTesting\tokens\PowerShell_tokens"
+        $testFile = Resolve-Path "$PSScriptRoot\..\samples\test-file.ps1"
 
-        It "doesn't split tokens to scopes" {
-            
+        # splitted in two lines, because of a bug in Sort-Object
+        $stScopes = cat -Raw $scopesFile | ConvertFrom-Json; $stScopes = $stScopes | sort -Property @('startOffset', 'endOffset')
+        # tokens are already sorted
+        $psScopes = Get-TokensFromFile $testFile | Convert-TokenToScope 
+
+        It "doesn't split tokens across the scopes" {
+            $stIndex = 0
+            $psScopes | %{
+                while ($stScopes[$stIndex].endOffset -le $_.startOffset) {
+                    $stIndex++
+                }
+                #Write-Host "PowerShell scope $_ "
+                #Write-Host "SublimeText scope $($stScopes[$stIndex])"
+                
+                if (-not (Test-ScopeInclosure $_ $stScopes[$stIndex])) {
+                    #Write-Warning "PowerShell scope not found in SublimeText scopes $_ "
+                    if (-not (Test-ScopeDisclosure $_ $stScopes[$stIndex])) {
+                        Write-Error "PowerShell scope $_ overlap with SublimeText scope $($stScopes[$stIndex]) "
+                        $false | Should be $true
+                    }
+                }
+            }
         }
     }
 }

tests/pester/SyntaxHelper.psm1

@@ -28,3 +28,72 @@ function Get-TokensFromInput
 
     return $tokens
 }
+
+function Convert-TokenToScope
+{
+    [CmdletBinding()]
+    param(
+        [Parameter(ValueFromPipeline)]
+        [System.Management.Automation.Language.Token] $token
+    )
+
+    process 
+    {
+        if (@('NewLine', 'EndOfInput') -contains $token.Kind) {
+            # ignore these tokens
+            return
+        }
+
+        $h = @{
+            text = $token.Text
+            # Adjust offsets for CRLF
+            startOffset = $token.Extent.StartOffset - $token.Extent.StartLineNumber + 1
+            endOffset = $token.Extent.EndOffset - $token.Extent.EndLineNumber + 1
+            kind = $token.Kind
+        }
+
+        New-Object -TypeName PSObject -Property $h
+    }    
+}
+
+function Test-ScopeInclosure
+{
+    [CmdletBinding()]
+    param(
+        $inScope,
+        $outScope
+    )
+
+    if ($inScope.startOffset -lt $outScope.startOffset)
+    {
+        return $false
+    }
+
+    if ($inScope.endOffset -gt $outScope.endOffset)
+    {
+        return $false
+    }    
+
+    return $true
+}
+
+function Test-ScopeDisclosure
+{
+    [CmdletBinding()]
+    param(
+        $leftScope,
+        $rightScope
+    )
+
+    if ($leftScope.startOffset -le $rightScope.endOffset)
+    {
+        return $true
+    }
+
+    if ($leftScope.endOffset -ge $rightScope.endOffset)
+    {
+        return $false
+    }    
+
+    return $false
+}

tests/py/__init__.py

@@ -39,9 +39,6 @@ def setUp(self):
 
 class PowerShellSyntaxTokenTest(PowerShellSyntaxTest):
 
-    def getTokenHeader(self):
-        return ['scope_name', 'str', 'start', 'end']
-
     def getTokens(self):
         selectors = [
             "comment.block",
@@ -79,7 +76,7 @@ def getTokens(self):
             "meta.scriptblock",
             "punctuation.end.definition.comment.block",
             "punctuation.start.definition.comment.block",
-            "source",
+            #"source",  # this scope represents the whole file
             "storage",
             "storage.modifier.scope",
             "string.quoted.double",
@@ -100,5 +97,5 @@ def getTokens(self):
         for selector in selectors:
             regions = self.view.find_by_selector(selector)
             for region in regions:
-                tokens += [{ 'scope_name': selector, 'str': self.view.substr(region), 'start': region.a, 'end': region.b }]
+                tokens += [{ 'kind': selector, 'text': self.view.substr(region), 'startOffset': region.a, 'endOffset': region.b }]
         return tokens