Formatting the AST (Fixes #166)

Author: StartAutomating

PipeScript.types.ps1xml

@@ -24,6 +24,26 @@ while ($thisParent) {
 
                     </Script>
       </ScriptMethod>
+      <ScriptProperty>
+        <Name>Tokens</Name>
+        <GetScriptBlock>
+                        $text = $this.Extent.ToString()
+    
+$previousToken = $null
+$tokenCount = 0
+@(foreach ($token in [Management.Automation.PSParser]::Tokenize($text, [ref]$null)) {
+    Add-Member NoteProperty Text $text -Force -InputObject $token
+    Add-Member NoteProperty PreviousToken $previousToken -Force -InputObject $token
+    if ($token.Type -in 'Variable', 'String') {
+        $realContent = $text.Substring($token.Start, $token.Length)
+        Add-Member NoteProperty Content $realContent  -Force -InputObject $token
+    }
+    $previousToken = $token
+    $tokenCount++
+    $token            
+})
+                    </GetScriptBlock>
+      </ScriptProperty>
     </Members>
   </Type>
   <Type>