add brackets to tokenizer

Author: maxijabase

SourcepawnCondenser/CondenserTest/MainWindow.xaml

@@ -12,7 +12,7 @@
       <ColumnDefinition />
       <ColumnDefinition />
     </Grid.ColumnDefinitions>
-    <TextBox Name="textBox" Grid.Column="0" HorizontalAlignment="Stretch" VerticalAlignment="Stretch" AcceptsReturn="True" AcceptsTab="True" TextChanged="TextBox_TextChanged" Margin="0,0,0,25" SelectionChanged="CaretPositionChangedEvent" />
+    <TextBox Name="textBox" Grid.Column="0" HorizontalAlignment="Stretch" VerticalAlignment="Stretch" FontSize="16" AcceptsReturn="True" AcceptsTab="True" TextChanged="TextBox_TextChanged" Margin="0,0,0,25" SelectionChanged="CaretPositionChangedEvent" />
     <Label Name="CaretLabel" Grid.Column="0" HorizontalAlignment="Left" VerticalAlignment="Bottom" Content="0 / 0" />
     <ScrollViewer Grid.Column="1" HorizontalAlignment="Stretch" VerticalAlignment="Stretch">
       <StackPanel Name="tokenStack" HorizontalAlignment="Stretch" VerticalAlignment="Stretch" Orientation="Vertical" />

SourcepawnCondenser/CondenserTest/MainWindow.xaml.cs

@@ -365,6 +365,8 @@ private static Brush ChooseBackgroundFromTokenKind(TokenKind kind)
         {
             TokenKind.BraceClose or
             TokenKind.BraceOpen => Brushes.LightGray,
+            TokenKind.BracketOpen or
+            TokenKind.BracketClose => Brushes.DarkKhaki,
             TokenKind.Character => Brushes.LightSalmon,
             TokenKind.EOF => Brushes.LimeGreen,
             TokenKind.Identifier => Brushes.LightSteelBlue,

SourcepawnCondenser/SourcepawnCondenser/Tokenizer/Token.cs

@@ -2,18 +2,18 @@
 
 public class Token
 {
-    public Token(string Value_, TokenKind Kind_, int Index_)
+    public Token(string value, TokenKind kind, int index)
     {
-        Value = Value_;
-        Kind = Kind_;
-        Index = Index_;
-        Length = Value_.Length;
+        Value = value;
+        Kind = kind;
+        Index = index;
+        Length = value.Length;
     }
-    public Token(char Value_, TokenKind Kind_, int Index_)
+    public Token(char value, TokenKind kind, int index)
     {
-        Value = Value_.ToString();
-        Kind = Kind_;
-        Index = Index_;
+        Value = value.ToString();
+        Kind = kind;
+        Index = index;
         Length = 1;
     }
     public string Value;

SourcepawnCondenser/SourcepawnCondenser/Tokenizer/TokenKind.cs

@@ -7,6 +7,8 @@ public enum TokenKind
     Character,              //d
     BraceOpen,              //d
     BraceClose,             //d
+    BracketOpen,
+    BracketClose,
     ParenthesisOpen,        //d
     ParenthesisClose,       //d
     Quote,                  //d

SourcepawnCondenser/SourcepawnCondenser/Tokenizer/Tokenizer.cs

@@ -57,25 +57,31 @@ public static List<Token> TokenizeString(string source, bool ignoreMultipleEOL)
                     continue;
                 }
                 case '{':
-                    token.Add(new Token("{", TokenKind.BraceOpen, i));
+                    token.Add(new Token(c, TokenKind.BraceOpen, i));
                     continue;
                 case '}':
-                    token.Add(new Token("}", TokenKind.BraceClose, i));
+                    token.Add(new Token(c, TokenKind.BraceClose, i));
+                    continue;
+                case '[':
+                    token.Add(new Token(c, TokenKind.BracketOpen, i));
+                    continue;
+                case ']':
+                    token.Add(new Token(c, TokenKind.BracketClose, i));
                     continue;
                 case '(':
-                    token.Add(new Token("(", TokenKind.ParenthesisOpen, i));
+                    token.Add(new Token(c, TokenKind.ParenthesisOpen, i));
                     continue;
                 case ')':
-                    token.Add(new Token(")", TokenKind.ParenthesisClose, i));
+                    token.Add(new Token(c, TokenKind.ParenthesisClose, i));
                     continue;
                 case ';':
-                    token.Add(new Token(";", TokenKind.Semicolon, i));
+                    token.Add(new Token(c, TokenKind.Semicolon, i));
                     continue;
                 case ',':
-                    token.Add(new Token(",", TokenKind.Comma, i));
+                    token.Add(new Token(c, TokenKind.Comma, i));
                     continue;
                 case '=':
-                    token.Add(new Token("=", TokenKind.Assignment, i));
+                    token.Add(new Token(c, TokenKind.Assignment, i));
                     continue;
                 case '/':
                 {
@@ -482,8 +488,8 @@ public static List<Token> TokenizeString(string source, bool ignoreMultipleEOL)
         }
 
         token.Add(new Token("", TokenKind.EOF, sArrayLength));
-        
-        
+
+
         return token;
     }