Merge pull request #15 from Dhritikrishna123/dev

feat: change the lexer to make it more modular

Author: Puskar-Roy

main.py

@@ -1,32 +1,40 @@
-"""Module providing Lexer Function"""
-from src.lexer import lexer
-
-source_code = '''
-# Variable declarations and assignments
-quack("Hello")
-let my_count = ((( 7 + my_idea ) * 7437 + 949) == 84)
-if (x = 10):
-    y = x + 20 * 3
-    if (a == b):
-        quack(90)
-'''
-
-def run_lexer(source_code):
-    '''Call the lexer to tokenize the source code'''
-    tokens = lexer(source_code)
-    
-    # Print header
-    print(f"{'Token Type':<25} {'Value':<30} {'Position'}")
-    print("="*80)  # Separator line
-    
-    # Print each token in a well-formatted way
-    for token in tokens:
-        token_type = token['type']
-        value = token['value']
-        position = token['position']
-        
-        # Print token info with proper formatting
-        print(f"{token_type:<25} {str(value):<30} {position}")
-    
-if __name__ == "__main__":
-    run_lexer(source_code)
+from src.lexer import Lexer
+
+# Sample source code input
+source_code =   """
+                    def match(self, text):
+        if self.source[self.position.index:self.position.index + len(text)] -> == text:
+            for . in range(len(text)):  
+                self.advance()
+            return True
+        return False
+
+                """
+
+# Initialize lexer
+lexer = Lexer()
+tokens = lexer.tokenize(source_code)
+
+# Print detailed token information
+print("\n📝 Token Details:")
+for token in tokens:
+    print(f"🔹 {token}")
+
+# Print a nicely formatted table
+print("\n📌 Token Table:")
+
+# Define column widths
+col_widths = [30, 12, 14, 12, 6, 8]
+line_sep = "+" + "+".join(["-" * w for w in col_widths]) + "+"
+
+# Print table header
+print(line_sep)
+print(f"| {'Type':<20} | {'Value':<12} | {'Start Index':<14} | {'End Index':<12} | {'Line':<6} | {'Column':<8} |")
+print(line_sep)
+
+# Print table rows
+for token in tokens:
+    print(f"| {str(token.token_type):<20} | {str(token.value):<12} | {str(token.start_pos.index):<14} | {str(token.end_pos.index):<12} | {str(token.start_pos.line):<6} | {str(token.start_pos.column):<8} |")
+
+# Print table footer
+print(line_sep)