refactor(tokenizer): Extract token printing into own file.

This structure is getting rather Java-y, isn't it...

Author: MarcellPerger1

parser/lexer/__init__.py

@@ -1,4 +1,5 @@
-from .tokenizer import Tokenizer, print_tokens, format_tokens
+from .tokenizer import Tokenizer
+from .token_print import print_tokens, format_tokens
 from .errors import (
     TokenizerError, LocatedTokenizerError, MalformedNumberError,
     LocatedMalformedNumberError)

parser/lexer/number_parser.py

@@ -2,10 +2,10 @@
 
 from string import digits
 
-from parser.common import StrRegion
-from parser.lexer import LocatedMalformedNumberError
-from parser.lexer.src_handler import UsesSrc
-from parser.lexer.tokens import Token, NumberToken
+from .errors import LocatedMalformedNumberError
+from .src_handler import UsesSrc
+from .tokens import Token, NumberToken
+from ..common import StrRegion
 
 
 class NumberParser(UsesSrc):

parser/lexer/token_print.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import sys
+from io import StringIO
+from typing import IO
+
+from .tokens import Token
+
+
+def print_tokens(src: str, tokens: list[Token], stream: IO[str] = None, do_ws=False):
+    if stream is None:
+        stream = sys.stdout
+    table = []
+    for tok in tokens:
+        if tok.is_whitespace:
+            if do_ws:
+                table.append(['(WS) ' + repr(tok.region.resolve(src)), tok.name])
+        else:
+            table.append([str(tok.region.resolve(src)), tok.name])
+    max0 = max(len(r[0]) for r in table)
+    max1 = max(len(r[1]) for r in table)
+    for s0, s1 in table:
+        print(f'{s0:>{max0}} | {s1:>{max1}}', file=stream)
+
+
+def format_tokens(src: str, tokens: list[Token], do_ws=False):
+    out = StringIO()
+    print_tokens(src, tokens, out, do_ws)
+    return out.getvalue()

parser/lexer/tokenizer.py

@@ -1,9 +1,6 @@
 from __future__ import annotations
 
-import sys
-from io import StringIO
 from string import ascii_letters, digits
-from typing import IO
 
 from .number_parser import NumberParser
 from .src_handler import UsesSrc
@@ -16,6 +13,19 @@
 IDENT_CONT = IDENT_START + digits
 
 
+GETATTR_VALID_AFTER_CLS = (
+    StringToken,
+    RParToken,
+    RSqBracket,
+    AttrNameToken,
+    IdentNameToken
+    # Not valid (directly) after floats (need parens) because we treat all
+    # numbers the same and we cannot have it after ints
+    #   2.3 => (2).3 (attribute) or `2.3` (float)
+    # Also it would be confusing to have 2.e3 => num, 2.e3.3 -> num.attr.
+)
+
+
 class Tokenizer(UsesSrc):
     def __init__(self, src: str):
         super().__init__(src)
@@ -207,38 +217,3 @@ def _t_ident_name(self, start: int) -> int:
         while self.get(idx) in IDENT_CONT:
             idx += 1
         return self.add_token(IdentNameToken(StrRegion(start, idx)))
-
-
-GETATTR_VALID_AFTER_CLS = (
-    StringToken,
-    RParToken,
-    RSqBracket,
-    AttrNameToken,
-    IdentNameToken
-    # Not valid (directly) after floats (need parens) because we treat all
-    # numbers the same and we cannot have it after ints
-    #   2.3 => (2).3 (attribute) or `2.3` (float)
-    # Also it would be confusing to have 2.e3 => num, 2.e3.3 -> num.attr.
-)
-
-
-def print_tokens(src: str, tokens: list[Token], stream: IO[str] = None, do_ws=False):
-    if stream is None:
-        stream = sys.stdout
-    table = []
-    for tok in tokens:
-        if tok.is_whitespace:
-            if do_ws:
-                table.append(['(WS) ' + repr(tok.region.resolve(src)), tok.name])
-        else:
-            table.append([str(tok.region.resolve(src)), tok.name])
-    max0 = max(len(r[0]) for r in table)
-    max1 = max(len(r[1]) for r in table)
-    for s0, s1 in table:
-        print(f'{s0:>{max0}} | {s1:>{max1}}', file=stream)
-
-
-def format_tokens(src: str, tokens: list[Token], do_ws=False):
-    out = StringIO()
-    print_tokens(src, tokens, out, do_ws)
-    return out.getvalue()