Merge pull request #1562 from lark-parser/issue1560

Bugfix: Restore support for custom input, alongside text and TextSlice

Author: erezsh

lark/lark.py

@@ -16,7 +16,7 @@
     from .parser_frontends import ParsingFrontend
 
 from .exceptions import ConfigurationError, assert_config, UnexpectedInput
-from .utils import Serialize, SerializeMemoizer, FS, logger, TextOrSlice
+from .utils import Serialize, SerializeMemoizer, FS, logger, TextOrSlice, LarkInput
 from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files, PackageResource, sha256_digest
 from .tree import Tree
 from .common import LexerConf, ParserConf, _ParserArgType, _LexerArgType
@@ -637,11 +637,11 @@ def get_terminal(self, name: str) -> TerminalDef:
         """Get information about a terminal"""
         return self._terminals_dict[name]
 
-    def parse_interactive(self, text: Optional[TextOrSlice]=None, start: Optional[str]=None) -> 'InteractiveParser':
+    def parse_interactive(self, text: Optional[LarkInput]=None, start: Optional[str]=None) -> 'InteractiveParser':
         """Start an interactive parsing session. Only works when parser='lalr'.
 
         Parameters:
-            text (TextOrSlice, optional): Text to be parsed. Required for ``resume_parse()``.
+            text (LarkInput, optional): Text to be parsed. Required for ``resume_parse()``.
             start (str, optional): Start symbol
 
         Returns:
@@ -651,12 +651,13 @@ def parse_interactive(self, text: Optional[TextOrSlice]=None, start: Optional[st
         """
         return self.parser.parse_interactive(text, start=start)
 
-    def parse(self, text: TextOrSlice, start: Optional[str]=None, on_error: 'Optional[Callable[[UnexpectedInput], bool]]'=None) -> 'ParseTree':
+    def parse(self, text: LarkInput, start: Optional[str]=None, on_error: 'Optional[Callable[[UnexpectedInput], bool]]'=None) -> 'ParseTree':
         """Parse the given text, according to the options provided.
 
         Parameters:
-            text (TextOrSlice): Text to be parsed, as `str` or `bytes`.
+            text (LarkInput): Text to be parsed, as `str` or `bytes`.
                 TextSlice may also be used, but only when lexer='basic' or 'contextual'.
+                If Lark was created with a custom lexer, this may be an object of any type.
             start (str, optional): Required if Lark was given multiple possible start symbols (using the start option).
             on_error (function, optional): if provided, will be called on UnexpectedInput error,
                 with the exception as its argument. Return true to resume parsing, or false to raise the exception.

lark/lexer.py

@@ -419,15 +419,16 @@ class LexerState:
     last_token: Optional[Token]
 
     def __init__(self, text: TextSlice, line_ctr: Optional[LineCounter] = None, last_token: Optional[Token]=None):
-        if line_ctr is None:
-            line_ctr = LineCounter(b'\n' if isinstance(text.text, bytes) else '\n')
+        if isinstance(text, TextSlice):
+            if line_ctr is None:
+                line_ctr = LineCounter(b'\n' if isinstance(text.text, bytes) else '\n')
 
-            if text.start > 0:
-                # Advance the line-count until line_ctr.char_pos == text.start
-                line_ctr.feed(TextSlice(text.text, 0, text.start))
+                if text.start > 0:
+                    # Advance the line-count until line_ctr.char_pos == text.start
+                    line_ctr.feed(TextSlice(text.text, 0, text.start))
 
-        if not (text.start <= line_ctr.char_pos <= text.end):
-            raise ValueError("LineCounter.char_pos is out of bounds")
+            if not (text.start <= line_ctr.char_pos <= text.end):
+                raise ValueError("LineCounter.char_pos is out of bounds")
 
         self.text = text
         self.line_ctr = line_ctr
@@ -457,6 +458,10 @@ def from_text(cls, lexer: 'Lexer', text_or_slice: TextOrSlice) -> 'LexerThread':
         text = TextSlice.cast_from(text_or_slice)
         return cls(lexer, LexerState(text))
 
+    @classmethod
+    def from_custom_input(cls, lexer: 'Lexer', text: Any) -> 'LexerThread':
+        return cls(lexer, LexerState(text))
+
     def lex(self, parser_state):
         if self.state is None:
             raise TypeError("Cannot lex: No text assigned to lexer state")

lark/parser_frontends.py

@@ -1,7 +1,7 @@
 from typing import Any, Callable, Dict, Optional, Collection, Union, TYPE_CHECKING
 
 from .exceptions import ConfigurationError, GrammarError, assert_config
-from .utils import get_regexp_width, Serialize, TextOrSlice, TextSlice
+from .utils import get_regexp_width, Serialize, TextOrSlice, TextSlice, LarkInput
 from .lexer import LexerThread, BasicLexer, ContextualLexer, Lexer
 from .parsers import earley, xearley, cyk
 from .parsers.lalr_parser import LALR_Parser
@@ -23,7 +23,7 @@ class CustomLexerWrapper1(Lexer):
             def __init__(self, lexer_conf):
                 self.lexer = lexer_class(lexer_conf)
             def lex(self, lexer_state, parser_state):
-                if not lexer_state.text.is_complete_text():
+                if isinstance(lexer_state.text, TextSlice) and not lexer_state.text.is_complete_text():
                     raise TypeError("Interface=1 Custom Lexer don't support TextSlice")
                 lexer_state.text = lexer_state.text
                 return self.lexer.lex(lexer_state, parser_state)
@@ -34,9 +34,11 @@ def __init__(self, lexer_conf):
                 self.lexer = lexer_class(lexer_conf)
 
             def lex(self, lexer_state, parser_state):
-                if not lexer_state.text.is_complete_text():
-                    raise TypeError("Interface=0 Custom Lexer don't support TextSlice")
-                return self.lexer.lex(lexer_state.text.text)
+                if isinstance(lexer_state.text, TextSlice):
+                    if not lexer_state.text.is_complete_text():
+                        raise TypeError("Interface=0 Custom Lexer don't support TextSlice")
+                    return self.lexer.lex(lexer_state.text.text)
+                return self.lexer.lex(lexer_state.text)
         return CustomLexerWrapper0
     else:
         raise ValueError(f"Unknown __future_interface__ value {future_interface}, integer 0-2 expected")
@@ -108,11 +110,17 @@ def _verify_start(self, start=None):
             raise ConfigurationError("Unknown start rule %s. Must be one of %r" % (start, self.parser_conf.start))
         return start
 
-    def _make_lexer_thread(self, text: Optional[TextOrSlice]) -> Union[TextOrSlice, LexerThread, None]:
+    def _make_lexer_thread(self, text: Optional[LarkInput]) -> Union[LarkInput, LexerThread, None]:
         cls = (self.options and self.options._plugins.get('LexerThread')) or LexerThread
-        return text if self.skip_lexer else cls(self.lexer, None) if text is None else cls.from_text(self.lexer, text)
-
-    def parse(self, text: Optional[TextOrSlice], start=None, on_error=None):
+        if self.skip_lexer:
+            return text
+        if text is None:
+            return cls(self.lexer, None)
+        if isinstance(text, (str, bytes, TextSlice)):
+            return cls.from_text(self.lexer, text)
+        return cls.from_custom_input(self.lexer, text)
+
+    def parse(self, text: Optional[LarkInput], start=None, on_error=None):
         if self.lexer_conf.lexer_type in ("dynamic", "dynamic_complete"):
             if isinstance(text, TextSlice) and not text.is_complete_text():
                 raise TypeError(f"Lexer {self.lexer_conf.lexer_type} does not support text slices.")

lark/utils.py

@@ -226,6 +226,7 @@ def rindex(self, substr: AnyStr):
 
 
 TextOrSlice = Union[AnyStr, 'TextSlice[AnyStr]']
+LarkInput = Union[AnyStr, TextSlice[AnyStr], Any]
 
 ###}
 

tests/test_parser.py

@@ -379,6 +379,36 @@ def test_lexer_token_limit(self):
 
         p = Lark(g, parser='lalr')
 
+    def test_custom_input(self):
+        class TypeLexer(Lexer):
+            def __init__(self, lexer_conf):
+                pass
+
+            def lex(self, data):
+                for obj in data:
+                    t = type(obj).__name__.upper()
+                    yield Token(t, obj)
+
+        parser = Lark("""
+                start: data_item+
+                data_item: STR INT*
+
+                %declare STR INT
+                """, parser='lalr', lexer=TypeLexer)
+
+
+        class ParseToDict(Transformer):
+            @v_args(inline=True)
+            def data_item(self, name, *numbers):
+                return name.value, [n.value for n in numbers]
+
+            start = dict
+
+        data = ['alice', 1, 27, 3, 'bob', 4, 'carrie', 'dan', 8, 6]
+        tree = parser.parse(data)
+        res = ParseToDict().transform(tree)
+        assert res == {'alice': [1, 27, 3], 'bob': [4], 'carrie': [], 'dan': [8, 6]}
+
 
 
 def _make_full_earley_test(LEXER):