Support Multiline input at the REPL prompt (#467)

* Support Multiline input at the REPL prompt

* Remove that

* Some changes

* Fix CLI REPL tests

* Refactor prompter class hierarchy

* More refactoring

* Partial Prompter tests

* Test prompter with mocks

* Ignore coverage on key binding event

* Test prompt enter keybinding handler

* Conditionalize Pygments styled print test

Author: chrisrink10

CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ### Added
  * Added support for Shebang-style line comments (#469)
+ * Added multiline REPL support using `prompt-toolkit` (#467)
 
 ### Changed
  * Change the default user namespace to `basilisp.user` (#466)

Pipfile

@@ -6,6 +6,7 @@ name = "pypi"
 [dev-packages]
 black = "*"
 docutils = "==0.15"
+pygments = "*"
 pytest-pycharm = "*"
 sphinx = "*"
 sphinx-rtd-theme = "*"
@@ -19,6 +20,7 @@ atomos = "*"
 attrs = "*"
 basilisp = {editable = true,path = "."}
 click = "*"
+prompt-toolkit = "~=3.0.0"
 pyfunctional = "*"
 pyrsistent = "*"
 python-dateutil = "*"

Pipfile.lock

@@ -11,5 +11,11 @@ ignore_missing_imports = True
 [mypy-functional.*]
 ignore_missing_imports = True
 
+[mypy-prompt_toolkit.*]
+ignore_missing_imports = True
+
+[mypy-pygments.*]
+ignore_missing_imports = True
+
 [mypy-pytest.*]
 ignore_missing_imports = True

mypy.ini

@@ -25,13 +25,14 @@
     "atomos",
     "attrs",
     "click",
+    "prompt-toolkit>=3.0.0",
     "pyfunctional",
     "pyrsistent",
     "pytest",
     "python-dateutil",
 ]
 
-EXTRAS = {}
+EXTRAS = {"pygments": ["pygments"]}
 
 # Copied from the excellent https://github.com/kennethreitz/setup.py
 

setup.py

@@ -1,7 +1,4 @@
-import atexit
 import importlib
-import os.path
-import platform
 import traceback
 import types
 from typing import Any
@@ -14,39 +11,14 @@
 import basilisp.lang.runtime as runtime
 import basilisp.lang.symbol as sym
 import basilisp.main as basilisp
+from basilisp.prompt import get_prompter
 
 CLI_INPUT_FILE_PATH = "<CLI Input>"
-BASILISP_REPL_HISTORY_FILE_PATH = os.path.join(
-    os.path.expanduser("~"), ".basilisp_history"
-)
-BASILISP_REPL_HISTORY_LENGTH = 1000
 REPL_INPUT_FILE_PATH = "<REPL Input>"
 STDIN_INPUT_FILE_PATH = "<stdin>"
 STDIN_FILE_NAME = "-"
 
 
-try:
-    import readline
-except ImportError:  # pragma: no cover
-    pass
-else:
-    readline.parse_and_bind("tab: complete")
-    readline.set_completer_delims("()[]{} \n\t")
-    readline.set_completer(runtime.repl_complete)
-
-    try:
-        readline.read_history_file(BASILISP_REPL_HISTORY_FILE_PATH)
-        readline.set_history_length(BASILISP_REPL_HISTORY_LENGTH)
-    except FileNotFoundError:  # pragma: no cover
-        pass
-    except Exception:  # noqa  # pragma: no cover
-        # PyPy 3.6's ncurses implementation throws an error here
-        if platform.python_implementation() != "PyPy":
-            raise
-    else:
-        atexit.register(readline.write_history_file, BASILISP_REPL_HISTORY_FILE_PATH)
-
-
 @click.group()
 def cli():
     """Basilisp is a Lisp dialect inspired by Clojure targeting Python 3."""
@@ -144,11 +116,12 @@ def repl(
         },
     )
     ns_var = runtime.set_current_ns(default_ns)
+    prompter = get_prompter()
     eof = object()
     while True:
         ns: runtime.Namespace = ns_var.value
         try:
-            lsrc = input(f"{ns.name}=> ")
+            lsrc = prompter.prompt(f"{ns.name}=> ")
         except EOFError:
             break
         except KeyboardInterrupt:  # pragma: no cover
@@ -162,7 +135,7 @@ def repl(
             result = eval_str(lsrc, ctx, ns.module, eof)
             if result is eof:  # pragma: no cover
                 continue
-            print(runtime.lrepr(result))
+            prompter.print(runtime.lrepr(result))
             repl_module.mark_repl_result(result)  # type: ignore
         except reader.SyntaxError as e:
             traceback.print_exception(reader.SyntaxError, e, e.__traceback__)

src/basilisp/cli.py

@@ -111,6 +111,14 @@ class SyntaxError(Exception):  # pylint:disable=redefined-builtin
     pass
 
 
+class UnexpectedEOFError(SyntaxError):
+    """Syntax Error type raised when the reader encounters an unexpected EOF
+    reading a form.
+
+    Useful for cases such as the REPL reader, where unexpected EOF errors
+    likely indicate the user is trying to enter a multiline form."""
+
+
 class StreamReader:
     """A simple stream reader with n-character lookahead."""
 
@@ -489,7 +497,7 @@ def _read_coll(
     while True:
         token = reader.peek()
         if token == "":
-            raise SyntaxError(f"Unexpected EOF in {coll_name}")
+            raise UnexpectedEOFError(f"Unexpected EOF in {coll_name}")
         if whitespace_chars.match(token):
             reader.advance()
             continue
@@ -567,7 +575,7 @@ def __read_map_elems(ctx: ReaderContext) -> Iterable[RawReaderForm]:
         if v is COMMENT or isinstance(v, Comment):
             continue
         elif v is ctx.eof:
-            raise SyntaxError("Unexpected EOF in map")
+            raise UnexpectedEOFError("Unexpected EOF in map")
         elif _should_splice_reader_conditional(ctx, v):
             assert isinstance(v, ReaderConditional)
             selected_feature = v.select_feature(ctx.reader_features)
@@ -726,7 +734,7 @@ def _read_str(ctx: ReaderContext, allow_arbitrary_escapes: bool = False) -> str:
     while True:
         token = reader.next_token()
         if token == "":
-            raise SyntaxError("Unexpected EOF in string")
+            raise UnexpectedEOFError("Unexpected EOF in string")
         if token == "\\":
             token = reader.next_token()
             escape_char = _STR_ESCAPE_CHARS.get(token, None)

src/basilisp/lang/reader.py

@@ -1257,18 +1257,16 @@ def lstr(o) -> str:
 __NOT_COMPLETEABLE = re.compile(r"^[0-9].*")
 
 
-def repl_complete(text: str, state: int) -> Optional[str]:
-    """Completer function for Python's readline/libedit implementation."""
+def repl_completions(text: str) -> Iterable[str]:
+    """Return an optional iterable of REPL completions."""
     # Can't complete Keywords, Numerals
     if __NOT_COMPLETEABLE.match(text):
-        return None
+        return ()
     elif text.startswith(":"):
-        completions = kw.complete(text)
+        return kw.complete(text)
     else:
         ns = get_current_ns()
-        completions = ns.complete(text)
-
-    return list(completions)[state] if completions is not None else None
+        return ns.complete(text)
 
 
 ####################