Custom EOF in Reader (#218)

Author: chrisrink10

src/basilisp/cli.py

@@ -3,6 +3,7 @@
 import readline  # noqa: F401
 import traceback
 import types
+from typing import Any
 
 import click
 import pytest
@@ -28,10 +29,10 @@ def eval_file(filename: str, ctx: compiler.CompilerContext, module: types.Module
     return last
 
 
-def eval_str(s: str, ctx: compiler.CompilerContext, module: types.ModuleType):
+def eval_str(s: str, ctx: compiler.CompilerContext, module: types.ModuleType, eof: Any):
     """Evaluate the forms in a string into a Python module AST node."""
-    last = None
-    for form in reader.read_str(s, resolver=runtime.resolve_alias):
+    last = eof
+    for form in reader.read_str(s, resolver=runtime.resolve_alias, eof=eof):
         last = compiler.compile_and_exec_form(form, ctx, module, source_filename='REPL Input')
     return last
 
@@ -55,6 +56,7 @@ def repl(default_ns):
     repl_module = bootstrap_repl(default_ns)
     ctx = compiler.CompilerContext()
     ns_var = runtime.set_current_ns(default_ns)
+    eof = object()
     while True:
         ns: runtime.Namespace = ns_var.value
         try:
@@ -69,7 +71,9 @@ def repl(default_ns):
             continue
 
         try:
-            result = eval_str(lsrc, ctx, ns.module)
+            result = eval_str(lsrc, ctx, ns.module, eof)
+            if result is eof:
+                continue
             print(compiler.lrepr(result))
             repl_module.mark_repl_result(result)
         except reader.SyntaxError as e:
@@ -97,9 +101,10 @@ def run(file_or_code, code, in_ns):
     ctx = compiler.CompilerContext()
     ns_var = runtime.set_current_ns(in_ns)
     ns: runtime.Namespace = ns_var.value
+    eof = object()
 
     if code:
-        print(compiler.lrepr(eval_str(file_or_code, ctx, ns.module)))
+        print(compiler.lrepr(eval_str(file_or_code, ctx, ns.module, eof)))
     else:
         print(compiler.lrepr(eval_file(file_or_code, ctx, ns.module)))
 

src/basilisp/core/__init__.lpy

@@ -1012,9 +1012,15 @@
   Note that read-string should not be used to read string input from
   untrusted sources."
   ([s]
-   (read-string {} s))
+   (read-string {:eof :eofthrow} s))
   ([opts s]
-   (first (basilisp.reader/read-str s *resolver* *data-readers*))))
+   (first (basilisp.reader/read-str s
+                                    *resolver*
+                                    *data-readers*
+                                    (:eof opts)
+                                    (if (= (:eof opts) :eofthrow)
+                                      true
+                                      false)))))
 
 (defn read
   "Read the next form from the stream. If no stream is specified, uses
@@ -1029,9 +1035,21 @@
   ([]
    (read *in*))
   ([stream]
-   (read {} stream))
+   (read stream true nil))
   ([opts stream]
-   (first (basilisp.reader/read stream *resolver* *data-readers*))))
+   (first (basilisp.reader/read stream
+                                *resolver*
+                                *data-readers*
+                                (:eof opts)
+                                (if (= (:eof opts) :eofthrow)
+                                  true
+                                  false))))
+  ([stream eof-error? eof-value]
+   (first (basilisp.reader/read stream
+                                *resolver*
+                                *data-readers*
+                                eof-value
+                                eof-error?))))
 
 (defn eval
   "Evaluate a form (not a string) and return its result."

src/basilisp/reader.py

@@ -172,11 +172,14 @@ class ReaderContext:
                  '_resolve',
                  '_in_anon_fn',
                  '_syntax_quoted',
-                 '_gensym_env')
+                 '_gensym_env',
+                 '_eof')
 
-    def __init__(self, reader: StreamReader,
+    def __init__(self,
+                 reader: StreamReader,
                  resolver: Resolver = None,
-                 data_readers: DataReaders = None) -> None:
+                 data_readers: DataReaders = None,
+                 eof: Any = None) -> None:
         data_readers = Maybe(data_readers).or_else_get(lmap.Map.empty())
         for entry in data_readers:
             if not isinstance(entry.key, symbol.Symbol):
@@ -192,11 +195,16 @@ def __init__(self, reader: StreamReader,
         self._in_anon_fn: Deque[bool] = collections.deque([])
         self._syntax_quoted: Deque[bool] = collections.deque([])
         self._gensym_env: Deque[GenSymEnvironment] = collections.deque([])
+        self._eof = eof
 
     @property
     def data_readers(self) -> lmap.Map:
         return self._data_readers
 
+    @property
+    def eof(self) -> Any:
+        return self._eof
+
     @property
     def reader(self) -> StreamReader:
         return self._reader
@@ -243,7 +251,7 @@ def is_syntax_quoted(self) -> bool:
             return False
 
 
-__EOF = 'EOF'
+EOF = 'EOF'
 
 
 def _with_loc(f: W) -> W:
@@ -954,7 +962,7 @@ def _read_comment(ctx: ReaderContext) -> LispReaderForm:
             reader.advance()
             return _read_next(ctx)
         if token == '':
-            return __EOF
+            return ctx.eof
         reader.advance()
 
 
@@ -963,8 +971,8 @@ def _read_next_consuming_comment(ctx: ReaderContext) -> LispForm:
     reader comments completely."""
     while True:
         v = _read_next(ctx)
-        if v is __EOF:
-            return __EOF
+        if v is ctx.eof:
+            return ctx.eof
         if v is COMMENT or isinstance(v, Comment):
             continue
         return v
@@ -1008,12 +1016,16 @@ def _read_next(ctx: ReaderContext) -> LispReaderForm:  # noqa: C901
     elif token == '@':
         return _read_deref(ctx)
     elif token == '':
-        return __EOF
+        return ctx.eof
     else:
         raise SyntaxError("Unexpected token '{token}'".format(token=token))
 
 
-def read(stream, resolver: Resolver = None, data_readers: DataReaders = None) -> Iterable[LispForm]:
+def read(stream,
+         resolver: Resolver = None,
+         data_readers: DataReaders = None,
+         eof: Any = EOF,
+         is_eof_error: bool = False) -> Iterable[LispForm]:
     """Read the contents of a stream as a Lisp expression.
 
     Callers may optionally specify a namespace resolver, which will be used
@@ -1028,29 +1040,47 @@ def read(stream, resolver: Resolver = None, data_readers: DataReaders = None) ->
 
     The caller is responsible for closing the input stream."""
     reader = StreamReader(stream)
-    ctx = ReaderContext(reader, resolver=resolver, data_readers=data_readers)
+    ctx = ReaderContext(reader, resolver=resolver, data_readers=data_readers, eof=eof)
     while True:
         expr = _read_next(ctx)
-        if expr is __EOF:
+        if expr is ctx.eof:
+            if is_eof_error:
+                raise EOFError
             return
         if expr is COMMENT or isinstance(expr, Comment):
             continue
         yield expr
 
 
-def read_str(s: str, resolver: Resolver = None, data_readers: DataReaders = None) -> Iterable[LispForm]:
+def read_str(s: str,
+             resolver: Resolver = None,
+             data_readers: DataReaders = None,
+             eof: Any = None,
+             is_eof_error: bool = False) -> Iterable[LispForm]:
     """Read the contents of a string as a Lisp expression.
 
     Keyword arguments to this function have the same meanings as those of
     basilisp.reader.read."""
     with io.StringIO(s) as buf:
-        yield from read(buf, resolver=resolver, data_readers=data_readers)
-
-
-def read_file(filename: str, resolver: Resolver = None, data_readers: DataReaders = None) -> Iterable[LispForm]:
+        yield from read(buf,
+                        resolver=resolver,
+                        data_readers=data_readers,
+                        eof=eof,
+                        is_eof_error=is_eof_error)
+
+
+def read_file(filename: str,
+              resolver: Resolver = None,
+              data_readers: DataReaders = None,
+              eof: Any = None,
+              is_eof_error: bool = False) -> Iterable[LispForm]:
     """Read the contents of a file as a Lisp expression.
 
     Keyword arguments to this function have the same meanings as those of
     basilisp.reader.read."""
     with open(filename) as f:
-        yield from read(f, resolver=resolver, data_readers=data_readers)
+        yield from read(f,
+                        resolver=resolver,
+                        data_readers=data_readers,
+                        eof=eof,
+                        is_eof_error=is_eof_error)

tests/basilisp/reader_test.py

@@ -25,11 +25,14 @@ def ns_var(test_ns: str):
     yield runtime.set_current_ns(test_ns)
 
 
-def read_str_first(s, resolver: reader.Resolver = None, data_readers=None):
+def read_str_first(s: str,
+                   resolver: reader.Resolver = None,
+                   data_readers=None,
+                   is_eof_error: bool = False):
     """Read the first form from the input string. If no form
     is found, return None."""
     try:
-        return next(reader.read_str(s, resolver=resolver, data_readers=data_readers))
+        return next(reader.read_str(s, resolver=resolver, data_readers=data_readers, is_eof_error=is_eof_error))
     except StopIteration:
         return None
 
@@ -677,15 +680,32 @@ def test_invalid_meta_attachment():
 
 
 def test_comment_reader_macro():
-    assert None is read_str_first('#_       (a list)')
-    assert None is read_str_first('#_1')
-    assert None is read_str_first('#_"string"')
-    assert None is read_str_first('#_:keyword')
-    assert None is read_str_first('#_symbol')
-    assert None is read_str_first('#_[]')
-    assert None is read_str_first('#_{}')
-    assert None is read_str_first('#_()')
-    assert None is read_str_first('#_#{}')
+    with pytest.raises(EOFError):
+        read_str_first('#_       (a list)', is_eof_error=True)
+
+    with pytest.raises(EOFError):
+        read_str_first('#_1', is_eof_error=True)
+
+    with pytest.raises(EOFError):
+        read_str_first('#_"string"', is_eof_error=True)
+
+    with pytest.raises(EOFError):
+        read_str_first('#_:keyword', is_eof_error=True)
+
+    with pytest.raises(EOFError):
+        read_str_first('#_symbol', is_eof_error=True)
+
+    with pytest.raises(EOFError):
+        read_str_first('#_[]', is_eof_error=True)
+
+    with pytest.raises(EOFError):
+        read_str_first('#_{}', is_eof_error=True)
+
+    with pytest.raises(EOFError):
+        read_str_first('#_()', is_eof_error=True)
+
+    with pytest.raises(EOFError):
+        read_str_first('#_#{}', is_eof_error=True)
 
     assert kw.keyword('kw2') == read_str_first('#_:kw1 :kw2')
 
@@ -719,11 +739,12 @@ def test_comment_reader_macro():
 
 
 def test_comment_line():
-    assert read_str_first("; I'm a little comment short and stout") is None
-    assert read_str_first(";; :kw1\n:kw2") == kw.keyword('kw2')
-    assert read_str_first(""";; Comment
+    assert None is read_str_first("; I'm a little comment short and stout")
+    assert kw.keyword('kw2') == read_str_first(";; :kw1\n:kw2")
+    assert llist.l(sym.symbol('form'), kw.keyword('keyword')) == read_str_first(
+        """;; Comment
         (form :keyword)
-        """) == llist.l(sym.symbol('form'), kw.keyword('keyword'))
+        """)
 
 
 def test_function_reader_macro():