Fixed a collection reader bug with trailing end tokens separated by comments (#520)

* Fixed a collection reader bug with trailing end tokens separated by comments

* Remove the unused _consume_whitespace function

Author: chrisrink10

CHANGELOG.md

@@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
  * Fixed a bug where the `quote` special form allowed more than one argument and raised an unintended exception when no argument was provided (#497)
  * Fixed a bug where compiler options specified via command-line argument or environment variable were not honored by the importer (#507)
  * Fixed a bug where private Vars from other Namespaces could be referenced if the Namespace was aliased when it was required (#514)
+ * Fixed a bug where collections with trailing end tokens separated from the collection only by a comment (#520)
 
 ## [v0.1.dev12] - 2020-01-26
 ### Added

src/basilisp/lang/reader.py

@@ -572,12 +572,6 @@ def _read_coll(
             coll.append(elem)
 
 
-def _consume_whitespace(reader: StreamReader) -> None:
-    token = reader.peek()
-    while whitespace_chars.match(token):
-        token = reader.advance()
-
-
 @_with_loc
 def _read_list(ctx: ReaderContext) -> llist.List:
     """Read a list element from the input stream."""
@@ -617,14 +611,18 @@ def __read_map_elems(ctx: ReaderContext) -> Iterable[RawReaderForm]:
     reader conditionals."""
     reader = ctx.reader
     while True:
-        if reader.peek() == "}":
+        token = reader.peek()
+        if token == "":
+            raise ctx.eof_error("Unexpected EOF in map}")
+        if whitespace_chars.match(token):
+            reader.advance()
+            continue
+        if token == "}":
             reader.next_token()
             return
         v = _read_next(ctx)
         if v is COMMENT or isinstance(v, Comment):
             continue
-        elif v is ctx.eof:
-            raise ctx.eof_error("Unexpected EOF in map")
         elif _should_splice_reader_conditional(ctx, v):
             assert isinstance(v, ReaderConditional)
             selected_feature = v.select_feature(ctx.reader_features)
@@ -1315,7 +1313,7 @@ def _read_comment(ctx: ReaderContext) -> LispReaderForm:
         token = reader.peek()
         if newline_chars.match(token):
             reader.advance()
-            return _read_next(ctx)
+            return COMMENT
         if token == "":
             return ctx.eof
         reader.advance()
@@ -1333,6 +1331,15 @@ def _read_next_consuming_comment(ctx: ReaderContext) -> RawReaderForm:
         return v
 
 
+def _read_next_consuming_whitespace(ctx: ReaderContext) -> LispReaderForm:
+    """Read the next full form from the input stream, consuming any whitespace."""
+    reader = ctx.reader
+    token = reader.peek()
+    while whitespace_chars.match(token):
+        token = reader.next_token()
+    return _read_next(ctx)
+
+
 def _read_next(ctx: ReaderContext) -> LispReaderForm:  # noqa: C901 MC0001
     """Read the next full form from the input stream."""
     reader = ctx.reader
@@ -1346,8 +1353,7 @@ def _read_next(ctx: ReaderContext) -> LispReaderForm:  # noqa: C901 MC0001
     elif begin_num_chars.match(token):
         return _read_num(ctx)
     elif whitespace_chars.match(token):
-        reader.next_token()
-        return _read_next(ctx)
+        return _read_next_consuming_whitespace(ctx)
     elif token == ":":
         return _read_kw(ctx)
     elif token == '"':

tests/basilisp/reader_test.py

@@ -376,6 +376,27 @@ def test_vector():
         1.4, kw.keyword("a"), "string"
     )
 
+    assert read_str_first("[\n]") == vec.Vector.empty()
+    assert read_str_first("[       :a\n :b\n]") == vec.v(
+        kw.keyword("a"), kw.keyword("b")
+    )
+    assert read_str_first("[:a :b\n]") == vec.v(kw.keyword("a"), kw.keyword("b"))
+    assert read_str_first("[:a :b      ]") == vec.v(kw.keyword("a"), kw.keyword("b"))
+    assert read_str_first("[\n;;comment\n]") == vec.Vector.empty()
+    assert read_str_first("[:a :b\n;;comment\n]") == vec.v(
+        kw.keyword("a"), kw.keyword("b")
+    )
+    assert read_str_first("[:a \n;;comment\n :b]") == vec.v(
+        kw.keyword("a"), kw.keyword("b")
+    )
+    assert read_str_first("[\n#_[:a :b]\n]") == vec.Vector.empty()
+    assert read_str_first("[:a :b\n#_[:a :b]\n]") == vec.v(
+        kw.keyword("a"), kw.keyword("b")
+    )
+    assert read_str_first("[:a \n#_[:a :b]\n :b]") == vec.v(
+        kw.keyword("a"), kw.keyword("b")
+    )
+
 
 def test_list():
     with pytest.raises(reader.SyntaxError):
@@ -393,6 +414,27 @@ def test_list():
     )
     assert read_str_first("(- -1 2)") == llist.l(sym.symbol("-"), -1, 2)
 
+    assert read_str_first("(\n)") == llist.List.empty()
+    assert read_str_first("(       :a\n :b\n)") == llist.l(
+        kw.keyword("a"), kw.keyword("b")
+    )
+    assert read_str_first("(:a :b\n)") == llist.l(kw.keyword("a"), kw.keyword("b"))
+    assert read_str_first("(:a :b      )") == llist.l(kw.keyword("a"), kw.keyword("b"))
+    assert read_str_first("(\n;;comment\n)") == llist.List.empty()
+    assert read_str_first("(:a :b\n;;comment\n)") == llist.l(
+        kw.keyword("a"), kw.keyword("b")
+    )
+    assert read_str_first("(:a \n;;comment\n :b)") == llist.l(
+        kw.keyword("a"), kw.keyword("b")
+    )
+    assert read_str_first("(\n#_[:a :b]\n)") == llist.List.empty()
+    assert read_str_first("(:a :b\n#_[:a :b]\n)") == llist.l(
+        kw.keyword("a"), kw.keyword("b")
+    )
+    assert read_str_first("(:a \n#_[:a :b]\n :b)") == llist.l(
+        kw.keyword("a"), kw.keyword("b")
+    )
+
 
 def test_set():
     with pytest.raises(reader.SyntaxError):
@@ -408,6 +450,27 @@ def test_set():
         kw.keyword("a"), 1, "some string"
     )
 
+    assert read_str_first("#{\n}") == lset.Set.empty()
+    assert read_str_first("#{       :a\n :b\n}") == lset.s(
+        kw.keyword("a"), kw.keyword("b")
+    )
+    assert read_str_first("#{:a :b\n}") == lset.s(kw.keyword("a"), kw.keyword("b"))
+    assert read_str_first("#{:a :b      }") == lset.s(kw.keyword("a"), kw.keyword("b"))
+    assert read_str_first("#{\n;;comment\n}") == lset.Set.empty()
+    assert read_str_first("#{:a :b\n;;comment\n}") == lset.s(
+        kw.keyword("a"), kw.keyword("b")
+    )
+    assert read_str_first("#{:a \n;;comment\n :b}") == lset.s(
+        kw.keyword("a"), kw.keyword("b")
+    )
+    assert read_str_first("#{\n#_[:a :b]\n}") == lset.Set.empty()
+    assert read_str_first("#{:a :b\n#_[:a :b]\n}") == lset.s(
+        kw.keyword("a"), kw.keyword("b")
+    )
+    assert read_str_first("#{:a \n#_[:a :b]\n :b}") == lset.s(
+        kw.keyword("a"), kw.keyword("b")
+    )
+
     with pytest.raises(reader.SyntaxError):
         read_str_first("#{:a :b :b}")
 
@@ -422,6 +485,29 @@ def test_map():
         {kw.keyword("a"): 1, kw.keyword("b"): "string"}
     )
 
+    assert read_str_first("{\n}") == lmap.Map.empty()
+    assert read_str_first("{       :a\n :b\n}") == lmap.map(
+        {kw.keyword("a"): kw.keyword("b")}
+    )
+    assert read_str_first("{:a :b\n}") == lmap.map({kw.keyword("a"): kw.keyword("b")})
+    assert read_str_first("{:a :b      }") == lmap.map(
+        {kw.keyword("a"): kw.keyword("b")}
+    )
+    assert read_str_first("{\n;;comment\n}") == lmap.Map.empty()
+    assert read_str_first("{:a :b\n;;comment\n}") == lmap.map(
+        {kw.keyword("a"): kw.keyword("b")}
+    )
+    assert read_str_first("{:a \n;;comment\n :b}") == lmap.map(
+        {kw.keyword("a"): kw.keyword("b")}
+    )
+    assert read_str_first("{\n#_[:a :b]\n}") == lmap.Map.empty()
+    assert read_str_first("{:a :b\n#_[:a :b]\n}") == lmap.map(
+        {kw.keyword("a"): kw.keyword("b")}
+    )
+    assert read_str_first("{:a \n#_[:a :b]\n :b}") == lmap.map(
+        {kw.keyword("a"): kw.keyword("b")}
+    )
+
     with pytest.raises(reader.SyntaxError):
         read_str_first("{:a 1 :b 2 :a 3}")