Merge pull request #81 from JuliaLang/sp/typed-comprehension-begin-handling

Correctly handle begin as kw in typed comprehension

Author: pfitzseb

README.md

@@ -565,6 +565,25 @@ name of compatibility, perhaps with a warning.)
   arises from `(set! pred char-hex?)` in `parse-number` accepting hex exponent
   digits, all of which are detected as invalid except for a trailing `f` when
   processed by `isnumtok_base`.
+* `begin` and `end` are not parsed as keywords when indexing. Typed comprehensions
+  initially look the same, but can be distinguished from indexing once we handle
+  a `for` token; it is safe to treat `begin` and `end` as keywords afterwards. The
+  reference parser *only* handles this well when there's a newline before `for`:
+  ```julia
+  Any[foo(i)
+      for i in x if begin
+          true
+      end
+  ]
+  ```
+  works, while
+  ```julia
+  Any[foo(i) for i in x if begin
+          true
+      end
+  ]
+  ```
+  does not. JuliaSyntax handles both cases.
 
 ## Parsing / AST oddities and warts
 

src/parser.jl

@@ -2526,6 +2526,7 @@ end
 # then reconstructing the nested flattens and generators when converting to Expr.
 #
 # [x for a = as for b = bs if cond1 for c = cs if cond2] ==> (comprehension (flatten x (= a as) (filter (= b bs) cond1) (filter (= c cs) cond2)))
+# [x for a = as if begin cond2 end]  =>  (comprehension (generator x (filter (= a as) (block cond2))))
 #
 # flisp: parse-generator
 function parse_generator(ps::ParseState, mark, flatten=false)
@@ -2562,7 +2563,8 @@ end
 # flisp: parse-comprehension
 function parse_comprehension(ps::ParseState, mark, closer)
     ps = ParseState(ps, whitespace_newline=true,
-                    space_sensitive=false)
+                    space_sensitive=false,
+                    end_symbol=false)
     parse_generator(ps, mark)
     bump_closing_token(ps, closer)
     return (K"comprehension", EMPTY_FLAGS)

test/parser.jl

@@ -620,6 +620,7 @@ tests = [
         "[x \n\n for a in as]"  =>  "(comprehension (generator x (= a as)))"
         # parse_generator
         "[x for a = as for b = bs if cond1 for c = cs if cond2]"  =>  "(comprehension (flatten x (= a as) (filter (= b bs) cond1) (filter (= c cs) cond2)))"
+        "[x for a = as if begin cond2 end]"  =>  "(comprehension (generator x (filter (= a as) (block cond2))))"
         "[(x)for x in xs]"  =>  "(comprehension (generator x (error-t) (= x xs)))"
         "(a for x in xs if cond)"  =>  "(generator a (filter (= x xs) cond))"
         "(xy for x in xs for y in ys)"  =>  "(flatten xy (= x xs) (= y ys))"