Merge pull request #220 from JuliaLang/c42f/bump-invisible-cleanup

Various AST cleanups related to removing invisible tokens

Author: c42f

src/expr.jl

@@ -278,6 +278,7 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
             pushfirst!(args[2].args, loc)
         end
     elseif headsym === :module
+        pushfirst!(args, !has_flags(node, BARE_MODULE_FLAG))
         pushfirst!(args[3].args, loc)
     elseif headsym == :inert || (headsym == :quote && length(args) == 1 &&
                  !(a1 = only(args); a1 isa Expr || a1 isa QuoteNode ||
@@ -303,6 +304,13 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
             args[1] = Expr(headsym, args[1].args...)
             headsym = :const
         end
+    elseif headsym === :return && isempty(args)
+        push!(args, nothing)
+    elseif headsym === :juxtapose
+        headsym = :call
+        pushfirst!(args, :*)
+    elseif headsym === :struct
+        pushfirst!(args, has_flags(node, MUTABLE_FLAG))
     end
     return Expr(headsym, args...)
 end

src/hooks.jl

@@ -62,7 +62,7 @@ function _incomplete_tag(n::SyntaxNode)
     elseif kp in KSet"for while function if"
         return i == 1 ? :other : :block
     elseif kp in KSet"module struct"
-        return i == 2 ? :other : :block
+        return i == 1 ? :other : :block
     elseif kp == K"do"
         return i < 3  ? :other : :block
     else

src/kinds.jl

@@ -83,7 +83,6 @@ const _kind_names =
         "CmdString"
         "true"
         "false"
-        "nothing"  # A literal `nothing`
     "END_LITERAL"
 
     "BEGIN_DELIMITERS"
@@ -880,6 +879,7 @@ const _kind_names =
         "comparison"
         "curly"
         "inert"          # QuoteNode; not quasiquote
+        "juxtapose"      # Numeric juxtaposition like 2x
         "string"         # A string interior node (possibly containing interpolations)
         "cmdstring"      # A cmd string node (containing delimiters plus string)
         "char"           # A char string node (containing delims + char data)

src/parse_stream.jl

@@ -5,27 +5,38 @@
 # TODO: Use `primitive type SyntaxFlags 16 end` rather than an alias?
 const RawFlags = UInt16
 const EMPTY_FLAGS = RawFlags(0)
-# Applied to tokens which are syntax trivia after parsing
+
+# Set for tokens or ranges which are syntax trivia after parsing
 const TRIVIA_FLAG = RawFlags(1<<0)
 
-# Record whether operators are dotted
+# Token flags - may be set for operator kinded tokens
+# Operator is dotted
 const DOTOP_FLAG = RawFlags(1<<1)
-# Record whether operator has a suffix
-const SUFFIXED_FLAG        = RawFlags(1<<2)
+# Operator has a suffix
+const SUFFIXED_FLAG = RawFlags(1<<2)
 
+# Set for K"call", K"dotcall" or any syntactic operator heads
 # Distinguish various syntaxes which are mapped to K"call"
 const PREFIX_CALL_FLAG = RawFlags(0<<3)
 const INFIX_FLAG       = RawFlags(1<<3)
 const PREFIX_OP_FLAG   = RawFlags(2<<3)
 const POSTFIX_OP_FLAG  = RawFlags(3<<3)
 
-# The next two bits could overlap with the previous two if necessary
-# Set when kind == K"String" was triple-delimited as with """ or ```
+# The following flags are quite head-specific and may overlap
+
+# Set when K"string" or K"cmdstring" was triple-delimited as with """ or ```
 const TRIPLE_STRING_FLAG = RawFlags(1<<5)
-# Set when a string or identifier needs "raw string" unescaping
+# Set when a K"string", K"cmdstring" or K"Identifier" needs raw string unescaping
 const RAW_STRING_FLAG = RawFlags(1<<6)
 
-const PARENS_FLAG = RawFlags(1<<7)
+# Set for K"tuple", K"block" or K"macrocall" which are delimited by parentheses
+const PARENS_FLAG = RawFlags(1<<5)
+
+# Set for K"struct" when mutable
+const MUTABLE_FLAG = RawFlags(1<<5)
+
+# Set for K"module" when it's not bare (`module`, not `baremodule`)
+const BARE_MODULE_FLAG = RawFlags(1<<5)
 
 # Flags holding the dimension of an nrow or other UInt8 not held in the source
 const NUMERIC_FLAGS = RawFlags(RawFlags(0xff)<<8)
@@ -78,10 +89,18 @@ function untokenize(head::SyntaxHead; unique=true, include_flag_suff=true)
         is_infix_op_call(head)   && (str = str*"-i")
         is_prefix_op_call(head)  && (str = str*"-pre")
         is_postfix_op_call(head) && (str = str*"-post")
-        has_flags(head, TRIPLE_STRING_FLAG) && (str = str*"-s")
-        has_flags(head, RAW_STRING_FLAG) && (str = str*"-r")
-        has_flags(head, PARENS_FLAG) && (str = str*"-p")
-        is_suffixed(head) && (str = str*"-S")
+
+        if kind(head) in KSet"string cmdstring Identifier"
+            has_flags(head, TRIPLE_STRING_FLAG) && (str = str*"-s")
+            has_flags(head, RAW_STRING_FLAG) && (str = str*"-r")
+        elseif kind(head) in KSet"tuple block macrocall"
+            has_flags(head, PARENS_FLAG) && (str = str*"-p")
+        elseif kind(head) == K"struct"
+            has_flags(head, MUTABLE_FLAG) && (str = str*"-mut")
+        elseif kind(head) == K"module"
+            has_flags(head, BARE_MODULE_FLAG) && (str = str*"-bare")
+        end
+        is_suffixed(head) && (str = str*"-suf")
         n = numeric_flags(head)
         n != 0 && (str = str*"-"*string(n))
     end

src/parser.jl

@@ -763,7 +763,7 @@ function parse_comparison(ps::ParseState, subtype_comparison=false)
     mark = position(ps)
     if subtype_comparison && is_reserved_word(peek(ps))
         # Recovery
-        # struct try end  ==>  (struct false (error (try)) (block))
+        # struct try end  ==>  (struct (error (try)) (block))
         name = untokenize(peek(ps))
         bump(ps)
         emit(ps, mark, K"error", error="Invalid type name `$name`")
@@ -1105,13 +1105,13 @@ function is_juxtapose(ps, prev_k, t)
     !is_initial_reserved_word(ps, k)
 end
 
-# Juxtoposition. Ugh!
+# Juxtoposition. Ugh! But so useful for units and Field identities like `im`
 #
-# 2x       ==>  (call-i 2 * x)
-# 2(x)     ==>  (call-i 2 * x)
-# (2)(3)x  ==>  (call-i 2 * 3 x)
-# (x-1)y   ==>  (call-i (call-i x - 1) * y)
-# x'y      ==>  (call-i (call-post x ') * y)
+# 2x       ==>  (juxtapose 2 x)
+# 2(x)     ==>  (juxtapose 2 x)
+# (2)(3)x  ==>  (juxtapose 2 3 x)
+# (x-1)y   ==>  (juxtapose (call-i x - 1) y)
+# x'y      ==>  (juxtapose (call-post x ') y)
 #
 # flisp: parse-juxtapose
 function parse_juxtapose(ps::ParseState)
@@ -1124,15 +1124,12 @@ function parse_juxtapose(ps::ParseState)
         if !is_juxtapose(ps, prev_kind, t)
             break
         end
-        if n_terms == 1
-            bump_invisible(ps, K"*")
-        end
         if prev_kind == K"string" || is_string_delim(t)
             # issue #20575
             #
-            # "a""b"  ==>  (call-i (string "a") * (error-t) (string "b"))
-            # "a"x    ==>  (call-i (string "a") * (error-t) x)
-            # "$y"x   ==>  (call-i (string (string y)) * (error-t) x)
+            # "a""b"  ==>  (juxtapose (string "a") (error-t) (string "b"))
+            # "a"x    ==>  (juxtapose (string "a") (error-t) x)
+            # "$y"x   ==>  (juxtapose (string (string y)) (error-t) x)
             bump_invisible(ps, K"error", TRIVIA_FLAG,
                            error="cannot juxtapose string literal")
         end
@@ -1144,7 +1141,7 @@ function parse_juxtapose(ps::ParseState)
         n_terms += 1
     end
     if n_terms > 1
-        emit(ps, mark, K"call", INFIX_FLAG)
+        emit(ps, mark, K"juxtapose")
     end
 end
 
@@ -1767,8 +1764,8 @@ function parse_struct_field(ps::ParseState)
     parse_eq(ps)
     if const_field
         # Const fields https://github.com/JuliaLang/julia/pull/43305
-        #v1.8: struct A const a end  ==>  (struct false A (block (const x)))
-        #v1.7: struct A const a end  ==>  (struct false A (block (error (const x))))
+        #v1.8: struct A const a end  ==>  (struct A (block (const x)))
+        #v1.7: struct A const a end  ==>  (struct A (block (error (const x))))
         emit(ps, mark, K"const")
         min_supported_version(v"1.8", ps, mark, "`const` struct field")
     end
@@ -1823,23 +1820,21 @@ function parse_resword(ps::ParseState)
         emit(ps, mark, K"for")
     elseif word == K"let"
         bump(ps, TRIVIA_FLAG)
-        if peek(ps) ∉ KSet"NewlineWs ;"
-            # let x=1\n end   ==>  (let (block (= x 1)) (block))
-            # let x=1 ; end   ==>  (let (block (= x 1)) (block))
-            m = position(ps)
-            n_subexprs = parse_comma_separated(ps, parse_eq_star)
-            kb = peek_behind(ps).kind
+        m = position(ps)
+        if peek(ps) in KSet"NewlineWs ;"
+            # let end           ==>  (let (block) (block))
+            # let ; end         ==>  (let (block) (block))
+            # let ; body end    ==>  (let (block) (block body))
+        else
+            # let x=1\n end     ==>  (let (block (= x 1)) (block))
+            # let x=1 ; end     ==>  (let (block (= x 1)) (block))
             # let x::1 ; end    ==>  (let (block (::-i x 1)) (block))
             # let x ; end       ==>  (let (block x) (block))
             # let x=1,y=2 ; end ==>  (let (block (= x 1) (= y 2) (block)))
             # let x+=1 ; end    ==>  (let (block (+= x 1)) (block))
-            emit(ps, m, K"block")
-        else
-            # let end           ==>  (let (block) (block))
-            # let ; end         ==>  (let (block) (block))
-            # let ; body end    ==>  (let (block) (block body))
-            bump_invisible(ps, K"block")
+            parse_comma_separated(ps, parse_eq_star)
         end
+        emit(ps, m, K"block")
         k = peek(ps)
         if k in KSet"NewlineWs ;"
             bump(ps, TRIVIA_FLAG)
@@ -1934,24 +1929,23 @@ function parse_resword(ps::ParseState)
         bump_closing_token(ps, K"end")
         emit(ps, mark, K"abstract")
     elseif word in KSet"struct mutable"
-        # struct A <: B \n a::X \n end  ==>  (struct false (<: A B) (block (::-i a X)))
-        # struct A \n a \n b \n end  ==>  (struct false A (block a b))
-        #v1.7: struct A const a end  ==>  (struct false A (block (error (const a))))
-        #v1.8: struct A const a end  ==>  (struct false A (block (const a)))
-        if word == K"mutable"
-            # mutable struct A end  ==>  (struct true A (block))
+        # struct A <: B \n a::X \n end  ==>  (struct (<: A B) (block (::-i a X)))
+        # struct A \n a \n b \n end  ==>  (struct A (block a b))
+        #v1.7: struct A const a end  ==>  (struct A (block (error (const a))))
+        #v1.8: struct A const a end  ==>  (struct A (block (const a)))
+        is_mut = word == K"mutable"
+        if is_mut
+            # mutable struct A end  ==>  (struct-mut A (block))
             bump(ps, TRIVIA_FLAG)
-            bump_invisible(ps, K"true")
         else
-            # struct A end  ==>  (struct false A (block))
-            bump_invisible(ps, K"false")
+            # struct A end  ==>  (struct A (block))
         end
         @check peek(ps) == K"struct"
         bump(ps, TRIVIA_FLAG)
         parse_subtype_spec(ps)
         parse_block(ps, parse_struct_field)
         bump_closing_token(ps, K"end")
-        emit(ps, mark, K"struct")
+        emit(ps, mark, K"struct", is_mut ? MUTABLE_FLAG : EMPTY_FLAGS)
     elseif word == K"primitive"
         # primitive type A 32 end             ==> (primitive A 32)
         # primitive type A 32 ; end           ==> (primitive A 32)
@@ -1973,9 +1967,8 @@ function parse_resword(ps::ParseState)
         bump(ps, TRIVIA_FLAG)
         k = peek(ps)
         if k == K"NewlineWs" || is_closing_token(ps, k)
-            # return\nx   ==>  (return nothing)
-            # return)     ==>  (return nothing)
-            bump_invisible(ps, K"nothing")
+            # return\nx   ==>  (return)
+            # return)     ==>  (return)
         else
             # return x    ==>  (return x)
             # return x,y  ==>  (return (tuple x y))
@@ -1992,22 +1985,22 @@ function parse_resword(ps::ParseState)
                     error="unexpected token after $(untokenize(word))")
         end
     elseif word in KSet"module baremodule"
-        # module A end  ==> (module true A (block))
-        # baremodule A end ==> (module false A (block))
+        # module A end  ==> (module A (block))
+        # baremodule A end ==> (module-bare A (block))
         bump(ps, TRIVIA_FLAG)
-        bump_invisible(ps, (word == K"module") ? K"true" : K"false")
         if is_reserved_word(peek(ps))
-            # module do \n end  ==>  (module true (error do) (block))
+            # module do \n end  ==>  (module (error do) (block))
             bump(ps, error="Invalid module name")
         else
-            # module $A end  ==>  (module true ($ A) (block))
+            # module $A end  ==>  (module ($ A) (block))
             parse_unary_prefix(ps)
         end
-        # module A \n a \n b \n end  ==>  (module true A (block a b))
-        # module A \n "x"\na \n end  ==>  (module true A (block (doc (string "x") a)))
+        # module A \n a \n b \n end  ==>  (module A (block a b))
+        # module A \n "x"\na \n end  ==>  (module A (block (doc (string "x") a)))
         parse_block(ps, parse_docstring)
         bump_closing_token(ps, K"end")
-        emit(ps, mark, K"module")
+        emit(ps, mark, K"module",
+             word == K"baremodule" ? BARE_MODULE_FLAG : EMPTY_FLAGS)
     elseif word == K"export"
         # export a         ==>  (export a)
         # export @a        ==>  (export @a)

src/syntax_tree.jl

@@ -107,8 +107,6 @@ function SyntaxNode(source::SourceFile, raw::GreenNode{SyntaxHead}, position::In
             isempty(val_range)  ?
                 Symbol(untokenize(k)) : # synthetic invisible tokens
                 Symbol(normalize_identifier(val_str))
-        elseif k == K"nothing"
-            nothing
         elseif k == K"error"
             ErrorVal()
         elseif k == K"MacroName"

test/expr.jl

@@ -296,10 +296,34 @@
                  Expr(:block, LineNumberNode(1), :z))
     end
 
+    @testset "juxtapose" begin
+        @test parse(Expr, "2x") == Expr(:call, :*, 2, :x)
+        @test parse(Expr, "(2)(3)x") == Expr(:call, :*, 2, 3, :x)
+    end
+
     @testset "Core.@doc" begin
         @test parse(Expr, "\"x\" f") ==
             Expr(:macrocall, GlobalRef(Core, Symbol("@doc")), LineNumberNode(1), "x", :f)
         @test parse(Expr, "\n\"x\" f") ==
             Expr(:macrocall, GlobalRef(Core, Symbol("@doc")), LineNumberNode(2), "x", :f)
     end
+
+    @testset "return" begin
+        @test parse(Expr, "return x") == Expr(:return, :x)
+        @test parse(Expr, "return")  == Expr(:return, nothing)
+    end
+
+    @testset "struct" begin
+        @test parse(Expr, "struct A end") ==
+            Expr(:struct, false, :A, Expr(:block, LineNumberNode(1)))
+        @test parse(Expr, "mutable struct A end") ==
+            Expr(:struct, true, :A, Expr(:block, LineNumberNode(1)))
+    end
+
+    @testset "module" begin
+        @test parse(Expr, "module A end") ==
+            Expr(:module, true,  :A, Expr(:block, LineNumberNode(1), LineNumberNode(1)))
+        @test parse(Expr, "baremodule A end") ==
+            Expr(:module, false, :A, Expr(:block, LineNumberNode(1), LineNumberNode(1)))
+    end
 end

test/hooks.jl

@@ -38,7 +38,6 @@
 
     @testset "Expr(:incomplete)" begin
         JuliaSyntax.enable_in_core!()
-
         @test Meta.isexpr(Meta.parse("[x"), :incomplete)
 
         for (str, tag) in [