Random lowering bugfix batch 2 (#45)

* fix showing MacroExpansionError when sourceref isa LineNumberNode
* Support K"gc_preserve" (Note that this lowering may change in JuliaLang/julia#59129)
* Support `Expr(:isglobal)`
* Support K"cmdstring" (just convert it to a macrocall)
* Delete two-arg dot form check (The number of forms isn't bounded)
* Don't fail on `Expr(:inbounds)`
* List known meta forms + improve compat.jl errors
* Turn off flisp fallback for now

---------

Co-authored-by: Claire Foster <[email protected]>

Author: mlechu

src/compat.jl

@@ -243,7 +243,7 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
         elseif a1 isa Function
             # pass
         else
-            error("Unknown macrocall form $(sprint(dump, e))")
+            error("Unknown macrocall form at $src: $(sprint(dump, e))")
             @assert false
         end
     elseif e.head === Symbol("'")
@@ -258,9 +258,6 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
             child_exprs = pushfirst!(tuple_exprs, e.args[1])
         elseif a2 isa QuoteNode && a2.value isa Symbol
             child_exprs[2] = a2.value
-        elseif a2 isa Expr && a2.head === :MacroName
-        else
-            @error "Unknown 2-arg dot form" e
         end
     elseif e.head === :for
         @assert nargs === 2
@@ -420,8 +417,15 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
                 setmeta!(SyntaxTree(graph, st_id); nospecialize=true)
                 return st_id, src
             end
+        elseif e.args[1] in (:inline, :noinline, :generated, :generated_only,
+                             :max_methods, :optlevel, :toplevel, :push_loc, :pop_loc,
+                             :aggressive_constprop, :specialize, :compile, :infer,
+                             :nospecializeinfer)
+            # TODO: Some need to be handled in lowering
+            child_exprs[1] = Expr(:quoted_symbol, e.args[1])
         else
-            @assert nargs === 1
+            # Can't throw a hard error; it is explicitly tested that meta can take arbitrary keys.
+            @error("Unknown meta form at $src: `$e`\n$(sprint(dump, e))")
             child_exprs[1] = Expr(:quoted_symbol, e.args[1])
         end
     elseif e.head === :scope_layer 
@@ -436,11 +440,16 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
         st_k = e.head === :symbolicgoto ? K"symbolic_label" : K"symbolic_goto"
         st_attrs[:name_val] = string(e.args[1])
         child_exprs = nothing
-    elseif e.head === :inline || e.head === :noinline
+    elseif e.head in (:inline, :noinline)
         @assert nargs === 1 && e.args[1] isa Bool
         # TODO: JuliaLowering doesn't accept this (non-:meta) form yet
         st_k = K"TOMBSTONE"
         child_exprs = nothing
+    elseif e.head === :inbounds
+        @assert nargs === 1 && typeof(e.args[1]) in (Symbol, Bool)
+        # TODO: JuliaLowering doesn't accept this form yet
+        st_k = K"TOMBSTONE"
+        child_exprs = nothing
     elseif e.head === :core
         @assert nargs === 1
         @assert e.args[1] isa Symbol
@@ -484,7 +493,7 @@ function _insert_convert_expr(@nospecialize(e), graph::SyntaxGraph, src::SourceA
     # Throw if this script isn't complete.  Finally, insert a new node into the
     # graph and recurse on child_exprs
     if st_k === K"None"
-        error("Unknown expr head `$(e.head)`\n$(sprint(dump, e))")
+        error("Unknown expr head at $src: `$(e.head)`\n$(sprint(dump, e))")
     end
 
     st_id = _insert_tree_node(graph, st_k, src, st_flags; st_attrs...)

src/desugaring.jl

@@ -4544,6 +4544,15 @@ function expand_forms_2(ctx::DesugaringContext, ex::SyntaxTree, docs=nothing)
         ]
     elseif k == K"inert"
         ex
+    elseif k == K"gc_preserve"
+        s = ssavar(ctx, ex)
+        r = ssavar(ctx, ex)
+        @ast ctx ex [K"block"
+            s := [K"gc_preserve_begin" children(ex)[2:end]...]
+            r := expand_forms_2(ctx, children(ex)[1])
+            [K"gc_preserve_end" s]
+            r
+        ]
     elseif k == K"&"
         throw(LoweringError(ex, "invalid syntax"))
     elseif k == K"$"

src/hooks.jl

@@ -25,10 +25,14 @@ function core_lowering_hook(@nospecialize(code), mod::Module,
         ex = to_lowered_expr(mod, st5)
         return Core.svec(ex, st5, ctx5)
     catch exc
-        @error("JuliaLowering failed — falling back to flisp!",
-               exception=(exc,catch_backtrace()),
-               code=code, file=file, line=line, mod=mod)
-        return Base.fl_lower(code, mod, file, line, world, warn)
+        @info("JuliaLowering threw given input:", code=code, st0=st0, file=file, line=line, mod=mod)
+        rethrow(exc)
+
+        # TODO: Re-enable flisp fallback once we're done collecting errors
+        # @error("JuliaLowering failed — falling back to flisp!",
+        #        exception=(exc,catch_backtrace()),
+        #        code=code, file=file, line=line, mod=mod)
+        # return Base.fl_lower(code, mod, file, line, world, warn)
     end
 end
 

src/kinds.jl

@@ -11,6 +11,7 @@ function _register_kinds()
             # Flag for @generated parts of a functon
             "generated"
             # Temporary rooting of identifiers (GC.@preserve)
+            "gc_preserve"
             "gc_preserve_begin"
             "gc_preserve_end"
             # A literal Julia value of any kind, as might be inserted into the
@@ -61,7 +62,7 @@ function _register_kinds()
             "static_eval"
             # Catch-all for additional syntax extensions without the need to
             # extend `Kind`. Known extensions include:
-            #   locals, islocal
+            #   locals, islocal, isglobal
             # The content of an assertion is not considered to be quoted, so
             # use K"Symbol" or K"inert" inside where necessary.
             "extension"

src/macro_expansion.jl

@@ -118,14 +118,18 @@ function Base.showerror(io::IO, exc::MacroExpansionError)
     # * How to deal with highlighting trivia? Could provide a token kind or
     #   child position within the raw tree? How to abstract this??
     src = sourceref(exc.ex)
-    fb = first_byte(src)
-    lb = last_byte(src)
-    pos = exc.position
-    byterange = pos == :all     ? (fb:lb)   :
-                pos == :begin   ? (fb:fb-1) :
-                pos == :end     ? (lb+1:lb) :
-                error("Unknown position $pos")
-    highlight(io, src.file, byterange, note=exc.msg)
+    if src isa LineNumberNode
+        highlight(io, src, note=exc.msg)
+    else
+        fb = first_byte(src)
+        lb = last_byte(src)
+        pos = exc.position
+        byterange = pos == :all     ? (fb:lb)   :
+            pos == :begin   ? (fb:fb-1) :
+            pos == :end     ? (lb+1:lb) :
+            error("Unknown position $pos")
+        highlight(io, src.file, byterange, note=exc.msg)
+    end
     if !isnothing(exc.err)
         print(io, "\nCaused by:\n")
         showerror(io, exc.err)
@@ -238,7 +242,10 @@ function expand_macro(ctx, ex)
     else
         # Compat: attempt to invoke an old-style macro if there's no applicable
         # method for new-style macro arguments.
-        macro_loc = source_location(LineNumberNode, ex)
+        macro_loc = let loc = source_location(LineNumberNode, ex)
+            # Some macros, e.g. @cmd, don't play nicely with file == nothing
+            isnothing(loc.file) ? LineNumberNode(loc.line, :none) : loc
+        end
         macro_args = Any[macro_loc, current_layer(ctx).mod]
         for arg in raw_args
             # For hygiene in old-style macros, we omit any additional scope
@@ -388,6 +395,10 @@ function expand_forms_1(ctx::MacroExpansionContext, ex::SyntaxTree)
             e2 = @ast ctx e2 e2=>K"Symbol"
         end
         @ast ctx ex [K"." expand_forms_1(ctx, ex[1]) e2]
+    elseif k == K"cmdstring"
+        @chk numchildren(ex) == 1
+        e2 = @ast ctx ex [K"macrocall" "@cmd"::K"core" ex[1]]
+        expand_macro(ctx, e2)
     elseif (k == K"call" || k == K"dotcall")
         # Do some initial desugaring of call and dotcall here to simplify
         # the later desugaring pass

src/scope_analysis.jl

@@ -494,6 +494,16 @@ function _resolve_scopes(ctx, ex::SyntaxTree)
             id = lookup_var(ctx, NameKey(ex[2]))
             islocal = !isnothing(id) && var_kind(ctx, id) != :global
             @ast ctx ex islocal::K"Bool"
+        elseif etype == "isglobal"
+            e2 = ex[2]
+            @chk kind(e2) in KSet"Identifier Placeholder"
+            isglobal = if kind(e2) == K"Identifier"
+                id = lookup_var(ctx, NameKey(e2))
+                isnothing(id) || var_kind(ctx, id) == :global
+            else
+                false
+            end
+            @ast ctx ex isglobal::K"Bool"
         elseif etype == "locals"
             stmts = SyntaxList(ctx)
             locals_dict = ssavar(ctx, ex, "locals_dict")

test/macros.jl

@@ -239,6 +239,13 @@ MacroExpansionError while expanding @oldstyle_error in module Main.macros.test_m
 Caused by:
 Some error in old style macro"""
 
+@test sprint(
+    showerror,
+    JuliaLowering.MacroExpansionError(
+        JuliaLowering.expr_to_syntaxtree(:(foo), LineNumberNode(1)),
+        "fake error")) ==
+            "MacroExpansionError:\n#= line 1 =# - fake error"
+
 # Old-style macros returning non-Expr values
 Base.eval(test_mod, :(
 macro oldstyle_non_Expr()
@@ -293,4 +300,33 @@ end
     MethodError: no method matching var"@sig_mismatch"(::JuliaLowering.MacroContext, ::JuliaLowering.SyntaxTree""")
 end
 
+@testset "old macros producing exotic expr heads" begin
+    @test JuliaLowering.include_string(test_mod, """
+    let # example from @preserve docstring
+        x = Ref{Int}(101)
+        p = Base.unsafe_convert(Ptr{Int}, x)
+        GC.@preserve x unsafe_load(p)
+    end""") === 101 # Expr(:gc_preserve)
+
+    # only invokelatest produces :isglobal now, so MWE here
+    Base.eval(test_mod, :(macro isglobal(x); esc(Expr(:isglobal, x)); end))
+    @test JuliaLowering.include_string(test_mod, """
+    some_global = 1
+    function isglobal_chk(some_arg)
+       local some_local = 1
+       (@isglobal(some_undefined), @isglobal(some_global), @isglobal(some_arg), @isglobal(some_local))
+    end
+    isglobal_chk(1)
+    """) === (true, true, false, false)
+    # with K"Placeholder"s
+    @test JuliaLowering.include_string(test_mod, """
+    __ = 1
+    function isglobal_chk(___)
+       local ____ = 1
+       (@isglobal(_), @isglobal(__), @isglobal(___), @isglobal(____))
+    end
+    isglobal_chk(1)
+    """) === (false, false, false, false)
+end
+
 end # module macros

test/macros_ir.jl

@@ -136,3 +136,14 @@ Caused by:
 UndefVarError: `@m_not_exist` not defined in `Main.TestMod`
 Suggestion: check for spelling errors or missing imports.
 
+########################################
+# Simple cmdstring
+`echo 1`
+#---------------------
+1   Base.cmd_gen
+2   (call core.tuple "echo")
+3   (call core.tuple "1")
+4   (call core.tuple %₂ %₃)
+5   (call %₁ %₄)
+6   (return %₅)
+