Add stacktrace capture to MacroExpansionError

`MacroExpansionError` has good information about error location, but
does not preserve information about the error raised by user macros
beyond the error message itself. Such information is very useful for
tooling like language servers, and stacktraces are particularly
important.

This commit adds a `stacktrace::Vector{Base.StackTraces.StackFrame}`
field to `MacroExpansionError`. New macro definitions still call the
`MacroExpansionError(ex::SyntaxTree, msg::AbstractString;
position=:all)` constructor, which internally calls `stacktrace(...)`,
so the user-facing interface remains unchanged. Additionally,
`scrub_expand_macro_stacktrace` is implemented to automatically trim
information about JL internal functions that are not useful to users.

Author: aviatesk

Project.toml

@@ -6,6 +6,9 @@ version = "1.0.0-DEV"
 [deps]
 JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4"
 
+[sources]
+JuliaSyntax = {rev = "e02f29f", url = "https://github.com/JuliaLang/JuliaSyntax.jl"}
+
 [compat]
 julia = "1"
 

src/macro_expansion.jl

@@ -79,13 +79,21 @@ struct MacroExpansionError
     ex::SyntaxTree
     msg::String
     position::Symbol
+    stacktrace::Vector{Base.StackTraces.StackFrame}
 end
 
 """
 `position` - the source position relative to the node - may be `:begin` or `:end` or `:all`
 """
 function MacroExpansionError(ex::SyntaxTree, msg::AbstractString; position=:all)
-    MacroExpansionError(nothing, ex, msg, position)
+    MacroExpansionError(nothing, ex, msg, position, scrub_expand_macro_stacktrace(stacktrace(backtrace())))
+end
+
+function scrub_expand_macro_stacktrace(stacktrace::Vector{Base.StackTraces.StackFrame})
+    idx = @something findfirst(stacktrace) do stackframe::Base.StackTraces.StackFrame
+        stackframe.func === :expand_macro && stackframe.file === Symbol(@__FILE__)
+    end error("`scrub_expand_macro_stacktrace` is expected to be called from `expand_macro`")
+    return stacktrace[1:idx-1]
 end
 
 function Base.showerror(io::IO, exc::MacroExpansionError)
@@ -113,7 +121,7 @@ function Base.showerror(io::IO, exc::MacroExpansionError)
     highlight(io, src.file, byterange, note=exc.msg)
 end
 
-function eval_macro_name(ctx, ex)
+function eval_macro_name(ctx::MacroExpansionContext, ex::SyntaxTree)
     # `ex1` might contain a nontrivial mix of scope layers so we can't just
     # `eval()` it, as it's already been partially lowered by this point.
     # Instead, we repeat the latter parts of `lower()` here.
@@ -127,7 +135,7 @@ function eval_macro_name(ctx, ex)
     eval(mod, expr_form)
 end
 
-function expand_macro(ctx, ex)
+function expand_macro(ctx::MacroExpansionContext, ex::SyntaxTree)
     @assert kind(ex) == K"macrocall"
 
     macname = ex[1]
@@ -151,9 +159,9 @@ function expand_macro(ctx, ex)
         if exc isa MacroExpansionError
             # Add context to the error.
             # TODO: Using rethrow() is kinda ugh. Is there a way to avoid it?
-            rethrow(MacroExpansionError(mctx, exc.ex, exc.msg, exc.position))
+            rethrow(MacroExpansionError(mctx, exc.ex, exc.msg, exc.position, exc.stacktrace))
         else
-            throw(MacroExpansionError(mctx, ex, "Error expanding macro", :all))
+            throw(MacroExpansionError(mctx, ex, "Error expanding macro", :all, scrub_expand_macro_stacktrace(stacktrace(catch_backtrace()))))
         end
     end
 
@@ -237,7 +245,7 @@ function expand_forms_1(ctx::MacroExpansionContext, ex::SyntaxTree)
         @chk numchildren(ex) == 1
         # TODO: Upstream should set a general flag for detecting parenthesized
         # expressions so we don't need to dig into `green_tree` here. Ugh!
-        plain_symbol = has_flags(ex, JuliaSyntax.COLON_QUOTE) && 
+        plain_symbol = has_flags(ex, JuliaSyntax.COLON_QUOTE) &&
                        kind(ex[1]) == K"Identifier" &&
                        (sr = sourceref(ex); sr isa SourceRef && kind(sr.green_tree[2]) != K"parens")
         if plain_symbol
@@ -337,4 +345,3 @@ function expand_forms_1(mod::Module, ex::SyntaxTree)
                                  ctx.current_layer)
     return ctx2, reparent(ctx2, ex2)
 end
-

test/ccall_demo.jl

@@ -105,7 +105,7 @@ function ccall_macro_lower(ex, convention, func, rettype, types, args, num_varar
         push!(roots, argi)
         push!(cargs, ast":(Base.unsafe_convert($type, $argi))")
     end
-    push!(statements, 
+    push!(statements,
           @ast ex ex [K"foreigncall"
               func
               rettype
@@ -126,5 +126,4 @@ function var"@ccall"(ctx::JuliaLowering.MacroContext, ex)
     ccall_macro_lower(ex, "ccall", ccall_macro_parse(ex)...)
 end
 
-end
-
+end # module CCall

test/macros.jl

@@ -1,6 +1,8 @@
-@testset "macros" begin
+module macros
 
-test_mod = Module()
+using JuliaLowering, Test
+
+module test_mod end
 
 JuliaLowering.include_string(test_mod, """
 module M
@@ -75,7 +77,7 @@ end
 """)
 
 @test JuliaLowering.include_string(test_mod, """
-let 
+let
     x = "`x` from outer scope"
     M.@foo x
 end
@@ -89,7 +91,7 @@ end
 
 @test !isdefined(test_mod.M, :a_global)
 @test JuliaLowering.include_string(test_mod, """
-begin 
+begin
     M.@set_a_global 42
     M.a_global
 end
@@ -133,13 +135,42 @@ M.@recursive 3
 """) == (3, (2, (1, 0)))
 
 @test let
-    ex = parsestmt(SyntaxTree, "M.@outer()", filename="foo.jl")
+    ex = JuliaLowering.parsestmt(JuliaLowering.SyntaxTree, "M.@outer()", filename="foo.jl")
     expanded = JuliaLowering.macroexpand(test_mod, ex)
-    sourcetext.(flattened_provenance(expanded[2]))
+    JuliaLowering.sourcetext.(JuliaLowering.flattened_provenance(expanded[2]))
 end == [
     "M.@outer()"
     "@inner"
     "2"
 ]
 
+JuliaLowering.include_string(test_mod, """
+f_throw(x) = throw(x)
+macro m_throw(x)
+    :(\$(f_throw(x)))
+end
+""")
+let ret = try
+        JuliaLowering.include_string(test_mod, "_never_exist = @m_throw 42")
+    catch err
+        err
+    end
+    @test ret isa JuliaLowering.MacroExpansionError
+    @test length(ret.stacktrace) == 2
+    @test ret.stacktrace[1].func === :f_throw
+    @test ret.stacktrace[2].func === Symbol("@m_throw")
 end
+
+include("ccall_demo.jl")
+@test JuliaLowering.include_string(CCall, "@ccall strlen(\"foo\"::Cstring)::Csize_t") == 3
+let ret = try
+        JuliaLowering.include_string(CCall, "@ccall strlen(\"foo\"::Cstring)")
+    catch e
+        e
+    end
+    @test ret isa JuliaLowering.MacroExpansionError
+    @test ret.msg == "Expected a return type annotation like `::T`"
+    @test any(sf->sf.func===:ccall_macro_parse, ret.stacktrace)
+end
+
+end # module macros

test/runtests.jl

@@ -3,7 +3,6 @@ using Test
 include("utils.jl")
 
 @testset "JuliaLowering.jl" begin
-
     include("syntax_graph.jl")
 
     include("ir_tests.jl")
@@ -20,11 +19,10 @@ include("utils.jl")
     include("generators.jl")
     include("import.jl")
     include("loops.jl")
-    include("macros.jl")
+    @testset "macros" include("macros.jl")
     include("misc.jl")
     include("modules.jl")
     include("quoting.jl")
     include("scopes.jl")
     include("typedefs.jl")
-
 end