Add K"deferred_toplevel_eval" for cfunction/ccall/cglobal

A few special forms have a kind of "deferred top level evaluation"
semantics for some of their children:
* `@cfunction` - the function name and types
* `ccall` / `foreigncall` / `@ccall` - the type arguments and sometimes
  the function name
* `cglobal` - the function name
* `@generated` - the expression defining the generated function stub

For example, in `@ccall f()::Int`, the `Int` means "the symbol `Int` as
looked up in global scope in the module", and should fail if `Int`
refers to a local variable. Currently all three of these cases are
handled through different mechanisms with varying levels of hygiene
inconsistency and ability to warn about access to local variables.

To fix this problem, introduce the new `K"deferred_toplevel_eval"` form
which wraps an expression and preserves it as a piece of AST in the
output (rather than producing IR), but still resolves scope and hygiene.

Use this new form to remove all the special case child-index-dependent
handling of these disparate forms from the IR.

Also fixes bugs in `Base.@cfunction` hygiene where the function name
might be resolved to a global symbol in the wrong module.

Also move demo `@ccall` implementation into JuliaLowering, clean up and
fix a few things which were broken and implement the gc_safe option from
`Base.@ccall`. Makes use of deferred_toplevel_eval kind for more precise
diagnostics.

Author: c42f

src/ast.jl

@@ -517,6 +517,8 @@ function getmeta(ex::SyntaxTree, name::Symbol, default)
     isnothing(meta) ? default : get(meta, name, default)
 end
 
+name_hint(name) = CompileHints(:name_hint, name)
+
 #-------------------------------------------------------------------------------
 # Predicates and accessors working on expression trees
 

src/closure_conversion.jl

@@ -320,7 +320,7 @@ function _convert_closures(ctx::ClosureConversionCtx, ex)
         else
             access
         end
-    elseif is_leaf(ex) || k == K"inert"
+    elseif is_leaf(ex) || k == K"inert" || k == K"deferred_toplevel_eval"
         ex
     elseif k == K"="
         convert_assignment(ctx, ex)

src/desugaring.jl

@@ -1626,17 +1626,24 @@ function expand_kw_call(ctx, srcref, farg, args, kws)
     ]
 end
 
+# Expand the (sym,lib) argument to ccall/cglobal
+function expand_C_library_symbol(ctx, ex)
+    expanded = expand_forms_2(ctx, ex)
+    if kind(ex) == K"tuple"
+        expanded = @ast ctx ex [K"deferred_toplevel_eval"(meta=name_hint("function name and library expression"))
+            expanded
+        ]
+    end
+    return expanded
+end
+
 function expand_ccall(ctx, ex)
     @assert kind(ex) == K"call" && is_core_ref(ex[1], "ccall")
     if numchildren(ex) < 4
         throw(LoweringError(ex, "too few arguments to ccall"))
     end
     cfunc_name = ex[2]
     # Detect calling convention if present.
-    #
-    # Note `@ccall` also emits `Expr(:cconv, convention, nreq)`, but this is a
-    # somewhat undocumented performance workaround. Instead we should just make
-    # sure @ccall can emit foreigncall directly and efficiently.
     known_conventions = ("cdecl", "stdcall", "fastcall", "thiscall", "llvmcall")
     cconv = if any(is_same_identifier_like(ex[3], id) for id in known_conventions)
         ex[3]
@@ -1748,11 +1755,15 @@ function expand_ccall(ctx, ex)
     @ast ctx ex [K"block"
         sctx.stmts...
         [K"foreigncall"
-            expand_forms_2(ctx, cfunc_name)
-            expand_forms_2(ctx, return_type)
-            [K"call"
-                "svec"::K"core"
-                expanded_types...
+            expand_C_library_symbol(ctx, cfunc_name)
+            [K"deferred_toplevel_eval"(meta=name_hint("ccall return type"))
+                expand_forms_2(ctx, return_type)
+            ]
+            [K"deferred_toplevel_eval"(meta=name_hint("ccall argument type"))
+                [K"call"
+                    "svec"::K"core"
+                    expanded_types...
+                ]
             ]
             num_required_args::K"Integer"
             if isnothing(cconv)
@@ -1828,6 +1839,15 @@ function expand_call(ctx, ex)
     farg = ex[1]
     if is_core_ref(farg, "ccall")
         return expand_ccall(ctx, ex)
+    elseif is_core_ref(farg, "cglobal")
+        @chk numchildren(ex) in 2:3  (ex, "cglobal must have one or two arguments")
+        return @ast ctx ex [K"call"
+            ex[1]
+            expand_C_library_symbol(ctx, ex[2])
+            if numchildren(ex) == 3
+                expand_forms_2(ctx, ex[3])
+            end
+        ]
     end
     args = copy(ex[2:end])
     kws = remove_kw_args!(ctx, args)

src/eval.jl

@@ -302,6 +302,15 @@ function to_lowered_expr(mod, ex, ssa_offset=0)
         # Unpack K"Symbol" QuoteNode as `Expr(:meta)` requires an identifier here.
         args[1] = args[1].value
         Expr(:meta, args...)
+    elseif k == K"deferred_toplevel_eval"
+        @assert numchildren(ex) == 1
+        to_lowered_expr(mod, ex[1], ssa_offset)
+    elseif k == K"cfunction"
+        args = Any[to_lowered_expr(mod, e, ssa_offset) for e in children(ex)]
+        if kind(ex[2]) == K"deferred_toplevel_eval"
+            args[2] = QuoteNode(args[2])
+        end
+        Expr(:cfunction, args...)
     else
         # Allowed forms according to https://docs.julialang.org/en/v1/devdocs/ast/
         #
@@ -324,7 +333,6 @@ function to_lowered_expr(mod, ex, ssa_offset=0)
                k == K"gc_preserve_begin" ? :gc_preserve_begin :
                k == K"gc_preserve_end"   ? :gc_preserve_end   :
                k == K"foreigncall"       ? :foreigncall       :
-               k == K"cfunction"         ? :cfunction         :
                k == K"new_opaque_closure" ? :new_opaque_closure :
                nothing
         if isnothing(head)

src/kinds.jl

@@ -51,6 +51,11 @@ function _register_kinds()
             # For expr-macro compatibility; gone after expansion
             "escape"
             "hygienic_scope"
+            # An expression which will eventually be evaluated at top level in
+            # the enclosing module. Used for the callable name in @cfunction;
+            # TODO: Use this for GeneratedFunctionStub and ccall types also?
+            # may be used for ccall type arguments at some point.
+            "deferred_toplevel_eval"
             # Catch-all for additional syntax extensions without the need to
             # extend `Kind`. Known extensions include:
             #   locals, islocal

src/linear_ir.jl

@@ -3,7 +3,7 @@
 
 function is_valid_ir_argument(ctx, ex)
     k = kind(ex)
-    if is_simple_atom(ctx, ex) || k in KSet"inert top core quote"
+    if is_simple_atom(ctx, ex) || k in KSet"inert top core quote deferred_toplevel_eval"
         true
     elseif k == K"BindingId"
         binfo = lookup_binding(ctx, ex)
@@ -112,7 +112,7 @@ end
 function is_simple_arg(ctx, ex)
     k = kind(ex)
     return is_simple_atom(ctx, ex) || k == K"BindingId" || k == K"quote" || k == K"inert" ||
-           k == K"top" || k == K"core" || k == K"globalref"
+           k == K"top" || k == K"core" || k == K"globalref" || k == K"deferred_toplevel_eval"
 end
 
 function is_single_assign_var(ctx::LinearIRContext, ex)
@@ -128,7 +128,7 @@ function is_const_read_arg(ctx, ex)
     # Even if we have side effects, we know that singly-assigned
     # locals cannot be affected by them so we can inline them anyway.
     # TODO from flisp: "We could also allow const globals here"
-    return k == K"inert" || k == K"top" || k == K"core" ||
+    return k == K"inert" || k == K"top" || k == K"core" || k == K"deferred_toplevel_eval" ||
         is_simple_atom(ctx, ex) || is_single_assign_var(ctx, ex)
 end
 
@@ -167,19 +167,6 @@ function compile_args(ctx, args)
     return args_out
 end
 
-# Compile the (sym,lib) argument to ccall/cglobal
-function compile_C_library_symbol(ctx, ex)
-    if kind(ex) == K"call" && kind(ex[1]) == K"core" && ex[1].name_val == "tuple"
-        # Tuples like core.tuple(:funcname, mylib_name) are allowed and are
-        # kept inline, but may only reference globals.
-        check_no_local_bindings(ctx, ex,
-            "function name and library expression cannot reference local variables")
-        ex
-    else
-        only(compile_args(ctx, (ex,)))
-    end
-end
-
 function emit(ctx::LinearIRContext, ex)
     push!(ctx.code, ex)
     return ex
@@ -593,7 +580,7 @@ function compile(ctx::LinearIRContext, ex, needs_value, in_tail_pos)
     k = kind(ex)
     if k == K"BindingId" || is_literal(k) || k == K"quote" || k == K"inert" ||
             k == K"top" || k == K"core" || k == K"Value" || k == K"Symbol" ||
-            k == K"SourceLocation"
+            k == K"SourceLocation" || k == K"deferred_toplevel_eval"
         if in_tail_pos
             emit_return(ctx, ex)
         elseif needs_value
@@ -614,39 +601,7 @@ function compile(ctx::LinearIRContext, ex, needs_value, in_tail_pos)
         nothing
     elseif k == K"call" || k == K"new" || k == K"splatnew" || k == K"foreigncall" ||
             k == K"new_opaque_closure" || k == K"cfunction"
-        if k == K"foreigncall"
-            args = SyntaxList(ctx)
-            push!(args, compile_C_library_symbol(ctx, ex[1]))
-            # 2nd to 5th arguments of foreigncall are special. They must be
-            # left in place but cannot reference locals.
-            check_no_local_bindings(ctx, ex[2], "ccall return type cannot reference local variables")
-            for argt in children(ex[3])
-                check_no_local_bindings(ctx, argt,
-                    "ccall argument types cannot reference local variables")
-            end
-            append!(args, ex[2:5])
-            append!(args, compile_args(ctx, ex[6:end]))
-            args
-        elseif k == K"cfunction"
-            # Arguments of cfunction must be left in place except for argument
-            # 2 (fptr)
-            args = copy(children(ex))
-            args[2] = only(compile_args(ctx, args[2:2]))
-            check_no_local_bindings(ctx, ex[3],
-                "cfunction return type cannot reference local variables")
-            for arg in children(ex[4])
-                check_no_local_bindings(ctx, arg,
-                    "cfunction argument cannot reference local variables")
-            end
-        elseif k == K"call" && is_core_ref(ex[1], "cglobal")
-            args = SyntaxList(ctx)
-            push!(args, ex[1])
-            push!(args, compile_C_library_symbol(ctx, ex[2]))
-            append!(args, compile_args(ctx, ex[3:end]))
-        else
-            args = compile_args(ctx, children(ex))
-        end
-        callex = makenode(ctx, ex, k, args)
+        callex = makenode(ctx, ex, k, compile_args(ctx, children(ex)))
         if in_tail_pos
             emit_return(ctx, ex, callex)
         elseif needs_value
@@ -909,7 +864,7 @@ function compile(ctx::LinearIRContext, ex, needs_value, in_tail_pos)
 end
 
 function _remove_vars_with_isdefined_check!(vars, ex)
-    if is_leaf(ex) || is_quoted(ex)
+    if is_leaf(ex) || is_quoted(ex) || kind(ex) == K"deferred_toplevel_eval"
         return
     elseif kind(ex) == K"isdefined"
         delete!(vars, ex[1].var_id)
@@ -1017,10 +972,11 @@ function _renumber(ctx, ssa_rewrites, slot_rewrites, label_table, ex)
                 makeleaf(ctx, ex, K"globalref", binfo.name, mod=binfo.mod)
             end
         end
-    elseif k == K"meta"
+    elseif k == K"meta" || k == K"deferred_toplevel_eval"
         # Somewhat-hack for Expr(:meta, :generated, gen) which has
         # weird top-level semantics for `gen`, but we still need to translate
-        # the binding it contains to a globalref.
+        # the binding it contains to a globalref. (TODO: use
+        # deferred_toplevel_eval for this meta, somehow)
         mapchildren(ctx, ex) do e
             _renumber(ctx, ssa_rewrites, slot_rewrites, label_table, e)
         end

src/macro_expansion.jl

@@ -115,6 +115,10 @@ function Base.showerror(io::IO, exc::MacroExpansionError)
                 pos == :end     ? (lb+1:lb) :
                 error("Unknown position $pos")
     highlight(io, src.file, byterange, note=exc.msg)
+    if !isnothing(exc.err)
+        print(io, "\nCaused by:\n")
+        showerror(io, exc.err)
+    end
 end
 
 function eval_macro_name(ctx::MacroExpansionContext, mctx::MacroContext, ex::SyntaxTree)
@@ -222,6 +226,9 @@ function expand_forms_1(ctx::MacroExpansionContext, ex::SyntaxTree)
         if all(==('_'), name_str)
             @ast ctx ex ex=>K"Placeholder"
         elseif is_ccall_or_cglobal(name_str)
+            # Lower special identifiers `cglobal` and `ccall` to `K"core"`
+            # psuedo-refs very early so that cglobal and ccall can never be
+            # turned into normal bindings (eg, assigned to)
             @ast ctx ex name_str::K"core"
         else
             layerid = get(ex, :scope_layer, ctx.current_layer.id)

src/scope_analysis.jl

@@ -630,6 +630,24 @@ function init_closure_bindings!(ctx, fname)
     end
 end
 
+function find_non_global_binding(ctx, ex)
+    k = kind(ex)
+    if k == K"BindingId"
+        binfo = lookup_binding(ctx, ex.var_id)
+        if binfo.kind != :global
+            return ex
+        end
+    elseif !is_leaf(ex) && !is_quoted(ex)
+        for e in children(ex)
+            r = find_non_global_binding(ctx, e)
+            if !isnothing(r)
+                return r
+            end
+        end
+    end
+    return nothing
+end
+
 # Update ctx.bindings and ctx.lambda_bindings metadata based on binding usage
 function analyze_variables!(ctx, ex)
     k = kind(ex)
@@ -649,6 +667,13 @@ function analyze_variables!(ctx, ex)
         end
     elseif is_leaf(ex) || is_quoted(ex)
         return
+    elseif k == K"deferred_toplevel_eval"
+        badvar = find_non_global_binding(ctx, ex[1])
+        if !isnothing(badvar)
+            name_hint = getmeta(ex, :name_hint, "syntax")
+            throw(LoweringError(badvar, "$(name_hint) cannot reference local variable"))
+        end
+        return
     elseif k == K"local" || k == K"global"
         # Presence of BindingId within local/global is ignored.
         return