Add K"deferred_toplevel_eval" for cfunction name

A few special forms have a kind of "deferred top level evaluation"
semantics for some of their children:
* `@cfunction` - the function name and types
* `ccall` / `foreigncall` - the type arguments
* `@generated` - the expression defining the generated function stub

For example, in `@ccall f()::Int`, the `Int` means "the symbol `Int` as
looked up in global scope in the module", and should fail if `Int`
refers to a local variable. Currently all three of these cases are
handled through different mechanisms with varying levels of hygiene
inconsistency and ability to warn about access to local variables.

To fix this problem, introduce the new `K"deferred_toplevel_eval"` form
which wraps an expression and preserves it as a piece of AST in the
output (rather than producing IR), but still resolves scope and hygiene.

Author: c42f

src/closure_conversion.jl

@@ -320,7 +320,7 @@ function _convert_closures(ctx::ClosureConversionCtx, ex)
         else
             access
         end
-    elseif is_leaf(ex) || k == K"inert"
+    elseif is_leaf(ex) || k == K"inert" || k == K"deferred_toplevel_eval"
         ex
     elseif k == K"="
         convert_assignment(ctx, ex)

src/eval.jl

@@ -302,6 +302,9 @@ function to_lowered_expr(mod, ex, ssa_offset=0)
         # Unpack K"Symbol" QuoteNode as `Expr(:meta)` requires an identifier here.
         args[1] = args[1].value
         Expr(:meta, args...)
+    elseif k == K"deferred_toplevel_eval"
+        @assert numchildren(ex) == 1
+        QuoteNode(to_lowered_expr(mod, ex[1], ssa_offset))
     else
         # Allowed forms according to https://docs.julialang.org/en/v1/devdocs/ast/
         #

src/kinds.jl

@@ -54,6 +54,11 @@ function _register_kinds()
             # For expr-macro compatibility; gone after expansion
             "escape"
             "hygienic_scope"
+            # An expression which will eventually be evaluated at top level in
+            # the enclosing module. Used for the callable name in @cfunction;
+            # TODO: Use this for GeneratedFunctionStub and ccall types also?
+            # may be used for ccall type arguments at some point.
+            "deferred_toplevel_eval"
             # Catch-all for additional syntax extensions without the need to
             # extend `Kind`. Known extensions include:
             #   locals, islocal

src/linear_ir.jl

@@ -3,7 +3,7 @@
 
 function is_valid_ir_argument(ctx, ex)
     k = kind(ex)
-    if is_simple_atom(ctx, ex) || k in KSet"inert top core quote"
+    if is_simple_atom(ctx, ex) || k in KSet"inert top core quote deferred_toplevel_eval"
         true
     elseif k == K"BindingId"
         binfo = lookup_binding(ctx, ex)
@@ -112,7 +112,7 @@ end
 function is_simple_arg(ctx, ex)
     k = kind(ex)
     return is_simple_atom(ctx, ex) || k == K"BindingId" || k == K"quote" || k == K"inert" ||
-           k == K"top" || k == K"core" || k == K"globalref"
+           k == K"top" || k == K"core" || k == K"globalref" || k == K"deferred_toplevel_eval"
 end
 
 function is_single_assign_var(ctx::LinearIRContext, ex)
@@ -128,7 +128,7 @@ function is_const_read_arg(ctx, ex)
     # Even if we have side effects, we know that singly-assigned
     # locals cannot be affected by them so we can inline them anyway.
     # TODO from flisp: "We could also allow const globals here"
-    return k == K"inert" || k == K"top" || k == K"core" ||
+    return k == K"inert" || k == K"top" || k == K"core" || k == K"deferred_toplevel_eval" ||
         is_simple_atom(ctx, ex) || is_single_assign_var(ctx, ex)
 end
 
@@ -593,7 +593,7 @@ function compile(ctx::LinearIRContext, ex, needs_value, in_tail_pos)
     k = kind(ex)
     if k == K"BindingId" || is_literal(k) || k == K"quote" || k == K"inert" ||
             k == K"top" || k == K"core" || k == K"Value" || k == K"Symbol" ||
-            k == K"SourceLocation"
+            k == K"SourceLocation" || k == K"deferred_toplevel_eval"
         if in_tail_pos
             emit_return(ctx, ex)
         elseif needs_value
@@ -909,7 +909,7 @@ function compile(ctx::LinearIRContext, ex, needs_value, in_tail_pos)
 end
 
 function _remove_vars_with_isdefined_check!(vars, ex)
-    if is_leaf(ex) || is_quoted(ex)
+    if is_leaf(ex) || is_quoted(ex) || kind(ex) == K"deferred_toplevel_eval"
         return
     elseif kind(ex) == K"isdefined"
         delete!(vars, ex[1].var_id)
@@ -1017,7 +1017,7 @@ function _renumber(ctx, ssa_rewrites, slot_rewrites, label_table, ex)
                 makeleaf(ctx, ex, K"globalref", binfo.name, mod=binfo.mod)
             end
         end
-    elseif k == K"meta"
+    elseif k == K"meta" || k == K"deferred_toplevel_eval"
         # Somewhat-hack for Expr(:meta, :generated, gen) which has
         # weird top-level semantics for `gen`, but we still need to translate
         # the binding it contains to a globalref.

src/scope_analysis.jl

@@ -488,6 +488,14 @@ function _resolve_scopes(ctx, ex::SyntaxTree)
         end
         pop!(ctx.scope_stack)
         @ast ctx ex [K"block" stmts...]
+    elseif k == K"deferred_toplevel_eval"
+        local_scopes = splice!(ctx.scope_stack, 2:lastindex(ctx.scope_stack))
+        scope = analyze_scope(ctx, ex, nothing, true)
+        push!(ctx.scope_stack, scope)
+        resolved = mapchildren(e->_resolve_scopes(ctx, e), ctx, ex)
+        pop!(ctx.scope_stack)
+        append!(ctx.scope_stack, local_scopes)
+        resolved
     elseif k == K"extension"
         etype = extension_type(ex)
         if etype == "islocal"
@@ -647,7 +655,7 @@ function analyze_variables!(ctx, ex)
                 update_binding!(ctx, ex, is_captured=true)
             end
         end
-    elseif is_leaf(ex) || is_quoted(ex)
+    elseif is_leaf(ex) || is_quoted(ex) || k == K"deferred_toplevel_eval"
         return
     elseif k == K"local" || k == K"global"
         # Presence of BindingId within local/global is ignored.

src/syntax_macros.jl

@@ -91,21 +91,7 @@ function Base.var"@cfunction"(__context__::MacroContext, callable, return_type,
         # Kinda weird semantics here - without `$`, the callable is a top level
         # expression which will be evaluated by `jl_resolve_globals_in_ir`,
         # implicitly within the module where the `@cfunction` is expanded into.
-        #
-        # TODO: The existing flisp implementation is arguably broken because it
-        # ignores macro hygiene when `callable` is the result of a macro
-        # expansion within a different module. For now we've inherited this
-        # brokenness.
-        #
-        # Ideally we'd fix this by bringing the scoping rules for this
-        # expression back into lowering. One option may be to wrap the
-        # expression in a form which pushes it to top level - maybe as a whole
-        # separate top level thunk like closure lowering - then use the
-        # K"captured_local" mechanism to interpolate it back in. This scheme
-        # would make the complicated scope semantics explicit and let them be
-        # dealt with in the right place in the frontend rather than putting the
-        # rules into the runtime itself.
-        fptr = @ast __context__ callable QuoteNode(Expr(callable))::K"Value"
+        fptr = @ast __context__ callable [K"deferred_toplevel_eval" callable]
         typ = Ptr{Cvoid}
     end
     @ast __context__ __context__.macrocall [K"cfunction"

test/misc.jl

@@ -42,7 +42,10 @@ cf_int = JuliaLowering.include_string(test_mod, """
 """)
 @test @ccall($cf_int(2::Int, 3::Int)::Int) == 32
 cf_float = JuliaLowering.include_string(test_mod, """
-@cfunction(f_ccallable, Float64, (Float64,Float64))
+let
+    f_ccallable = "irrelevant"
+    @cfunction(f_ccallable, Float64, (Float64,Float64))
+end
 """)
 @test @ccall($cf_float(2::Float64, 3::Float64)::Float64) == 32.0
 

test/misc_ir.jl

@@ -323,7 +323,7 @@ JuxtTest.@emit_juxt
 # @cfunction expansion with global generic function as function argument
 @cfunction(callable, Int, (Int, Float64))
 #---------------------
-1   (cfunction Ptr{Nothing} :(:callable) TestMod.Int (call core.svec TestMod.Int TestMod.Float64) :ccall)
+1   (cfunction Ptr{Nothing} (deferred_toplevel_eval TestMod.callable) TestMod.Int (call core.svec TestMod.Int TestMod.Float64) :ccall)
 2   (return %₁)
 
 ########################################