Front end macro and lowering of cfunction

Quoting of the callable seems broken with respect to macro hygiene, but
for now we follow the Julia reference implementation and defer fixing
this to later. See also issue JuliaLang/JuliaLowering.jl#9.

Author: c42f

JuliaLowering/src/eval.jl

@@ -290,6 +290,7 @@ function to_lowered_expr(mod, ex, ssa_offset=0)
                k == K"gc_preserve_begin" ? :gc_preserve_begin :
                k == K"gc_preserve_end"   ? :gc_preserve_end   :
                k == K"foreigncall"       ? :foreigncall       :
+               k == K"cfunction"         ? :cfunction         :
                k == K"opaque_closure_method" ? :opaque_closure_method :
                nothing
         if isnothing(head)

JuliaLowering/src/kinds.jl

@@ -29,6 +29,8 @@ function _register_kinds()
             "loopinfo"
             # Call into foreign code. Emitted by `@ccall`
             "foreigncall"
+            # Special form for constructing a function callable from C
+            "cfunction"
             # Special form emitted by `Base.Experimental.@opaque`
             "opaque_closure"
             # Test whether a variable is defined

JuliaLowering/src/linear_ir.jl

@@ -143,14 +143,17 @@ function is_valid_ir_rvalue(ctx, lhs, rhs)
     return is_ssa(ctx, lhs) ||
            is_valid_ir_argument(ctx, rhs) ||
            (kind(lhs) == K"BindingId" &&
-            # FIXME: add: invoke cfunction gc_preserve_begin copyast
-            kind(rhs) in KSet"new splatnew isdefined call foreigncall gc_preserve_begin foreigncall new_opaque_closure")
+            # FIXME: add: invoke ?
+            kind(rhs) in KSet"new splatnew cfunction isdefined call foreigncall gc_preserve_begin foreigncall new_opaque_closure")
 end
 
-function contains_nonglobal_binding(ctx, ex)
-    contains_unquoted(ex) do e
+function check_no_local_bindings(ctx, ex, msg)
+    contains_nonglobal_binding = contains_unquoted(ex) do e
         kind(e) == K"BindingId" && lookup_binding(ctx, e).kind !== :global
     end
+    if contains_nonglobal_binding
+        throw(LoweringError(ex, msg))
+    end
 end
 
 # evaluate the arguments of a call, creating temporary locations as needed
@@ -582,18 +585,15 @@ function compile(ctx::LinearIRContext, ex, needs_value, in_tail_pos)
         @chk !needs_value (ex,"TOMBSTONE encountered in value position")
         nothing
     elseif k == K"call" || k == K"new" || k == K"splatnew" || k == K"foreigncall" ||
-            k == K"new_opaque_closure"
-        # TODO k ∈ cfunction cglobal
+            k == K"new_opaque_closure" || k == K"cfunction"
         if k == K"foreigncall"
             args = SyntaxList(ctx)
             # todo: is is_leaf correct here? flisp uses `atom?`
             func = ex[1]
             if kind(func) == K"call" && kind(func[1]) == K"core" && func[1].name_val == "tuple"
                 # Tuples like core.tuple(:funcname, mylib_name) are allowed,
                 # but may only reference globals.
-                if contains_nonglobal_binding(ctx, func)
-                    throw(LoweringError(func, "ccall function name and library expression cannot reference local variables"))
-                end
+                check_no_local_bindings(ctx, func, "ccall function name and library expression cannot reference local variables")
                 append!(args, compile_args(ctx, ex[1:1]))
             elseif is_leaf(func)
                 append!(args, compile_args(ctx, ex[1:1]))
@@ -602,18 +602,27 @@ function compile(ctx::LinearIRContext, ex, needs_value, in_tail_pos)
             end
             # 2nd to 5th arguments of foreigncall are special. They must be
             # left in place but cannot reference locals.
-            if contains_nonglobal_binding(ctx, ex[2])
-                throw(LoweringError(ex[2], "ccall return type cannot reference local variables"))
-            end
+            check_no_local_bindings(ctx, ex[2], "ccall return type cannot reference local variables")
             for argt in children(ex[3])
-                if contains_nonglobal_binding(ctx, argt)
-                    throw(LoweringError(argt, "ccall argument types cannot reference local variables"))
-                end
+                check_no_local_bindings(ctx, argt,
+                    "ccall argument types cannot reference local variables")
             end
             append!(args, ex[2:5])
             append!(args, compile_args(ctx, ex[6:end]))
             args
+        elseif k == K"cfunction"
+            # Arguments of cfunction must be left in place except for argument
+            # 2 (fptr)
+            args = copy(children(ex))
+            args[2] = only(compile_args(ctx, args[2:2]))
+            check_no_local_bindings(ctx, ex[3],
+                "cfunction return type cannot reference local variables")
+            for arg in children(ex[4])
+                check_no_local_bindings(ctx, arg,
+                    "cfunction argument cannot reference local variables")
+            end
         else
+            # TODO: cglobal
             args = compile_args(ctx, children(ex))
         end
         callex = makenode(ctx, ex, k, args)

JuliaLowering/src/syntax_macros.jl

@@ -75,6 +75,47 @@ function Base.var"@generated"(__context__::MacroContext, ex)
     ]
 end
 
+function Base.var"@cfunction"(__context__::MacroContext, callable, return_type, arg_types)
+    if kind(arg_types) != K"tuple"
+        throw(MacroExpansionError(arg_types, "@cfunction argument types must be a literal tuple"))
+    end
+    arg_types_svec = @ast __context__ arg_types [K"call"
+        "svec"::K"core"
+        children(arg_types)...
+    ]
+    if kind(callable) == K"$"
+        fptr = callable[1]
+        typ = Base.CFunction
+    else
+        # Kinda weird semantics here - without `$`, the callable is a top level
+        # expression which will be evaluated by `jl_resolve_globals_in_ir`,
+        # implicitly within the module where the `@cfunction` is expanded into.
+        #
+        # TODO: The existing flisp implementation is arguably broken because it
+        # ignores macro hygiene when `callable` is the result of a macro
+        # expansion within a different module. For now we've inherited this
+        # brokenness.
+        #
+        # Ideally we'd fix this by bringing the scoping rules for this
+        # expression back into lowering. One option may be to wrap the
+        # expression in a form which pushes it to top level - maybe as a whole
+        # separate top level thunk like closure lowering - then use the
+        # K"captured_local" mechanism to interpolate it back in. This scheme
+        # would make the complicated scope semantics explicit and let them be
+        # dealt with in the right place in the frontend rather than putting the
+        # rules into the runtime itself.
+        fptr = @ast __context__ callable QuoteNode(Expr(callable))::K"Value"
+        typ = Ptr{Cvoid}
+    end
+    @ast __context__ __context__.macrocall [K"cfunction"
+        typ::K"Value"
+        fptr
+        return_type
+        arg_types_svec
+        "ccall"::K"Symbol"
+    ]
+end
+
 function Base.GC.var"@preserve"(__context__::MacroContext, exs...)
     idents = exs[1:end-1]
     for e in idents

JuliaLowering/test/misc.jl

@@ -17,7 +17,7 @@ let x = [1,2]
     GC.@preserve x begin
         x
     end
-end        
+end
 """) == [1,2]
 
 @test JuliaLowering.include_string(test_mod, """
@@ -31,4 +31,19 @@ end
 ccall(:strlen, Csize_t, (Cstring,), "asdfg")
 """) == 5
 
+# cfunction
+JuliaLowering.include_string(test_mod, """
+function f_ccallable(x, y)
+    x + y * 10
+end
+""")
+cf_int = JuliaLowering.include_string(test_mod, """
+@cfunction(f_ccallable, Int, (Int,Int))
+""")
+@test @ccall($cf_int(2::Int, 3::Int)::Int) == 32
+cf_float = JuliaLowering.include_string(test_mod, """
+@cfunction(f_ccallable, Float64, (Float64,Float64))
+""")
+@test @ccall($cf_float(2::Float64, 3::Float64)::Float64) == 32.0
+
 end

JuliaLowering/test/misc_ir.jl

@@ -319,6 +319,53 @@ JuxtTest.@emit_juxt
 3   (call %₁ 10 %₂)
 4   (return %₃)
 
+########################################
+# @cfunction expansion with global generic function as function argument
+@cfunction(callable, Int, (Int, Float64))
+#---------------------
+1   (cfunction Ptr{Nothing} :(:callable) TestMod.Int (call core.svec TestMod.Int TestMod.Float64) :ccall)
+2   (return %₁)
+
+########################################
+# @cfunction expansion with closed-over callable argument
+@cfunction($close_over, Int, (Int, Float64))
+#---------------------
+1   TestMod.close_over
+2   (cfunction Base.CFunction %₁ TestMod.Int (call core.svec TestMod.Int TestMod.Float64) :ccall)
+3   (return %₂)
+
+########################################
+# Error: Bad arg types to @cfunction
+@cfunction(f, Int, NotATuple)
+#---------------------
+MacroExpansionError while expanding @cfunction in module Main.TestMod:
+@cfunction(f, Int, NotATuple)
+#                  └───────┘ ── @cfunction argument types must be a literal tuple
+
+########################################
+# Error: Locals used in @cfunction return type
+let T=Float64
+    @cfunction(f, T, (Float64,))
+end
+#---------------------
+LoweringError:
+let T=Float64
+    @cfunction(f, T, (Float64,))
+#                 ╙ ── cfunction return type cannot reference local variables
+end
+
+########################################
+# Error: Locals used in @cfunction arg type
+let T=Float64
+    @cfunction(f, Float64, (Float64,T))
+end
+#---------------------
+LoweringError:
+let T=Float64
+    @cfunction(f, Float64, (Float64,T))
+#                                   ╙ ── cfunction argument cannot reference local variables
+end
+
 ########################################
 # Error: unary & syntax
 &x