ccall: make distinction of pointer vs name a syntactic distinction (#59165)

We have long expected users to be explicit about the library name for
`ccall`, and the `@ccall` macro has even always enforced that. That
means users should have already been using explicit syntax, even though
it wasn't strictly enforced. And indeed, the other syntax forms weren't
handled reliably anyways (since doing so would require linearizing IR if
and only if the runtime values required it, which is not something that
is computable, and thus was often done wrong). This now intends to align
the runtime and compiler to expect only those syntax forms that we could
reliably handle in the past without errors, and adds explicit errors for
other cases, most of which we previously knew would be unreliable due to
reliance upon inference in making particular decisions for the
semantics. The `ccall` function is already very special since it is more
like a actual macro (it does not exist as a binding or value), so we can
make unusual syntax decisions like this, mirroring `@ccall` also.

This fixes #57931, mostly by restricting the set of things that are
allowed to the set of things that have an obvious and pre-existing
behavior to be guaranteed to do that behavior. The hope is to PkgEval
this to check if any packages are doing something unusual and see if we
even need to allow anything else.

This drops support for #37123,
since we were going to use that for LazyLibraries, be we decided that
approach was quite buggy and that PR would make static compilation quite
impossible to support, so we instead actually implemented LazyLibraries
with a different approach. It could be re-enabled, but we never had
correct lowering or inference support for it, so it is presumably still
unused.

The goal is to cause breakage only where the package authors really
failed to express intent with syntax, and otherwise to explicitly
maintain support by adding cases to normalize the given syntax into one
of the supported cases. All existing functionality (and more) can be
accessed by explicit management of a pointer or by a LazyLibrary-like
type, so this shouldn't cause any reduction in possible functionality,
just possibly altered syntax.

In followup work, we may want to apply some similar improvements to the
cglobal semantics, though not nearly as urgent, since cglobal on a
pointer is permitted as a complicated bitcast right now, which doesn't
make sense and probably should be deprecated, so that there is no
ambiguity with that function's meaning.

Written with help from Claude

Author: vtjnash

Compiler/src/abstractinterpretation.jl

@@ -3480,65 +3480,29 @@ function refine_partial_type(@nospecialize t)
     return t
 end
 
-abstract_eval_nonlinearized_foreigncall_name(
-        ::AbstractInterpreter, @nospecialize(e), ::StatementState, ::IRInterpretationState
-    ) = nothing
-
-function abstract_eval_nonlinearized_foreigncall_name(
-        interp::AbstractInterpreter, @nospecialize(e), sstate::StatementState, sv::InferenceState
-    )
-    if isexpr(e, :call)
-        n = length(e.args)
-        argtypes = Vector{Any}(undef, n)
-        callresult = Future{CallMeta}()
-        i::Int = 1
-        nextstate::UInt8 = 0x0
-        local ai, res
-        function evalargs(interp, sv)
-            if nextstate === 0x1
-                @goto state1
-            elseif nextstate === 0x2
-                @goto state2
-            end
-            while i <= n
-                ai = abstract_eval_nonlinearized_foreigncall_name(interp, e.args[i], sstate, sv)
-                if !isready(ai)
-                    nextstate = 0x1
-                    return false
-                    @label state1
-                end
-                argtypes[i] = ai[].rt
-                i += 1
-            end
-            res = abstract_call(interp, ArgInfo(e.args, argtypes), sstate, sv)
-            if !isready(res)
-                nextstate = 0x2
-                return false
-                @label state2
-            end
-            callresult[] = res[]
-            return true
-        end
-        evalargs(interp, sv) || push!(sv.tasks, evalargs)
-        return callresult
-    else
-        return Future(abstract_eval_basic_statement(interp, e, sstate, sv))
-    end
-end
-
 function abstract_eval_foreigncall(interp::AbstractInterpreter, e::Expr, sstate::StatementState, sv::AbsIntState)
     callee = e.args[1]
-    if isexpr(callee, :call) && length(callee.args) > 1 && callee.args[1] == GlobalRef(Core, :tuple)
-        # NOTE these expressions are not properly linearized
-        abstract_eval_nonlinearized_foreigncall_name(interp, callee.args[2], sstate, sv)
-        if length(callee.args) > 2
-            abstract_eval_nonlinearized_foreigncall_name(interp, callee.args[3], sstate, sv)
+    if isexpr(callee, :tuple)
+        if length(callee.args) >= 1
+            abstract_eval_value(interp, callee.args[1], sstate, sv)
+            if length(callee.args) >= 2
+                abstract_eval_value(interp, callee.args[2], sstate, sv)
+                #TODO: implement abstract_eval_nonlinearized_foreigncall_name correctly?
+                # lib_effects = abstract_call(interp, ArgInfo(e.args, Any[typeof(Libdl.dlopen), lib]), sstate, sv)::Future
+            end
         end
     else
         abstract_eval_value(interp, callee, sstate, sv)
     end
     mi = frame_instance(sv)
     t = sp_type_rewrap(e.args[2], mi, true)
+    let fptr = e.args[1]
+        if !isexpr(fptr, :tuple)
+            if !hasintersect(widenconst(abstract_eval_value(interp, fptr, sstate, sv)), Ptr)
+                return RTEffects(Bottom, Any, EFFECTS_THROWS)
+            end
+        end
+    end
     for i = 3:length(e.args)
         if abstract_eval_value(interp, e.args[i], sstate, sv) === Bottom
             return RTEffects(Bottom, Any, EFFECTS_THROWS)

Compiler/src/optimize.jl

@@ -1439,8 +1439,11 @@ function statement_cost(ex::Expr, line::Int, src::Union{CodeInfo, IRCode}, sptyp
         return params.inline_nonleaf_penalty
     elseif head === :foreigncall
         foreigncall = ex.args[1]
-        if foreigncall isa QuoteNode && foreigncall.value === :jl_string_ptr
-            return 1
+        if isexpr(foreigncall, :tuple, 1)
+            foreigncall = foreigncall.args[1]
+            if foreigncall isa QuoteNode && foreigncall.value === :jl_string_ptr
+                return 1
+            end
         end
         return 20
     elseif head === :invoke || head === :invoke_modify

Compiler/src/ssair/EscapeAnalysis.jl

@@ -1037,8 +1037,10 @@ function escape_foreigncall!(astate::AnalysisState, pc::Int, args::Vector{Any})
     # NOTE array allocations might have been proven as nothrow (https://github.com/JuliaLang/julia/pull/43565)
     nothrow = is_nothrow(astate.ir, pc)
     name_info = nothrow ? ⊥ : ThrownEscape(pc)
-    add_escape_change!(astate, name, name_info)
-    add_liveness_change!(astate, name, pc)
+    if !isexpr(name, :tuple)
+        add_escape_change!(astate, name, name_info)
+        add_liveness_change!(astate, name, pc)
+    end
     for i = 1:nargs
         # we should escape this argument if it is directly called,
         # otherwise just impose ThrownEscape if not nothrow

Compiler/src/ssair/inlining.jl

@@ -1753,7 +1753,7 @@ function late_inline_special_case!(ir::IRCode, idx::Int, stmt::Expr, flag::UInt3
             length(stmt.args) == 2 ? Any : stmt.args[end])
         return SomeCase(typevar_call)
     elseif f === UnionAll && length(argtypes) == 3 && ⊑(optimizer_lattice(state.interp), argtypes[2], TypeVar)
-        unionall_call = Expr(:foreigncall, QuoteNode(:jl_type_unionall), Any, svec(Any, Any),
+        unionall_call = Expr(:foreigncall, Expr(:tuple, QuoteNode(:jl_type_unionall)), Any, svec(Any, Any),
             0, QuoteNode(:ccall), stmt.args[2], stmt.args[3])
         return SomeCase(unionall_call)
     elseif is_return_type(f)

Compiler/src/ssair/verify.jl

@@ -77,10 +77,10 @@ function check_op(ir::IRCode, domtree::DomTree, @nospecialize(op), use_bb::Int,
         end
     elseif isa(op, Expr)
         # Only Expr(:boundscheck) is allowed in value position
-        if isforeigncall && arg_idx == 1 && op.head === :call
-            # Allow a tuple in symbol position for foreigncall - this isn't actually
-            # a real call - it's interpreted in global scope by codegen. However,
-            # we do need to keep this a real use, because it could also be a pointer.
+        if isforeigncall && arg_idx == 1 && op.head === :tuple
+            # Allow a tuple literal in symbol position for foreigncall - this
+            # is syntax for a literal value or globalref - it is interpreted in
+            # global scope by codegen.
         elseif !is_value_pos_expr_head(op.head)
             if !allow_frontend_forms || op.head !== :opaque_closure_method
                 @verify_error "Expr not allowed in value position"

Compiler/src/verifytrim.jl

@@ -279,9 +279,17 @@ function verify_codeinstance!(interp::NativeInterpreter, codeinst::CodeInstance,
             error = "unresolved cfunction"
         elseif isexpr(stmt, :foreigncall)
             foreigncall = stmt.args[1]
-            if foreigncall isa QuoteNode
-                if foreigncall.value in runtime_functions
-                    error = "disallowed ccall into a runtime function"
+            if isexpr(foreigncall, :tuple, 1)
+                foreigncall = foreigncall.args[1]
+                if foreigncall isa String
+                    foreigncall = QuoteNode(Symbol(foreigncall))
+                end
+                if foreigncall isa QuoteNode
+                    if foreigncall.value in runtime_functions
+                        error = "disallowed ccall into a runtime function"
+                    end
+                else
+                    error = "disallowed ccall with non-constant name and no library"
                 end
             end
         elseif isexpr(stmt, :new_opaque_closure)

Compiler/test/inference.jl

@@ -5219,11 +5219,11 @@ end
 @testset "#45956: non-linearized cglobal needs special treatment for stmt effects" begin
     function foo()
         cglobal((a, ))
-        ccall(0, Cvoid, (Nothing,), b)
+        ccall(C_NULL, Cvoid, (Nothing,), b)
     end
     @test only(code_typed() do
         cglobal((a, ))
-        ccall(0, Cvoid, (Nothing,), b)
+        ccall(C_NULL, Cvoid, (Nothing,), b)
     end)[2] === Nothing
 end
 

Compiler/test/irpasses.jl

@@ -1071,7 +1071,7 @@ let # Test for https://github.com/JuliaLang/julia/issues/43402
     end
 
     refs = map(Core.SSAValue, findall(@nospecialize(x)->Meta.isexpr(x, :new), src.code))
-    some_ccall = findfirst(@nospecialize(x) -> Meta.isexpr(x, :foreigncall) && x.args[1] == :(:some_ccall), src.code)
+    some_ccall = findfirst(@nospecialize(x) -> Meta.isexpr(x, :foreigncall) && x.args[1] == Expr(:tuple, :(:some_ccall)), src.code)
     @assert some_ccall !== nothing
     stmt = src.code[some_ccall]
     nccallargs = length(stmt.args[3]::Core.SimpleVector)

base/c.jl

@@ -313,11 +313,15 @@ function ccall_macro_parse(exprs)
     # get the function symbols
     func = let f = call.args[1]
         if isexpr(f, :.)
-            :(($(f.args[2]), $(f.args[1])))
+            Expr(:tuple, f.args[2], f.args[1])
         elseif isexpr(f, :$)
-            f
+            func = f.args[1]
+            if isa(func, String) || (isa(func, QuoteNode) && !isa(func.value, Ptr)) || isa(func, Tuple) || isexpr(func, :tuple)
+                throw(ArgumentError("interpolated value should be a variable or expression, not a literal name or tuple"))
+            end
+            func
         elseif f isa Symbol
-            QuoteNode(f)
+            Expr(:tuple, QuoteNode(f))
         else
             throw(ArgumentError("@ccall function name must be a symbol, a `.` node (e.g. `libc.printf`) or an interpolated function pointer (with `\$`)"))
         end
@@ -363,33 +367,13 @@ end
 
 
 function ccall_macro_lower(convention, func, rettype, types, args, gc_safe, nreq)
-    statements = []
-
-    # if interpolation was used, ensure the value is a function pointer at runtime.
-    if isexpr(func, :$)
-        push!(statements, Expr(:(=), :func, esc(func.args[1])))
-        name = QuoteNode(func.args[1])
-        func = :func
-        check = quote
-            if !isa(func, Ptr{Cvoid})
-                name = $name
-                throw(ArgumentError(LazyString("interpolated function `", name, "` was not a Ptr{Cvoid}, but ", typeof(func))))
-            end
-        end
-        push!(statements, check)
-    else
-        func = esc(func)
-    end
-    cconv = nothing
     if convention isa Tuple
         cconv = Expr(:cconv, (convention..., gc_safe), nreq)
     else
         cconv = Expr(:cconv, (convention, UInt16(0), gc_safe), nreq)
     end
-
-    return Expr(:block, statements...,
-                Expr(:call, :ccall, func, cconv, esc(rettype),
-                     Expr(:tuple, map!(esc, types, types)...), map!(esc, args, args)...))
+    return Expr(:call, :ccall, esc(func), cconv, esc(rettype),
+                 Expr(:tuple, map!(esc, types, types)...), map!(esc, args, args)...)
 end
 
 """

base/deprecated.jl

@@ -24,9 +24,10 @@ const __internal_changes_list = (
     :invertedlinetables,
     :codeinforefactor,
     :miuninferredrm,
-    :codeinfonargs,  # #54341
+    :codeinfonargs, #54341
     :ocnopartial,
     :printcodeinfocalls,
+    :syntacticccall, #59165
     # Add new change names above this line
 )