[NFC] codegen: Move target ABI calculation into jl_jit_abi_converter (#58718)

Makes the API more directly reflect the operation here, which is
essentially "Give me a specptr with ABI `abi` that invokes CodeInstance
`codeinst`"

This relieves the caller of the responsibility to peek into the
CodeInstance and pull out all of the correct bits to determine the
appropriate callee ABI.

Author: topolarity

src/jitlayers.cpp

@@ -255,12 +255,46 @@ static void finish_params(Module *M, jl_codegen_params_t &params, SmallVector<or
     }
 }
 
+// Return a specptr that is ABI-compatible with `from_abi` which invokes `codeinst`.
+//
+// If `codeinst` is NULL, the returned specptr instead performs a standard `apply_generic`
+// call via a dynamic dispatch.
 extern "C" JL_DLLEXPORT_CODEGEN
-void *jl_jit_abi_converter_impl(jl_task_t *ct, void *unspecialized, jl_abi_t from_abi,
-                                jl_code_instance_t *codeinst, jl_callptr_t invoke, void *target, int target_specsig)
+void *jl_jit_abi_converter_impl(jl_task_t *ct, jl_abi_t from_abi,
+                                jl_code_instance_t *codeinst)
 {
-    if (codeinst == nullptr && unspecialized != nullptr)
-        return unspecialized;
+    void *target = nullptr;
+    bool target_specsig = false;
+    jl_callptr_t invoke = nullptr;
+    if (codeinst != nullptr) {
+        uint8_t specsigflags;
+        jl_method_instance_t *mi = jl_get_ci_mi(codeinst);
+        void *specptr = nullptr;
+        jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &specptr, /* waitcompile */ 1);
+        if (invoke != nullptr) {
+            if (invoke == jl_fptr_const_return_addr) {
+                target = nullptr;
+                target_specsig = false;
+            }
+            else if (invoke == jl_fptr_args_addr) {
+                assert(specptr != nullptr);
+                if (!from_abi.specsig && jl_subtype(codeinst->rettype, from_abi.rt))
+                    return specptr; // no adapter required
+
+                target = specptr;
+                target_specsig = false;
+            }
+            else if (specsigflags & 0b1) {
+                assert(specptr != nullptr);
+                if (from_abi.specsig && jl_egal(mi->specTypes, from_abi.sigt) && jl_egal(codeinst->rettype, from_abi.rt))
+                    return specptr; // no adapter required
+
+                target = specptr;
+                target_specsig = true;
+            }
+        }
+    }
+
     orc::ThreadSafeModule result_m;
     std::string gf_thunk_name;
     {

src/julia_internal.h

@@ -1639,8 +1639,7 @@ JL_DLLEXPORT jl_value_t *jl_get_cfunction_trampoline(
     void *(*init_trampoline)(void *tramp, void **nval),
     jl_unionall_t *env, jl_value_t **vals);
 JL_DLLEXPORT void *jl_get_abi_converter(jl_task_t *ct, void *data);
-JL_DLLIMPORT void *jl_jit_abi_converter(jl_task_t *ct, void *unspecialized, jl_abi_t from_abi,
-    jl_code_instance_t *codeinst, jl_callptr_t invoke, void *target, int target_specsig);
+JL_DLLIMPORT void *jl_jit_abi_converter(jl_task_t *ct, jl_abi_t from_abi, jl_code_instance_t *codeinst);
 
 
 // Special filenames used to refer to internal julia libraries

src/runtime_ccall.cpp

@@ -422,42 +422,25 @@ void *jl_get_abi_converter(jl_task_t *ct, void *data)
         JL_UNLOCK(&cfun_lock);
         return f;
     };
-    jl_callptr_t invoke = nullptr;
     bool is_opaque_closure = false;
     jl_abi_t from_abi = { sigt, declrt, nargs, specsig, is_opaque_closure };
-    if (codeinst != NULL) {
-        jl_value_t *astrt = codeinst->rettype;
-        if (astrt != (jl_value_t*)jl_bottom_type &&
-            jl_type_intersection(astrt, declrt) == jl_bottom_type) {
-            // Do not warn if the function never returns since it is
-            // occasionally required by the C API (typically error callbacks)
-            // even though we're likely to encounter memory errors in that case
-            jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match\n", name_from_method_instance(mi));
-        }
-        uint8_t specsigflags;
-        jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &f, 1);
-        if (invoke != nullptr) {
-            if (invoke == jl_fptr_const_return_addr) {
-                return assign_fptr(jl_jit_abi_converter(ct, cfuncdata->unspecialized, from_abi, codeinst, invoke, nullptr, false));
-            }
-            else if (invoke == jl_fptr_args_addr) {
-                assert(f);
-                if (!specsig && jl_subtype(astrt, declrt))
-                    return assign_fptr(f);
-                return assign_fptr(jl_jit_abi_converter(ct, cfuncdata->unspecialized, from_abi, codeinst, invoke, f, false));
-            }
-            else if (specsigflags & 0b1) {
-                assert(f);
-                if (specsig && jl_egal(mi->specTypes, sigt) && jl_egal(declrt, astrt))
-                    return assign_fptr(f);
-                return assign_fptr(jl_jit_abi_converter(ct, cfuncdata->unspecialized, from_abi, codeinst, invoke, f, true));
-            }
-        }
+    if (codeinst == nullptr) {
+        // Generate an adapter to a dynamic dispatch
+        if (cfuncdata->unspecialized == nullptr)
+            cfuncdata->unspecialized = jl_jit_abi_converter(ct, from_abi, nullptr);
+
+        return assign_fptr(cfuncdata->unspecialized);
+    }
+
+    jl_value_t *astrt = codeinst->rettype;
+    if (astrt != (jl_value_t*)jl_bottom_type &&
+        jl_type_intersection(astrt, declrt) == jl_bottom_type) {
+        // Do not warn if the function never returns since it is
+        // occasionally required by the C API (typically error callbacks)
+        // even though we're likely to encounter memory errors in that case
+        jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match\n", name_from_method_instance(mi));
     }
-    f = jl_jit_abi_converter(ct, cfuncdata->unspecialized, from_abi, codeinst, invoke, nullptr, false);
-    if (codeinst == nullptr)
-        cfuncdata->unspecialized = f;
-    return assign_fptr(f);
+    return assign_fptr(jl_jit_abi_converter(ct, from_abi, codeinst));
 }
 
 void jl_init_runtime_ccall(void)