codegen non-overloaded llvm intrinsics based on their name, and verify the rust signature against it

Author: sayantn

compiler/rustc_codegen_gcc/src/type_of.rs

@@ -1,6 +1,6 @@
 use std::fmt::Write;
 
-use gccjit::{Struct, Type};
+use gccjit::{RValue, Struct, Type};
 use rustc_abi as abi;
 use rustc_abi::Primitive::*;
 use rustc_abi::{
@@ -373,7 +373,11 @@ impl<'gcc, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         unimplemented!();
     }
 
-    fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Type<'gcc> {
+    fn fn_decl_backend_type(
+        &self,
+        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
+        _fn_ptr: RValue<'gcc>,
+    ) -> Type<'gcc> {
         // FIXME(antoyo): Should we do something with `FnAbiGcc::fn_attributes`?
         let FnAbiGcc { return_type, arguments_type, is_c_variadic, .. } = fn_abi.gcc_type(self);
         self.context.new_function_pointer_type(None, return_type, &arguments_type, is_c_variadic)

compiler/rustc_codegen_llvm/src/abi.rs

@@ -1,6 +1,7 @@
 use std::borrow::Borrow;
 use std::cmp;
 
+use itertools::zip_eq;
 use libc::c_uint;
 use rustc_abi::{BackendRepr, HasDataLayout, Primitive, Reg, RegKind, Size};
 use rustc_codegen_ssa::MemFlags;
@@ -308,7 +309,9 @@ impl<'ll, 'tcx> ArgAbiBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
 }
 
 pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> {
-    fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
+    fn llvm_return_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
+    fn llvm_argument_types(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<&'ll Type>;
+    fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>, name: &[u8]) -> &'ll Type;
     fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
     fn llvm_cconv(&self, cx: &CodegenCx<'ll, 'tcx>) -> llvm::CallConv;
 
@@ -324,27 +327,83 @@ pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> {
     fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value);
 }
 
+/// checks that the Rust signature of a **non-overloaded** llvm intrinsic is correct
+fn match_intrinsic_signature<'ll>(
+    cx: &CodegenCx<'ll, '_>,
+    intrinsic: llvm::Intrinsic,
+    rust_return_ty: &'ll Type,
+    rust_argument_tys: &[&'ll Type],
+    name: &[u8],
+) -> &'ll Type {
+    macro_rules! error {
+        ($($t:tt)*) => {
+            cx.tcx.dcx().fatal(format!($($t)*, fn_name=str::from_utf8(name).unwrap()))
+        };
+    }
+
+    let base_name = intrinsic.base_name();
+    if name != base_name {
+        error!(
+            "Unsupported overload `{fn_name}` for non-overloaded intrinsic `{}`",
+            str::from_utf8(base_name).unwrap()
+        );
+    }
+
+    let fn_ty = cx.intrinsic_type(intrinsic, &[]);
+
+    let llvm_return_ty = cx.get_return_type(fn_ty);
+    let llvm_argument_tys = cx.func_params_types(fn_ty);
+
+    if rust_argument_tys.len() != llvm_argument_tys.len() {
+        error!(
+            "Intrinsic signature mismatch: expected {} arguments for `{fn_name}`, found {} arguments",
+            llvm_argument_tys.len(),
+            rust_argument_tys.len()
+        );
+    }
+
+    if rust_return_ty != llvm_return_ty {
+        error!(
+            "Intrinsic signature mismatch: could not match `{rust_return_ty:?}` (found) with  {llvm_return_ty:?} (expected) as return type for `{fn_name}`"
+        );
+    }
+    for (idx, (&rust_argument_ty, llvm_argument_ty)) in
+        zip_eq(rust_argument_tys, llvm_argument_tys).enumerate()
+    {
+        if rust_argument_ty != llvm_argument_ty {
+            error!(
+                "Intrinsic signature mismatch: could not match `{rust_return_ty:?}` (found) with  {llvm_return_ty:?} (expected) as argument {idx} for `{fn_name}`"
+            );
+        }
+    }
+
+    fn_ty
+}
+
 impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
-    fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
+    fn llvm_return_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
+        match &self.ret.mode {
+            PassMode::Ignore => cx.type_void(),
+            PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx),
+            PassMode::Cast { cast, pad_i32: _ } => cast.llvm_type(cx),
+            PassMode::Indirect { .. } => cx.type_void(),
+        }
+    }
+
+    fn llvm_argument_types(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<&'ll Type> {
+        let indirect_return = matches!(self.ret.mode, PassMode::Indirect { .. });
+
         // Ignore "extra" args from the call site for C variadic functions.
         // Only the "fixed" args are part of the LLVM function signature.
         let args =
             if self.c_variadic { &self.args[..self.fixed_count as usize] } else { &self.args };
 
-        // This capacity calculation is approximate.
-        let mut llargument_tys = Vec::with_capacity(
-            self.args.len() + if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 },
-        );
+        let mut llargument_tys =
+            Vec::with_capacity(args.len() + if indirect_return { 1 } else { 0 });
 
-        let llreturn_ty = match &self.ret.mode {
-            PassMode::Ignore => cx.type_void(),
-            PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx),
-            PassMode::Cast { cast, pad_i32: _ } => cast.llvm_type(cx),
-            PassMode::Indirect { .. } => {
-                llargument_tys.push(cx.type_ptr());
-                cx.type_void()
-            }
-        };
+        if indirect_return {
+            llargument_tys.push(cx.type_ptr());
+        }
 
         for arg in args {
             // Note that the exact number of arguments pushed here is carefully synchronized with
@@ -391,10 +450,44 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             llargument_tys.push(llarg_ty);
         }
 
+        llargument_tys
+    }
+
+    fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>, name: &[u8]) -> &'ll Type {
+        let return_ty = self.llvm_return_type(cx);
+        let argument_tys = self.llvm_argument_types(cx);
+
+        if name.starts_with(b"llvm.") {
+            if let Some(intrinsic) = llvm::Intrinsic::lookup(name)
+                && !intrinsic.is_overloaded()
+            {
+                if !intrinsic.is_overloaded() {
+                    // FIXME: also do this for overloaded intrinsics
+                    return match_intrinsic_signature(
+                        cx,
+                        intrinsic,
+                        return_ty,
+                        &argument_tys,
+                        name,
+                    );
+                }
+            } else {
+                // it's one of 2 cases,
+                // - either the base name is invalid
+                // - it has been superceded by something else, so the intrinsic was removed entirely
+                //
+                // anyway, let's log it
+                tracing::warn!(
+                    "Couldn't find intrinsic `{}`, either invalid or deprecated",
+                    str::from_utf8(name).unwrap()
+                );
+            }
+        }
+
         if self.c_variadic {
-            cx.type_variadic_func(&llargument_tys, llreturn_ty)
+            cx.type_variadic_func(&argument_tys, return_ty)
         } else {
-            cx.type_func(&llargument_tys, llreturn_ty)
+            cx.type_func(&argument_tys, return_ty)
         }
     }
 

compiler/rustc_codegen_llvm/src/builder.rs

@@ -31,7 +31,7 @@ use tracing::{debug, instrument};
 
 use crate::abi::FnAbiLlvmExt;
 use crate::attributes;
-use crate::common::Funclet;
+use crate::common::{AsCCharPtr, Funclet};
 use crate::context::{CodegenCx, FullCx, GenericCx, SCx};
 use crate::llvm::{
     self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, GEPNoWrapFlags, Metadata, True,
@@ -1611,15 +1611,33 @@ impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
         type_params: &[&'ll Type],
         args: &[&'ll Value],
     ) -> &'ll Value {
+        // todo: cache?
+
         let intrinsic = llvm::Intrinsic::lookup(base_name.as_bytes())
             .unwrap_or_else(|| bug!("Intrinsic `{base_name}` not found"));
-        let (fn_ty, llfn) = self.cx.get_intrinsic(intrinsic, type_params);
+
+        let fn_ty = self.cx.intrinsic_type(intrinsic, type_params);
+
+        let full_name = if intrinsic.is_overloaded() {
+            &intrinsic.overloaded_name(self.cx.llmod(), type_params)
+        } else {
+            assert!(type_params.is_empty());
+            base_name.as_bytes()
+        };
+        let llfn = unsafe {
+            llvm::LLVMRustGetOrInsertFunction(
+                self.cx.llmod(),
+                full_name.as_c_char_ptr(),
+                full_name.len(),
+                fn_ty,
+            )
+        };
         self.simple_call(fn_ty, llfn, args)
     }
 }
 
 impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
-    fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) {
+    fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) {
         let size = size.bytes();
         if size == 0 {
             return;

compiler/rustc_codegen_llvm/src/context.rs

@@ -836,25 +836,17 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
 }
 
 impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
-    pub(crate) fn get_intrinsic(
+    pub(crate) fn intrinsic_type(
         &self,
         intrinsic: llvm::Intrinsic,
         type_params: &[&'ll Type],
-    ) -> (&'ll Type, &'ll Value) {
+    ) -> &'ll Type {
         unsafe {
-            (
-                llvm::LLVMIntrinsicGetType(
-                    self.llcx(),
-                    intrinsic.id(),
-                    type_params.as_ptr(),
-                    type_params.len(),
-                ),
-                llvm::LLVMGetIntrinsicDeclaration(
-                    self.llmod(),
-                    intrinsic.id(),
-                    type_params.as_ptr(),
-                    type_params.len(),
-                ),
+            llvm::LLVMIntrinsicGetType(
+                self.llcx(),
+                intrinsic.id(),
+                type_params.as_ptr(),
+                type_params.len(),
             )
         }
     }

compiler/rustc_codegen_llvm/src/declare.rs

@@ -158,7 +158,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
             fn_abi.llvm_cconv(self),
             llvm::UnnamedAddr::Global,
             llvm::Visibility::Default,
-            fn_abi.llvm_type(self),
+            fn_abi.llvm_type(self, name.as_bytes()),
         );
         fn_abi.apply_attrs_llfn(self, llfn, instance);
 

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -1089,7 +1089,7 @@ fn gen_fn<'a, 'll, 'tcx>(
     codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
 ) -> (&'ll Type, &'ll Value) {
     let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
-    let llty = fn_abi.llvm_type(cx);
+    let llty = fn_abi.llvm_type(cx, name.as_bytes());
     let llfn = cx.declare_fn(name, fn_abi, None);
     cx.set_frame_pointer_type(llfn);
     cx.apply_target_cpu_attr(llfn);

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -1077,7 +1077,6 @@ unsafe extern "C" {
 
     // Operations on other types
     pub(crate) fn LLVMVoidTypeInContext(C: &Context) -> &Type;
-    pub(crate) fn LLVMMetadataTypeInContext(C: &Context) -> &Type;
 
     // Operations on all values
     pub(crate) fn LLVMTypeOf(Val: &Value) -> &Type;
@@ -1196,18 +1195,24 @@ unsafe extern "C" {
 
     // Operations about llvm intrinsics
     pub(crate) fn LLVMLookupIntrinsicID(Name: *const c_char, NameLen: size_t) -> c_uint;
-    pub(crate) fn LLVMGetIntrinsicDeclaration<'a>(
-        M: &'a Module,
-        ID: c_uint,
-        ParamTypes: *const &'a Type,
-        ParamCount: size_t,
-    ) -> &'a Value;
     pub(crate) fn LLVMIntrinsicGetType<'a>(
         C: &'a Context,
         ID: c_uint,
         ParamTypes: *const &'a Type,
         ParamCount: size_t,
     ) -> &'a Type;
+    pub(crate) fn LLVMRustIntrinsicGetBaseName(
+        ID: c_uint,
+        NameLength: *mut size_t,
+    ) -> *const c_char;
+    pub(crate) fn LLVMIntrinsicIsOverloaded(ID: c_uint) -> Bool;
+    pub(crate) fn LLVMIntrinsicCopyOverloadedName2<'a>(
+        Mod: &'a Module,
+        ID: c_uint,
+        ParamTypes: *const &'a Type,
+        ParamCount: size_t,
+        NameLength: *mut size_t,
+    ) -> *mut c_char;
 
     // Operations on parameters
     pub(crate) fn LLVMIsAArgument(Val: &Value) -> Option<&Value>;

compiler/rustc_codegen_llvm/src/llvm/mod.rs

@@ -1,9 +1,9 @@
 #![allow(non_snake_case)]
 
 use std::ffi::{CStr, CString};
-use std::ptr;
 use std::str::FromStr;
 use std::string::FromUtf8Error;
+use std::{ptr, slice};
 
 use libc::c_uint;
 use rustc_abi::{Align, Size, WrappingRange};
@@ -341,6 +341,66 @@ impl Intrinsic {
     pub(crate) fn id(self) -> c_uint {
         self.id
     }
+
+    pub(crate) fn is_overloaded(self) -> bool {
+        unsafe { LLVMIntrinsicIsOverloaded(self.id) == True }
+    }
+
+    pub(crate) fn base_name<'ll>(self) -> &'ll [u8] {
+        let mut len = 0;
+        let ptr = unsafe { LLVMRustIntrinsicGetBaseName(self.id, &mut len) };
+        unsafe { slice::from_raw_parts(ptr.cast(), len) }
+    }
+
+    pub(crate) fn overloaded_name<'ll>(
+        self,
+        llmod: &'ll Module,
+        type_params: &[&'ll Type],
+    ) -> CSlice<u8> {
+        let mut len = 0;
+        let ptr = unsafe {
+            LLVMIntrinsicCopyOverloadedName2(
+                llmod,
+                self.id,
+                type_params.as_ptr(),
+                type_params.len(),
+                &mut len,
+            )
+        };
+        unsafe { CSlice::from_raw(ptr.cast(), len) }
+    }
+}
+
+pub(crate) struct CSlice<T: Copy> {
+    ptr: *mut T,
+    len: usize,
+}
+
+impl<T: Copy> CSlice<T> {
+    pub(crate) unsafe fn from_raw(ptr: *mut T, len: usize) -> Self {
+        assert!(std::mem::size_of::<T>() != 0, "ZSTs aren't supported yet");
+        Self { ptr, len }
+    }
+}
+
+impl<T: Copy> std::ops::Deref for CSlice<T> {
+    type Target = [T];
+
+    fn deref(&self) -> &Self::Target {
+        unsafe { slice::from_raw_parts(self.ptr, self.len) }
+    }
+}
+
+impl<T: Copy> std::ops::DerefMut for CSlice<T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        unsafe { slice::from_raw_parts_mut(self.ptr, self.len) }
+    }
+}
+
+impl<T: Copy> Drop for CSlice<T> {
+    fn drop(&mut self) {
+        unsafe { libc::free(self.ptr.cast()) }
+    }
 }
 
 /// Safe wrapper for `LLVMSetValueName2` from a byte slice