Check for upgradability of intrinsics

 - Emit notes when can't match, and fall back to default implementation

Author: sayantn

compiler/rustc_codegen_llvm/src/abi.rs

@@ -308,10 +308,29 @@ impl<'ll, 'tcx> ArgAbiBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
     }
 }
 
+pub(crate) enum FunctionSignature<'ll> {
+    /// The signature is obtained directly from LLVM, and **may not match the Rust signature**
+    Intrinsic(llvm::Intrinsic, &'ll Type),
+    /// The name starts with `llvm.`, but can't obtain the intrinsic ID. May be invalid or upgradable
+    MaybeInvalidIntrinsic(&'ll Type),
+    /// Just the Rust signature
+    Rust(&'ll Type),
+}
+
+impl<'ll> FunctionSignature<'ll> {
+    pub(crate) fn fn_ty(&self) -> &'ll Type {
+        match self {
+            FunctionSignature::Intrinsic(_, fn_ty)
+            | FunctionSignature::MaybeInvalidIntrinsic(fn_ty)
+            | FunctionSignature::Rust(fn_ty) => fn_ty,
+        }
+    }
+}
+
 pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> {
     fn llvm_return_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
     fn llvm_argument_types(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<&'ll Type>;
-    fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>, name: &[u8]) -> &'ll Type;
+    fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>, name: &[u8]) -> FunctionSignature<'ll>;
     fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
     fn llvm_cconv(&self, cx: &CodegenCx<'ll, 'tcx>) -> llvm::CallConv;
 
@@ -327,57 +346,25 @@ pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> {
     fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value);
 }
 
-/// checks that the Rust signature of a **non-overloaded** llvm intrinsic is correct
-fn match_intrinsic_signature<'ll>(
-    cx: &CodegenCx<'ll, '_>,
-    intrinsic: llvm::Intrinsic,
+/// checks that the Rust signature of a llvm intrinsic is correct
+fn verify_intrinsic_signature<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    fn_ty: &'ll Type,
     rust_return_ty: &'ll Type,
     rust_argument_tys: &[&'ll Type],
-    name: &[u8],
-) -> &'ll Type {
-    macro_rules! error {
-        ($($t:tt)*) => {
-            cx.tcx.dcx().fatal(format!($($t)*, fn_name=str::from_utf8(name).unwrap()))
-        };
-    }
-
-    let base_name = intrinsic.base_name();
-    if name != base_name {
-        error!(
-            "Unsupported overload `{fn_name}` for non-overloaded intrinsic `{}`",
-            str::from_utf8(base_name).unwrap()
-        );
-    }
-
-    let fn_ty = cx.intrinsic_type(intrinsic, &[]);
-
+    rust_is_variadic: bool,
+) -> bool {
     let llvm_return_ty = cx.get_return_type(fn_ty);
     let llvm_argument_tys = cx.func_params_types(fn_ty);
+    let llvm_is_variadic = cx.func_is_variadic(fn_ty);
 
-    if rust_argument_tys.len() != llvm_argument_tys.len() {
-        error!(
-            "Intrinsic signature mismatch: expected {} arguments for `{fn_name}`, found {} arguments",
-            llvm_argument_tys.len(),
-            rust_argument_tys.len()
-        );
+    if rust_is_variadic != llvm_is_variadic || rust_argument_tys.len() != llvm_argument_tys.len() {
+        return false;
     }
 
-    if !cx.equate_ty(rust_return_ty, llvm_return_ty) {
-        error!(
-            "Intrinsic signature mismatch: could not match `{rust_return_ty:?}` (found) with  {llvm_return_ty:?} (expected) as return type for `{fn_name}`"
-        );
-    }
-    for (idx, (&rust_argument_ty, llvm_argument_ty)) in
-        iter::zip(rust_argument_tys, llvm_argument_tys).enumerate()
-    {
-        if !cx.equate_ty(rust_argument_ty, llvm_argument_ty) {
-            error!(
-                "Intrinsic signature mismatch: could not match `{rust_return_ty:?}` (found) with  {llvm_return_ty:?} (expected) as argument {idx} for `{fn_name}`"
-            );
-        }
-    }
-
-    fn_ty
+    iter::once((&rust_return_ty, &llvm_return_ty))
+        .chain(iter::zip(rust_argument_tys, &llvm_argument_tys))
+        .all(|(rust_ty, llvm_ty)| cx.equate_ty(rust_ty, llvm_ty))
 }
 
 impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
@@ -407,19 +394,12 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
                 vector_size_bits == 8192
             }
             TypeKind::BFloat => rust_ty == self.type_i16(),
-            TypeKind::Vector if self.type_kind(rust_ty) == TypeKind::Vector => {
+            TypeKind::Vector => {
                 let llvm_element_count = self.vector_length(llvm_ty);
-                let rust_element_count = self.vector_length(rust_ty);
-
-                if llvm_element_count != rust_element_count {
-                    return false;
-                }
-
                 let llvm_element_ty = self.element_type(llvm_ty);
-                let rust_element_ty = self.element_type(rust_ty);
 
                 if llvm_element_ty == self.type_bf16() {
-                    rust_element_ty == self.type_i16()
+                    rust_ty == self.type_vector(self.type_i16(), llvm_element_count as u64)
                 } else {
                     false
                 }
@@ -502,41 +482,62 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         llargument_tys
     }
 
-    fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>, name: &[u8]) -> &'ll Type {
+    fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>, name: &[u8]) -> FunctionSignature<'ll> {
         let return_ty = self.llvm_return_type(cx);
         let argument_tys = self.llvm_argument_types(cx);
 
+        let mut maybe_invalid = false;
+
         if name.starts_with(b"llvm.") {
-            if let Some(intrinsic) = llvm::Intrinsic::lookup(name)
-                && !intrinsic.is_overloaded()
-            {
+            if let Some(intrinsic) = llvm::Intrinsic::lookup(name) {
                 if !intrinsic.is_overloaded() {
                     // FIXME: also do this for overloaded intrinsics
-                    return match_intrinsic_signature(
+                    let llvm_fn_ty = cx.intrinsic_type(intrinsic, &[]);
+                    if verify_intrinsic_signature(
                         cx,
-                        intrinsic,
+                        llvm_fn_ty,
                         return_ty,
                         &argument_tys,
-                        name,
-                    );
+                        self.c_variadic,
+                    ) {
+                        return FunctionSignature::Intrinsic(intrinsic, llvm_fn_ty);
+                    } else {
+                        // FIXME: ideally we would like to hard error, but there are edge cases like `i1xN`
+                        // and differences between struct representations that we have to tackle first
+
+                        let rust_fn_ty = if self.c_variadic {
+                            cx.type_variadic_func(&argument_tys, return_ty)
+                        } else {
+                            cx.type_func(&argument_tys, return_ty)
+                        };
+                        cx.tcx.dcx().note(format!(
+                            "Intrinsic signature mismatch for `{}`: expected signature `{llvm_fn_ty:?}`, found signature `{rust_fn_ty:?}`",
+                            str::from_utf8(name).unwrap()
+                        ));
+                        // fallback to the rust signature
+                        return FunctionSignature::Rust(rust_fn_ty);
+                    }
                 }
             } else {
                 // it's one of 2 cases,
                 // - either the base name is invalid
                 // - it has been superceded by something else, so the intrinsic was removed entirely
-                //
-                // anyway, let's log it
-                tracing::debug!(
-                    "Couldn't find intrinsic `{}`, either invalid or deprecated",
-                    str::from_utf8(name).unwrap()
-                );
+                // to check for upgrades, we need the `llfn`, so we defer it for now
+
+                maybe_invalid = true;
             }
         }
 
-        if self.c_variadic {
+        let fn_ty = if self.c_variadic {
             cx.type_variadic_func(&argument_tys, return_ty)
         } else {
             cx.type_func(&argument_tys, return_ty)
+        };
+
+        if maybe_invalid {
+            FunctionSignature::MaybeInvalidIntrinsic(fn_ty)
+        } else {
+            FunctionSignature::Rust(fn_ty)
         }
     }
 

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1705,10 +1705,12 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         let actual_ty = self.cx.val_ty(ret);
 
         if expected_ty != actual_ty {
-            warn!(
-                "Type mismatch in function call of {llfn:?}. \
-                    Expected {expected_ty:?} for return type, got {actual_ty:?}"
+            assert!(
+                self.cx.equate_ty(expected_ty, actual_ty),
+                "type mismatch in function call of `{llfn:?}`. \
+                        Can't match `{expected_ty:?}` (expected) as return type with `{actual_ty:?}` (found)",
             );
+
             if self.cx.type_kind(actual_ty) == TypeKind::X86_AMX {
                 self.cast_tile_to_vector(ret, expected_ty)
             } else {

compiler/rustc_codegen_llvm/src/declare.rs

@@ -22,7 +22,7 @@ use rustc_target::callconv::FnAbi;
 use smallvec::SmallVec;
 use tracing::debug;
 
-use crate::abi::FnAbiLlvmExt;
+use crate::abi::{FnAbiLlvmExt, FunctionSignature};
 use crate::common::AsCCharPtr;
 use crate::context::{CodegenCx, GenericCx, SCx};
 use crate::llvm::AttributePlace::Function;
@@ -150,17 +150,53 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
     ) -> &'ll Value {
         debug!("declare_rust_fn(name={:?}, fn_abi={:?})", name, fn_abi);
 
-        // Function addresses in Rust are never significant, allowing functions to
-        // be merged.
-        let llfn = declare_raw_fn(
-            self,
-            name,
-            fn_abi.llvm_cconv(self),
-            llvm::UnnamedAddr::Global,
-            llvm::Visibility::Default,
-            fn_abi.llvm_type(self, name.as_bytes()),
-        );
-        fn_abi.apply_attrs_llfn(self, llfn, instance);
+        let signature = fn_abi.llvm_type(self, name.as_bytes());
+        let llfn;
+
+        if let FunctionSignature::Intrinsic(intrinsic, fn_ty) = signature {
+            // intrinsics have a specified set of attributes, so we don't use the `FnAbi` set for them
+            llfn = declare_simple_fn(
+                self,
+                name.as_bytes(),
+                fn_abi.llvm_cconv(self),
+                llvm::UnnamedAddr::Global,
+                llvm::Visibility::Default,
+                fn_ty,
+            );
+            unsafe {
+                llvm::LLVMRustSetIntrinsicAttributes(self.llcx, llfn, intrinsic.id());
+            }
+        } else {
+            // Function addresses in Rust are never significant, allowing functions to
+            // be merged.
+            llfn = declare_raw_fn(
+                self,
+                name,
+                fn_abi.llvm_cconv(self),
+                llvm::UnnamedAddr::Global,
+                llvm::Visibility::Default,
+                signature.fn_ty(),
+            );
+            fn_abi.apply_attrs_llfn(self, llfn, instance);
+        }
+
+        if let FunctionSignature::MaybeInvalidIntrinsic(..) = signature {
+            let mut new_llfn = None;
+            let can_upgrade =
+                unsafe { llvm::LLVMRustUpgradeIntrinsicFunction(llfn, &mut new_llfn, false) };
+
+            if can_upgrade {
+                // not all intrinsics are upgraded to some other intrinsics, most are upgraded to instruction sequences
+                if let Some(new_llfn) = new_llfn {
+                    self.tcx.dcx().note(format!(
+                        "Using deprecated intrinsic `{name}`, `{}` can be used instead",
+                        str::from_utf8(llvm::get_value_name(new_llfn)).unwrap()
+                    ));
+                }
+            } else {
+                self.tcx.dcx().fatal(format!("Invalid LLVM intrinsic: `{name}`"))
+            }
+        }
 
         if self.tcx.sess.is_sanitizer_cfi_enabled() {
             if let Some(instance) = instance {

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -1089,7 +1089,7 @@ fn gen_fn<'a, 'll, 'tcx>(
     codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
 ) -> (&'ll Type, &'ll Value) {
     let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
-    let llty = fn_abi.llvm_type(cx, name.as_bytes());
+    let llty = fn_abi.llvm_type(cx, name.as_bytes()).fn_ty();
     let llfn = cx.declare_fn(name, fn_abi, None);
     cx.set_frame_pointer_type(llfn);
     cx.apply_target_cpu_attr(llfn);

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -1062,6 +1062,7 @@ unsafe extern "C" {
     ) -> &'a Type;
     pub(crate) fn LLVMCountParamTypes(FunctionTy: &Type) -> c_uint;
     pub(crate) fn LLVMGetParamTypes<'a>(FunctionTy: &'a Type, Dest: *mut &'a Type);
+    pub(crate) fn LLVMIsFunctionVarArg(FunctionTy: &Type) -> Bool;
 
     // Operations on struct types
     pub(crate) fn LLVMStructTypeInContext<'a>(
@@ -1204,10 +1205,6 @@ unsafe extern "C" {
         ParamTypes: *const &'a Type,
         ParamCount: size_t,
     ) -> &'a Type;
-    pub(crate) fn LLVMRustIntrinsicGetBaseName(
-        ID: c_uint,
-        NameLength: *mut size_t,
-    ) -> *const c_char;
     pub(crate) fn LLVMIntrinsicIsOverloaded(ID: c_uint) -> Bool;
     pub(crate) fn LLVMIntrinsicCopyOverloadedName2<'a>(
         Mod: &'a Module,
@@ -1216,6 +1213,12 @@ unsafe extern "C" {
         ParamCount: size_t,
         NameLength: *mut size_t,
     ) -> *mut c_char;
+    pub(crate) fn LLVMRustUpgradeIntrinsicFunction<'a>(
+        Fn: &'a Value,
+        NewFn: &mut Option<&'a Value>,
+        CanUpgradeDebugIntrinsicsToRecords: bool,
+    ) -> bool;
+    pub(crate) fn LLVMRustSetIntrinsicAttributes<'a>(C: &'a Context, Fn: &'a Value, ID: c_uint);
 
     // Operations on parameters
     pub(crate) fn LLVMIsAArgument(Val: &Value) -> Option<&Value>;

compiler/rustc_codegen_llvm/src/llvm/mod.rs

@@ -346,12 +346,6 @@ impl Intrinsic {
         unsafe { LLVMIntrinsicIsOverloaded(self.id) == True }
     }
 
-    pub(crate) fn base_name<'ll>(self) -> &'ll [u8] {
-        let mut len = 0;
-        let ptr = unsafe { LLVMRustIntrinsicGetBaseName(self.id, &mut len) };
-        unsafe { slice::from_raw_parts(ptr.cast(), len) }
-    }
-
     pub(crate) fn overloaded_name<'ll>(
         self,
         llmod: &'ll Module,

compiler/rustc_codegen_llvm/src/type_.rs

@@ -68,6 +68,10 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
             args
         }
     }
+
+    pub(crate) fn func_is_variadic(&self, ty: &'ll Type) -> bool {
+        unsafe { llvm::LLVMIsFunctionVarArg(ty) == True }
+    }
 }
 impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
     pub(crate) fn type_bool(&self) -> &'ll Type {
@@ -288,7 +292,7 @@ impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
         fn_ptr: &'ll Value,
     ) -> &'ll Type {
-        fn_abi.llvm_type(self, llvm::get_value_name(fn_ptr))
+        fn_abi.llvm_type(self, llvm::get_value_name(fn_ptr)).fn_ty()
     }
     fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type {
         fn_abi.ptr_to_llvm_type(self)