Check for AutoUpgraded intrinsics, and emit a hard error for unknown intrinsics

sayantn · sayantn · commit 260f09811c35 · 2025-09-04T12:34:12.000+05:30
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs
@@ -7,6 +7,7 @@ use rustc_abi::{
     X86Call,
 };
 use rustc_codegen_ssa::MemFlags;
+use rustc_codegen_ssa::common::TypeKind;
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
 use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
 use rustc_codegen_ssa::traits::*;
@@ -22,7 +23,7 @@ use smallvec::SmallVec;
 
 use crate::attributes::{self, llfn_attrs_from_instance};
 use crate::builder::Builder;
-use crate::context::CodegenCx;
+use crate::context::{CodegenCx, GenericCx, SCx};
 use crate::llvm::{self, Attribute, AttributePlace};
 use crate::llvm_util;
 use crate::type_::Type;
@@ -359,6 +360,36 @@ pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> {
     );
 }
 
+impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
+    pub(crate) fn equate_ty(&self, rust_ty: &'ll Type, llvm_ty: &'ll Type) -> bool {
+        if rust_ty == llvm_ty {
+            return true;
+        }
+
+        // Some LLVM intrinsics return **non-packed** structs, but they can't be mimicked from Rust
+        // due to auto field-alignment in non-packed structs (packed structs are represented in LLVM
+        // as, well, packed structs, so they won't match with those either)
+        if self.type_kind(llvm_ty) == TypeKind::Struct
+            && self.type_kind(rust_ty) == TypeKind::Struct
+        {
+            let rust_element_tys = self.struct_element_types(rust_ty);
+            let llvm_element_tys = self.struct_element_types(llvm_ty);
+
+            if rust_element_tys.len() != llvm_element_tys.len() {
+                return false;
+            }
+
+            iter::zip(rust_element_tys, llvm_element_tys).all(
+                |(rust_element_ty, llvm_element_ty)| {
+                    self.equate_ty(rust_element_ty, llvm_element_ty)
+                },
+            )
+        } else {
+            false
+        }
+    }
+}
+
 impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
     fn llvm_return_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
         match &self.ret.mode {
@@ -448,7 +479,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
 
         iter::once((rust_return_ty, llvm_return_ty))
             .chain(iter::zip(rust_argument_tys, llvm_argument_tys))
-            .all(|(rust_ty, llvm_ty)| rust_ty == llvm_ty)
+            .all(|(rust_ty, llvm_ty)| cx.equate_ty(rust_ty, llvm_ty))
     }
 
     fn llvm_type(
diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs
@@ -68,7 +68,6 @@ impl<'a, 'll> SBuilder<'a, 'll> {
     ) -> &'ll Value {
         debug!("call {:?} with args ({:?})", llfn, args);
 
-        let args = self.check_call("call", llty, llfn, args);
         let funclet_bundle = funclet.map(|funclet| funclet.bundle());
         let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
         if let Some(funclet_bundle) = funclet_bundle {
@@ -415,7 +414,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     ) -> &'ll Value {
         debug!("invoke {:?} with args ({:?})", llfn, args);
 
-        let args = self.check_call("invoke", llty, llfn, args);
+        let args = self.cast_arguments("invoke", llty, llfn, args, fn_abi.is_some());
         let funclet_bundle = funclet.map(|funclet| funclet.bundle());
         let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
         if let Some(funclet_bundle) = funclet_bundle {
@@ -447,8 +446,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         };
         if let Some(fn_abi) = fn_abi {
             fn_abi.apply_attrs_callsite(self, invoke, llfn);
+            self.cast_return(fn_abi, llfn, invoke)
+        } else {
+            invoke
         }
-        invoke
     }
 
     fn unreachable(&mut self) {
@@ -1401,7 +1402,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     ) -> &'ll Value {
         debug!("call {:?} with args ({:?})", llfn, args);
 
-        let args = self.check_call("call", llty, llfn, args);
+        let args = self.cast_arguments("call", llty, llfn, args, fn_abi.is_some());
         let funclet_bundle = funclet.map(|funclet| funclet.bundle());
         let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
         if let Some(funclet_bundle) = funclet_bundle {
@@ -1454,8 +1455,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 
         if let Some(fn_abi) = fn_abi {
             fn_abi.apply_attrs_callsite(self, call, llfn);
+            self.cast_return(fn_abi, llfn, call)
+        } else {
+            call
         }
-        call
     }
 
     fn tail_call(
@@ -1645,47 +1648,6 @@ impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
         ret.expect("LLVM does not have support for catchret")
     }
 
-    fn check_call<'b>(
-        &mut self,
-        typ: &str,
-        fn_ty: &'ll Type,
-        llfn: &'ll Value,
-        args: &'b [&'ll Value],
-    ) -> Cow<'b, [&'ll Value]> {
-        assert!(
-            self.cx.type_kind(fn_ty) == TypeKind::Function,
-            "builder::{typ} not passed a function, but {fn_ty:?}"
-        );
-
-        let param_tys = self.cx.func_params_types(fn_ty);
-
-        let all_args_match = iter::zip(&param_tys, args.iter().map(|&v| self.cx.val_ty(v)))
-            .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty);
-
-        if all_args_match {
-            return Cow::Borrowed(args);
-        }
-
-        let casted_args: Vec<_> = iter::zip(param_tys, args)
-            .enumerate()
-            .map(|(i, (expected_ty, &actual_val))| {
-                let actual_ty = self.cx.val_ty(actual_val);
-                if expected_ty != actual_ty {
-                    debug!(
-                        "type mismatch in function call of {:?}. \
-                            Expected {:?} for param {}, got {:?}; injecting bitcast",
-                        llfn, expected_ty, i, actual_ty
-                    );
-                    self.bitcast(actual_val, expected_ty)
-                } else {
-                    actual_val
-                }
-            })
-            .collect();
-
-        Cow::Owned(casted_args)
-    }
-
     pub(crate) fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
         unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
     }
@@ -1741,6 +1703,93 @@ impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
     }
 }
 impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
+    fn autocast(
+        &mut self,
+        llfn: &'ll Value,
+        val: &'ll Value,
+        src_ty: &'ll Type,
+        dest_ty: &'ll Type,
+        is_argument: bool,
+    ) -> &'ll Value {
+        let (rust_ty, llvm_ty) = if is_argument { (src_ty, dest_ty) } else { (dest_ty, src_ty) };
+
+        if rust_ty == llvm_ty {
+            return val;
+        }
+
+        match self.type_kind(llvm_ty) {
+            TypeKind::Struct => {
+                let mut ret = self.const_poison(dest_ty);
+                for (idx, (src_element_ty, dest_element_ty)) in
+                    iter::zip(self.struct_element_types(src_ty), self.struct_element_types(dest_ty))
+                        .enumerate()
+                {
+                    let elt = self.extract_value(val, idx as u64);
+                    let casted_elt =
+                        self.autocast(llfn, elt, src_element_ty, dest_element_ty, is_argument);
+                    ret = self.insert_value(ret, casted_elt, idx as u64);
+                }
+                ret
+            }
+            _ => unreachable!(),
+        }
+    }
+
+    fn cast_arguments<'b>(
+        &mut self,
+        typ: &str,
+        fn_ty: &'ll Type,
+        llfn: &'ll Value,
+        args: &'b [&'ll Value],
+        has_fnabi: bool,
+    ) -> Cow<'b, [&'ll Value]> {
+        assert_eq!(
+            self.type_kind(fn_ty),
+            TypeKind::Function,
+            "{typ} not passed a function, but {fn_ty:?}"
+        );
+
+        let param_tys = self.func_params_types(fn_ty);
+
+        let mut casted_args = Cow::Borrowed(args);
+
+        for (idx, (dest_ty, &arg)) in iter::zip(param_tys, args).enumerate() {
+            let src_ty = self.val_ty(arg);
+            assert!(
+                self.equate_ty(src_ty, dest_ty),
+                "Cannot match `{dest_ty:?}` (expected) with `{src_ty:?}` (found) in `{llfn:?}`"
+            );
+
+            let casted_arg = self.autocast(llfn, arg, src_ty, dest_ty, true);
+            if arg != casted_arg {
+                assert!(
+                    has_fnabi,
+                    "Should inject autocasts in function call of {llfn:?}, but not able to get Rust signature"
+                );
+
+                casted_args.to_mut()[idx] = casted_arg;
+            }
+        }
+
+        casted_args
+    }
+
+    fn cast_return(
+        &mut self,
+        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
+        llfn: &'ll Value,
+        ret: &'ll Value,
+    ) -> &'ll Value {
+        let src_ty = self.val_ty(ret);
+        let dest_ty = fn_abi.llvm_return_type(self);
+        assert!(
+            self.equate_ty(dest_ty, src_ty),
+            "Cannot match `{src_ty:?}` (expected) with `{dest_ty:?}` (found) in `{llfn:?}`"
+        );
+
+        self.autocast(llfn, ret, src_ty, dest_ty, false)
+    }
+
     pub(crate) fn landing_pad(
         &mut self,
         ty: &'ll Type,
@@ -1770,7 +1819,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
     ) -> &'ll Value {
         debug!("invoke {:?} with args ({:?})", llfn, args);
 
-        let args = self.check_call("callbr", llty, llfn, args);
+        let args = self.cast_arguments("callbr", llty, llfn, args, fn_abi.is_some());
         let funclet_bundle = funclet.map(|funclet| funclet.bundle());
         let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
         if let Some(funclet_bundle) = funclet_bundle {
@@ -1803,8 +1852,10 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         };
         if let Some(fn_abi) = fn_abi {
             fn_abi.apply_attrs_callsite(self, callbr, llfn);
+            self.cast_return(fn_abi, llfn, callbr)
+        } else {
+            callbr
         }
-        callbr
     }
 
     // Emits CFI pointer type membership tests.
diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs
@@ -177,7 +177,27 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
             fn_abi.apply_attrs_llfn(self, llfn, instance);
         }
 
-        // todo: check for upgrades, and emit error if not upgradable
+        if let FunctionSignature::MaybeInvalidIntrinsic(..) = signature {
+            let mut new_llfn = None;
+            let can_upgrade =
+                unsafe { llvm::LLVMRustUpgradeIntrinsicFunction(llfn, &mut new_llfn, false) };
+
+            if can_upgrade {
+                // not all intrinsics are upgraded to some other intrinsics, most are upgraded to instruction sequences
+                if let Some(new_llfn) = new_llfn {
+                    self.tcx.dcx().note(format!(
+                        "Using deprecated intrinsic `{name}`, `{}` can be used instead",
+                        str::from_utf8(&llvm::get_value_name(new_llfn)).unwrap()
+                    ));
+                } else {
+                    self.tcx.dcx().note(format!(
+                        "Using deprecated intrinsic `{name}`, consider using other intrinsics/instructions"
+                    ));
+                }
+            } else {
+                self.tcx.dcx().fatal(format!("Invalid LLVM intrinsic: `{name}`"))
+            }
+        }
 
         if self.tcx.sess.is_sanitizer_cfi_enabled() {
             if let Some(instance) = instance {
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -1267,6 +1267,11 @@ unsafe extern "C" {
         ParamTypes: *const &'a Type,
         ParamCount: size_t,
     ) -> &'a Value;
+    pub(crate) fn LLVMRustUpgradeIntrinsicFunction<'a>(
+        Fn: &'a Value,
+        NewFn: &mut Option<&'a Value>,
+        CanUpgradeDebugIntrinsicsToRecords: bool,
+    ) -> bool;
 
     // Operations on parameters
     pub(crate) fn LLVMIsAArgument(Val: &Value) -> Option<&Value>;
@@ -1783,6 +1788,9 @@ unsafe extern "C" {
         Packed: Bool,
     );
 
+    pub(crate) fn LLVMCountStructElementTypes(StructTy: &Type) -> c_uint;
+    pub(crate) fn LLVMGetStructElementTypes<'a>(StructTy: &'a Type, Dest: *mut &'a Type);
+
     pub(crate) safe fn LLVMMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value;
 
     pub(crate) safe fn LLVMSetUnnamedAddress(Global: &Value, UnnamedAddr: UnnamedAddr);
diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs
@@ -87,6 +87,16 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
     pub(crate) fn func_is_variadic(&self, ty: &'ll Type) -> bool {
         unsafe { llvm::LLVMIsFunctionVarArg(ty).is_true() }
     }
+
+    pub(crate) fn struct_element_types(&self, ty: &'ll Type) -> Vec<&'ll Type> {
+        unsafe {
+            let n_args = llvm::LLVMCountStructElementTypes(ty) as usize;
+            let mut args = Vec::with_capacity(n_args);
+            llvm::LLVMGetStructElementTypes(ty, args.as_mut_ptr());
+            args.set_len(n_args);
+            args
+        }
+    }
 }
 impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
     pub(crate) fn type_bool(&self) -> &'ll Type {
diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
@@ -9,6 +9,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/Magic.h"
 #include "llvm/Bitcode/BitcodeWriter.h"
+#include "llvm/IR/AutoUpgrade.h"
 #include "llvm/IR/DIBuilder.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/DiagnosticHandler.h"
@@ -1897,6 +1898,17 @@ extern "C" void LLVMRustGetMangledName(LLVMValueRef V, RustStringRef Str) {
   Mangler().getNameWithPrefix(OS, GV, true);
 }
 
+extern "C" bool
+LLVMRustUpgradeIntrinsicFunction(LLVMValueRef Fn, LLVMValueRef *NewFn,
+                                 bool canUpgradeDebugIntrinsicsToRecords) {
+  Function *F = unwrap<Function>(Fn);
+  Function *NewF = nullptr;
+  bool CanUpgrade =
+      UpgradeIntrinsicFunction(F, NewF, canUpgradeDebugIntrinsicsToRecords);
+  *NewFn = wrap(NewF);
+  return CanUpgrade;
+}
+
 extern "C" int32_t LLVMRustGetElementTypeArgIndex(LLVMValueRef CallSite) {
   auto *CB = unwrap<CallBase>(CallSite);
   switch (CB->getIntrinsicID()) {
