From 6be93fe9d3e5bde34bfaa6a0ad9b888a8dd001ab Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 5 Jun 2025 11:33:03 +0000 Subject: [PATCH 1/7] Split out ABI handling for LLVM intrinsics --- compiler/rustc_codegen_ssa/src/mir/block.rs | 86 ++++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index cce33107e2c27..2d779c504d8f9 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -14,7 +14,7 @@ use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; use rustc_span::Span; use rustc_span::source_map::Spanned; -use rustc_target::callconv::{ArgAbi, CastTarget, FnAbi, PassMode}; +use rustc_target::callconv::{ArgAbi, ArgAttributes, CastTarget, FnAbi, PassMode}; use tracing::{debug, info}; use super::operand::OperandRef; @@ -1036,6 +1036,90 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { _ => bug!("{} is not callable", callee.layout.ty), }; + if let Some(instance) = instance + && let Some(name) = bx.tcx().codegen_fn_attrs(instance.def_id()).symbol_name + && name.as_str().starts_with("llvm.") + { + let mut llargs = Vec::with_capacity(args.len()); + + let dest_ty = destination.ty(&self.mir.local_decls, bx.tcx()).ty; + let return_dest = if dest_ty.is_unit() { + ReturnDest::Nothing + } else if let Some(index) = destination.as_local() { + match self.locals[index] { + LocalRef::Place(dest) => ReturnDest::Store(dest), + LocalRef::UnsizedPlace(_) => bug!("return type must be sized"), + LocalRef::PendingOperand => { + // Handle temporary places, specifically `Operand` ones, as + // they don't have `alloca`s. + ReturnDest::DirectOperand(index) + } + LocalRef::Operand(_) => bug!("place local already assigned to"), + } + } else { + ReturnDest::Store(self.codegen_place(bx, destination.as_ref())) + }; + + for arg in args { + let op = self.codegen_operand(bx, &arg.node); + + match op.val { + ZeroSized => {} + Immediate(_) => llargs.push(op.immediate()), + Pair(a, b) => { + llargs.push(a); + llargs.push(b); + } + Ref(op_place_val) => { + let mut llval = op_place_val.llval; + // We can't use `PlaceRef::load` here because the argument + // may have a type we don't treat as immediate, but the ABI + // used for this call is passing it by-value. In that case, + // the load would just produce `OperandValue::Ref` instead + // of the `OperandValue::Immediate` we need for the call. + llval = bx.load(bx.backend_type(op.layout), llval, op_place_val.align); + if let BackendRepr::Scalar(scalar) = op.layout.backend_repr { + if scalar.is_bool() { + bx.range_metadata(llval, WrappingRange { start: 0, end: 1 }); + } + // We store bools as `i8` so we need to truncate to `i1`. + llval = bx.to_immediate_scalar(llval, scalar); + } + llargs.push(llval); + } + } + } + + let fn_ptr = bx.get_fn_addr(instance); + self.set_debug_loc(bx, source_info); + + // FIXME remove usage of fn_abi + let fn_abi = bx.fn_abi_of_instance(instance, ty::List::empty()); + assert!(!fn_abi.ret.is_indirect()); + let fn_ty = bx.fn_decl_backend_type(fn_abi); + + let llret = bx.call(fn_ty, None, None, fn_ptr, &llargs, None, None); + if self.mir[helper.bb].is_cleanup { + bx.apply_attrs_to_cleanup_callsite(llret); + } + + if let Some(target) = target { + self.store_return( + bx, + return_dest, + &ArgAbi { + layout: bx.layout_of(dest_ty), + mode: PassMode::Direct(ArgAttributes::new()), + }, + llret, + ); + return helper.funclet_br(self, bx, target, mergeable_succ); + } else { + bx.unreachable(); + return MergingSucc::False; + } + } + // FIXME(eddyb) avoid computing this if possible, when `instance` is // available - right now `sig` is only needed for getting the `abi` // and figuring out how many extra args were passed to a C-variadic `fn`. From 60030491fa9d54bc7add140eb091635903b88250 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 18 Sep 2025 13:25:01 +0000 Subject: [PATCH 2/7] Move llvm intrinsic call to backend --- .../rustc_codegen_gcc/src/intrinsic/mod.rs | 52 +++++++++++++++++- compiler/rustc_codegen_llvm/src/intrinsic.rs | 54 ++++++++++++++++++- compiler/rustc_codegen_ssa/src/mir/block.rs | 45 ++-------------- .../rustc_codegen_ssa/src/traits/intrinsic.rs | 7 +++ 4 files changed, 114 insertions(+), 44 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 8225df5686417..217a400c7f8f3 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -9,7 +9,7 @@ use gccjit::Type; use gccjit::{ComparisonOp, Function, FunctionType, RValue, ToRValue, UnaryOp}; #[cfg(feature = "master")] use rustc_abi::ExternAbi; -use rustc_abi::{BackendRepr, HasDataLayout}; +use rustc_abi::{BackendRepr, HasDataLayout, WrappingRange}; use rustc_codegen_ssa::MemFlags; use rustc_codegen_ssa::base::wants_msvc_seh; use rustc_codegen_ssa::common::IntPredicate; @@ -20,7 +20,7 @@ use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::MiscCodegenMethods; use rustc_codegen_ssa::traits::{ ArgAbiBuilderMethods, BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, - IntrinsicCallBuilderMethods, + IntrinsicCallBuilderMethods, LayoutTypeCodegenMethods, }; use rustc_middle::bug; #[cfg(feature = "master")] @@ -636,6 +636,54 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc Ok(()) } + fn codegen_llvm_intrinsic_call( + &mut self, + instance: ty::Instance<'tcx>, + args: &[OperandRef<'tcx, Self::Value>], + is_cleanup: bool, + ) -> Self::Value { + let fn_ptr = self.get_fn_addr(instance); + let fn_ty = fn_ptr.get_type(); + + let mut llargs = vec![]; + + for arg in args { + match arg.val { + OperandValue::ZeroSized => {} + OperandValue::Immediate(_) => llargs.push(arg.immediate()), + OperandValue::Pair(a, b) => { + llargs.push(a); + llargs.push(b); + } + OperandValue::Ref(op_place_val) => { + let mut llval = op_place_val.llval; + // We can't use `PlaceRef::load` here because the argument + // may have a type we don't treat as immediate, but the ABI + // used for this call is passing it by-value. In that case, + // the load would just produce `OperandValue::Ref` instead + // of the `OperandValue::Immediate` we need for the call. + llval = self.load(self.backend_type(arg.layout), llval, op_place_val.align); + if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr { + if scalar.is_bool() { + self.range_metadata(llval, WrappingRange { start: 0, end: 1 }); + } + // We store bools as `i8` so we need to truncate to `i1`. + llval = self.to_immediate_scalar(llval, scalar); + } + llargs.push(llval); + } + } + } + + // FIXME directly use the llvm intrinsic adjustment functions here + let llret = self.call(fn_ty, None, None, fn_ptr, &llargs, None, None); + if is_cleanup { + self.apply_attrs_to_cleanup_callsite(llret); + } + + llret + } + fn abort(&mut self) { let func = self.context.get_builtin_function("abort"); let func: RValue<'gcc> = unsafe { std::mem::transmute(func) }; diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 84fc6ebbc3172..d76ee4e0610dc 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1,7 +1,9 @@ use std::assert_matches::assert_matches; use std::cmp::Ordering; -use rustc_abi::{Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size}; +use rustc_abi::{ + Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size, WrappingRange, +}; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}; use rustc_codegen_ssa::codegen_attrs::autodiff_attrs; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; @@ -608,6 +610,56 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { Ok(()) } + fn codegen_llvm_intrinsic_call( + &mut self, + instance: ty::Instance<'tcx>, + args: &[OperandRef<'tcx, Self::Value>], + is_cleanup: bool, + ) -> Self::Value { + let fn_ptr = self.get_fn_addr(instance); + // FIXME remove usage of fn_abi + let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); + assert!(!fn_abi.ret.is_indirect()); + let fn_ty = self.fn_decl_backend_type(fn_abi); + + let mut llargs = vec![]; + + for arg in args { + match arg.val { + OperandValue::ZeroSized => {} + OperandValue::Immediate(_) => llargs.push(arg.immediate()), + OperandValue::Pair(a, b) => { + llargs.push(a); + llargs.push(b); + } + OperandValue::Ref(op_place_val) => { + let mut llval = op_place_val.llval; + // We can't use `PlaceRef::load` here because the argument + // may have a type we don't treat as immediate, but the ABI + // used for this call is passing it by-value. In that case, + // the load would just produce `OperandValue::Ref` instead + // of the `OperandValue::Immediate` we need for the call. + llval = self.load(self.backend_type(arg.layout), llval, op_place_val.align); + if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr { + if scalar.is_bool() { + self.range_metadata(llval, WrappingRange { start: 0, end: 1 }); + } + // We store bools as `i8` so we need to truncate to `i1`. + llval = self.to_immediate_scalar(llval, scalar); + } + llargs.push(llval); + } + } + } + + let llret = self.call(fn_ty, None, None, fn_ptr, &llargs, None, None); + if is_cleanup { + self.apply_attrs_to_cleanup_callsite(llret); + } + + llret + } + fn abort(&mut self) { self.call_intrinsic("llvm.trap", &[], &[]); } diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 2d779c504d8f9..7f1ae81a15ce5 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -1040,8 +1040,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { && let Some(name) = bx.tcx().codegen_fn_attrs(instance.def_id()).symbol_name && name.as_str().starts_with("llvm.") { - let mut llargs = Vec::with_capacity(args.len()); - let dest_ty = destination.ty(&self.mir.local_decls, bx.tcx()).ty; let return_dest = if dest_ty.is_unit() { ReturnDest::Nothing @@ -1060,48 +1058,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { ReturnDest::Store(self.codegen_place(bx, destination.as_ref())) }; - for arg in args { - let op = self.codegen_operand(bx, &arg.node); - - match op.val { - ZeroSized => {} - Immediate(_) => llargs.push(op.immediate()), - Pair(a, b) => { - llargs.push(a); - llargs.push(b); - } - Ref(op_place_val) => { - let mut llval = op_place_val.llval; - // We can't use `PlaceRef::load` here because the argument - // may have a type we don't treat as immediate, but the ABI - // used for this call is passing it by-value. In that case, - // the load would just produce `OperandValue::Ref` instead - // of the `OperandValue::Immediate` we need for the call. - llval = bx.load(bx.backend_type(op.layout), llval, op_place_val.align); - if let BackendRepr::Scalar(scalar) = op.layout.backend_repr { - if scalar.is_bool() { - bx.range_metadata(llval, WrappingRange { start: 0, end: 1 }); - } - // We store bools as `i8` so we need to truncate to `i1`. - llval = bx.to_immediate_scalar(llval, scalar); - } - llargs.push(llval); - } - } - } + let args = + args.into_iter().map(|arg| self.codegen_operand(bx, &arg.node)).collect::>(); - let fn_ptr = bx.get_fn_addr(instance); self.set_debug_loc(bx, source_info); - // FIXME remove usage of fn_abi - let fn_abi = bx.fn_abi_of_instance(instance, ty::List::empty()); - assert!(!fn_abi.ret.is_indirect()); - let fn_ty = bx.fn_decl_backend_type(fn_abi); - - let llret = bx.call(fn_ty, None, None, fn_ptr, &llargs, None, None); - if self.mir[helper.bb].is_cleanup { - bx.apply_attrs_to_cleanup_callsite(llret); - } + let llret = + bx.codegen_llvm_intrinsic_call(instance, &args, self.mir[helper.bb].is_cleanup); if let Some(target) = target { self.store_return( diff --git a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs index c5ecf43046c74..6761f6c512964 100644 --- a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs @@ -25,6 +25,13 @@ pub trait IntrinsicCallBuilderMethods<'tcx>: BackendTypes { span: Span, ) -> Result<(), ty::Instance<'tcx>>; + fn codegen_llvm_intrinsic_call( + &mut self, + instance: ty::Instance<'tcx>, + args: &[OperandRef<'tcx, Self::Value>], + is_cleanup: bool, + ) -> Self::Value; + fn abort(&mut self); fn assume(&mut self, val: Self::Value); fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value; From 730ccff93859d862243aa27ca604fdf4011dc818 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 17 Nov 2025 18:29:05 +0000 Subject: [PATCH 3/7] Fix for uninhabited llvm intrinsic return type --- compiler/rustc_codegen_ssa/src/mir/block.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 7f1ae81a15ce5..560880a76e51a 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -1040,8 +1040,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { && let Some(name) = bx.tcx().codegen_fn_attrs(instance.def_id()).symbol_name && name.as_str().starts_with("llvm.") { - let dest_ty = destination.ty(&self.mir.local_decls, bx.tcx()).ty; - let return_dest = if dest_ty.is_unit() { + let result_layout = + self.cx.layout_of(self.monomorphized_place_ty(destination.as_ref())); + + let return_dest = if result_layout.is_zst() { ReturnDest::Nothing } else if let Some(index) = destination.as_local() { match self.locals[index] { @@ -1070,10 +1072,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { self.store_return( bx, return_dest, - &ArgAbi { - layout: bx.layout_of(dest_ty), - mode: PassMode::Direct(ArgAttributes::new()), - }, + &ArgAbi { layout: result_layout, mode: PassMode::Direct(ArgAttributes::new()) }, llret, ); return helper.funclet_br(self, bx, target, mergeable_succ); From 2fc922beabe117f3388be53856a4f48998b1e256 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 11 Nov 2025 11:03:25 +0100 Subject: [PATCH 4/7] Don't assume get_fn is only called from codegen_mir in cg_gcc --- compiler/rustc_codegen_gcc/src/builder.rs | 1 + compiler/rustc_codegen_gcc/src/context.rs | 4 +--- compiler/rustc_codegen_gcc/src/declare.rs | 8 ++------ 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 132c43ef3cdad..cd8ebd064194f 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -514,6 +514,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { type CodegenCx = CodegenCx<'gcc, 'tcx>; fn build(cx: &'a CodegenCx<'gcc, 'tcx>, block: Block<'gcc>) -> Builder<'a, 'gcc, 'tcx> { + *cx.current_func.borrow_mut() = Some(block.get_function()); Builder::with_cx(cx, block) } diff --git a/compiler/rustc_codegen_gcc/src/context.rs b/compiler/rustc_codegen_gcc/src/context.rs index c9ae96777de44..453214edf2e0f 100644 --- a/compiler/rustc_codegen_gcc/src/context.rs +++ b/compiler/rustc_codegen_gcc/src/context.rs @@ -391,9 +391,7 @@ impl<'gcc, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> { } fn get_fn(&self, instance: Instance<'tcx>) -> Function<'gcc> { - let func = get_fn(self, instance); - *self.current_func.borrow_mut() = Some(func); - func + get_fn(self, instance) } fn get_fn_addr(&self, instance: Instance<'tcx>) -> RValue<'gcc> { diff --git a/compiler/rustc_codegen_gcc/src/declare.rs b/compiler/rustc_codegen_gcc/src/declare.rs index 691fd8729e39a..43225662762f3 100644 --- a/compiler/rustc_codegen_gcc/src/declare.rs +++ b/compiler/rustc_codegen_gcc/src/declare.rs @@ -100,18 +100,14 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { let return_type = self.type_i32(); let variadic = false; self.linkage.set(FunctionType::Exported); - let func = declare_raw_fn( + declare_raw_fn( self, name, callconv, return_type, &[self.type_i32(), const_string], variadic, - ); - // NOTE: it is needed to set the current_func here as well, because get_fn() is not called - // for the main function. - *self.current_func.borrow_mut() = Some(func); - func + ) } pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Function<'gcc> { From e5dd38bbe7a10f7fb0186280353c2e5076a25dad Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 11 Nov 2025 11:14:03 +0100 Subject: [PATCH 5/7] Pass Function to Builder::function_call in cg_gcc --- compiler/rustc_codegen_gcc/src/builder.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index cd8ebd064194f..0c9a8d4647623 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -316,12 +316,10 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { fn function_call( &mut self, - func: RValue<'gcc>, + func: Function<'gcc>, args: &[RValue<'gcc>], _funclet: Option<&Funclet>, ) -> RValue<'gcc> { - // TODO(antoyo): remove when the API supports a different type for functions. - let func: Function<'gcc> = self.cx.rvalue_as_function(func); let args = self.check_call("call", func, args); // gccjit requires to use the result of functions, even when it's not used. @@ -1763,6 +1761,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { // FIXME(antoyo): remove when having a proper API. let gcc_func = unsafe { std::mem::transmute::, Function<'gcc>>(func) }; let call = if self.functions.borrow().values().any(|value| *value == gcc_func) { + // TODO(antoyo): remove when the API supports a different type for functions. + let func: Function<'gcc> = self.cx.rvalue_as_function(func); self.function_call(func, args, funclet) } else { // If it's a not function that was defined, it's a function pointer. From 539a6a41c90fbbe214cd82906f84cfedbc34390c Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 11 Nov 2025 11:52:43 +0100 Subject: [PATCH 6/7] Inline BuilderMethods::call for intrinsics Intrinsics only need a fraction of the functionality offered by BuilderMethods::call and in particular don't need the FnAbi to be computed other than (currently) as step towards computing the function value type. --- compiler/rustc_codegen_llvm/src/builder.rs | 2 +- compiler/rustc_codegen_llvm/src/intrinsic.rs | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index b10a1282f4dd0..756b1fb9fe775 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1682,7 +1682,7 @@ impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { ret.expect("LLVM does not have support for catchret") } - fn check_call<'b>( + pub(crate) fn check_call<'b>( &mut self, typ: &str, fn_ty: &'ll Type, diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index d76ee4e0610dc..f8f4d515501e9 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1,5 +1,7 @@ use std::assert_matches::assert_matches; use std::cmp::Ordering; +use std::ffi::c_uint; +use std::ptr; use rustc_abi::{ Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size, WrappingRange, @@ -652,7 +654,20 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } - let llret = self.call(fn_ty, None, None, fn_ptr, &llargs, None, None); + debug!("call intrinsic {:?} with args ({:?})", instance, llargs); + let args = self.check_call("call", fn_ty, fn_ptr, &llargs); + let llret = unsafe { + llvm::LLVMBuildCallWithOperandBundles( + self.llbuilder, + fn_ty, + fn_ptr, + args.as_ptr() as *const &llvm::Value, + args.len() as c_uint, + ptr::dangling(), + 0, + c"".as_ptr(), + ) + }; if is_cleanup { self.apply_attrs_to_cleanup_callsite(llret); } From b32b23239718cab44f94f8d9873d43d0092c0e17 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 10 Nov 2025 13:42:32 +0000 Subject: [PATCH 7/7] Partially inline get_fn_addr/get_fn in codegen_llvm_intrinsic_call This moves all LLVM intrinsic handling out of the regular call path for cg_gcc and makes it easier to hook into this code for future cg_llvm changes. --- compiler/rustc_codegen_gcc/src/builder.rs | 2 +- compiler/rustc_codegen_gcc/src/context.rs | 7 ++-- compiler/rustc_codegen_gcc/src/declare.rs | 14 ------- .../rustc_codegen_gcc/src/intrinsic/mod.rs | 42 ++++++++++++++++++- compiler/rustc_codegen_llvm/src/context.rs | 3 ++ compiler/rustc_codegen_llvm/src/intrinsic.rs | 35 +++++++++++++++- compiler/rustc_codegen_ssa/src/mir/block.rs | 5 ++- 7 files changed, 86 insertions(+), 22 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 0c9a8d4647623..d634166b0c508 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -314,7 +314,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { self.block.get_function() } - fn function_call( + pub fn function_call( &mut self, func: Function<'gcc>, args: &[RValue<'gcc>], diff --git a/compiler/rustc_codegen_gcc/src/context.rs b/compiler/rustc_codegen_gcc/src/context.rs index 453214edf2e0f..f437d44e48b17 100644 --- a/compiler/rustc_codegen_gcc/src/context.rs +++ b/compiler/rustc_codegen_gcc/src/context.rs @@ -92,6 +92,8 @@ pub struct CodegenCx<'gcc, 'tcx> { pub instances: RefCell, LValue<'gcc>>>, /// Cache function instances of monomorphic and polymorphic items pub function_instances: RefCell, Function<'gcc>>>, + /// Cache function instances of intrinsics + pub intrinsic_instances: RefCell, Function<'gcc>>>, /// Cache generated vtables pub vtables: RefCell, Option>), RValue<'gcc>>>, @@ -280,6 +282,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { linkage: Cell::new(FunctionType::Internal), instances: Default::default(), function_instances: Default::default(), + intrinsic_instances: Default::default(), on_stack_params: Default::default(), on_stack_function_params: Default::default(), vtables: Default::default(), @@ -397,9 +400,7 @@ impl<'gcc, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> { fn get_fn_addr(&self, instance: Instance<'tcx>) -> RValue<'gcc> { let func_name = self.tcx.symbol_name(instance).name; - let func = if self.intrinsics.borrow().contains_key(func_name) { - self.intrinsics.borrow()[func_name] - } else if let Some(variable) = self.get_declared_value(func_name) { + let func = if let Some(variable) = self.get_declared_value(func_name) { return variable; } else { get_fn(self, instance) diff --git a/compiler/rustc_codegen_gcc/src/declare.rs b/compiler/rustc_codegen_gcc/src/declare.rs index 43225662762f3..706f9bd1f64f4 100644 --- a/compiler/rustc_codegen_gcc/src/declare.rs +++ b/compiler/rustc_codegen_gcc/src/declare.rs @@ -8,7 +8,6 @@ use rustc_target::callconv::FnAbi; use crate::abi::{FnAbiGcc, FnAbiGccExt}; use crate::context::CodegenCx; -use crate::intrinsic::llvm; impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { pub fn get_or_insert_global( @@ -162,19 +161,6 @@ fn declare_raw_fn<'gcc>( param_types: &[Type<'gcc>], variadic: bool, ) -> Function<'gcc> { - if name.starts_with("llvm.") { - let intrinsic = match name { - "llvm.fma.f16" => { - // fma is not a target builtin, but a normal builtin, so we handle it differently - // here. - cx.context.get_builtin_function("fma") - } - _ => llvm::intrinsic(name, cx), - }; - - cx.intrinsics.borrow_mut().insert(name.to_string(), intrinsic); - return intrinsic; - } let func = if cx.functions.borrow().contains_key(name) { cx.functions.borrow()[name] } else { diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 217a400c7f8f3..09adb7854365a 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -642,7 +642,47 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc args: &[OperandRef<'tcx, Self::Value>], is_cleanup: bool, ) -> Self::Value { - let fn_ptr = self.get_fn_addr(instance); + let func = if let Some(&func) = self.intrinsic_instances.borrow().get(&instance) { + func + } else { + let sym = self.tcx.symbol_name(instance).name; + + let func = if let Some(func) = self.intrinsics.borrow().get(sym) { + *func + } else { + self.linkage.set(FunctionType::Extern); + let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); + let fn_ty = fn_abi.gcc_type(self); + + let func = match sym { + "llvm.fma.f16" => { + // fma is not a target builtin, but a normal builtin, so we handle it differently + // here. + self.context.get_builtin_function("fma") + } + _ => llvm::intrinsic(sym, self), + }; + + self.intrinsics.borrow_mut().insert(sym.to_string(), func); + + self.on_stack_function_params + .borrow_mut() + .insert(func, fn_ty.on_stack_param_indices); + #[cfg(feature = "master")] + for fn_attr in fn_ty.fn_attributes { + func.add_attribute(fn_attr); + } + + crate::attributes::from_fn_attrs(self, func, instance); + + func + }; + + self.intrinsic_instances.borrow_mut().insert(instance, func); + + func + }; + let fn_ptr = func.get_address(None); let fn_ty = fn_ptr.get_type(); let mut llargs = vec![]; diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index b60c8a7d37193..c3f068a767188 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -100,6 +100,8 @@ pub(crate) struct FullCx<'ll, 'tcx> { /// Cache instances of monomorphic and polymorphic items pub instances: RefCell, &'ll Value>>, + /// Cache instances of intrinsics + pub intrinsic_instances: RefCell, &'ll Value>>, /// Cache generated vtables pub vtables: RefCell, Option>), &'ll Value>>, /// Cache of constant strings, @@ -620,6 +622,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { tls_model, codegen_unit, instances: Default::default(), + intrinsic_instances: Default::default(), vtables: Default::default(), const_str_cache: Default::default(), const_globals: Default::default(), diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index f8f4d515501e9..8cf0b26b9b4ef 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -29,6 +29,7 @@ use crate::abi::FnAbiLlvmExt; use crate::builder::Builder; use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call}; use crate::context::CodegenCx; +use crate::declare::declare_raw_fn; use crate::errors::{AutoDiffWithoutEnable, AutoDiffWithoutLto}; use crate::llvm::{self, Metadata, Type, Value}; use crate::type_of::LayoutLlvmExt; @@ -618,11 +619,41 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { args: &[OperandRef<'tcx, Self::Value>], is_cleanup: bool, ) -> Self::Value { - let fn_ptr = self.get_fn_addr(instance); + let tcx = self.tcx(); + // FIXME remove usage of fn_abi let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); assert!(!fn_abi.ret.is_indirect()); - let fn_ty = self.fn_decl_backend_type(fn_abi); + let fn_ty = fn_abi.llvm_type(self); + + let fn_ptr = if let Some(&llfn) = self.intrinsic_instances.borrow().get(&instance) { + llfn + } else { + let sym = tcx.symbol_name(instance).name; + + // FIXME use get_intrinsic + let llfn = if let Some(llfn) = self.get_declared_value(sym) { + llfn + } else { + // Function addresses in Rust are never significant, allowing functions to + // be merged. + let llfn = declare_raw_fn( + self, + sym, + fn_abi.llvm_cconv(self), + llvm::UnnamedAddr::Global, + llvm::Visibility::Default, + fn_ty, + ); + fn_abi.apply_attrs_llfn(self, llfn, Some(instance)); + + llfn + }; + + self.intrinsic_instances.borrow_mut().insert(instance, llfn); + + llfn + }; let mut llargs = vec![]; diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 560880a76e51a..77d55766d837e 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -9,7 +9,7 @@ use rustc_lint_defs::builtin::TAIL_CALL_TRACK_CALLER; use rustc_middle::mir::{self, AssertKind, InlineAsmMacro, SwitchTargets, UnwindTerminateReason}; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, ValidityRequirement}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; -use rustc_middle::ty::{self, Instance, Ty}; +use rustc_middle::ty::{self, Instance, Ty, TypeVisitableExt}; use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; use rustc_span::Span; @@ -1040,6 +1040,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { && let Some(name) = bx.tcx().codegen_fn_attrs(instance.def_id()).symbol_name && name.as_str().starts_with("llvm.") { + assert!(!instance.args.has_infer()); + assert!(!instance.args.has_escaping_bound_vars()); + let result_layout = self.cx.layout_of(self.monomorphized_place_ty(destination.as_ref()));