Skip to content

Commit 6684561

Browse files
committed
Codegen **non-overloaded** LLVM intrinsics using their name
1 parent ff5be13 commit 6684561

File tree

13 files changed

+216
-46
lines changed

13 files changed

+216
-46
lines changed

compiler/rustc_codegen_gcc/src/type_of.rs

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use std::fmt::Write;
22

3-
use gccjit::{Struct, Type};
3+
use gccjit::{RValue, Struct, Type};
44
use rustc_abi as abi;
55
use rustc_abi::Primitive::*;
66
use rustc_abi::{
@@ -373,7 +373,11 @@ impl<'gcc, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
373373
unimplemented!();
374374
}
375375

376-
fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Type<'gcc> {
376+
fn fn_decl_backend_type(
377+
&self,
378+
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
379+
_fn_ptr: RValue<'gcc>,
380+
) -> Type<'gcc> {
377381
// FIXME(antoyo): Should we do something with `FnAbiGcc::fn_attributes`?
378382
let FnAbiGcc { return_type, arguments_type, is_c_variadic, .. } = fn_abi.gcc_type(self);
379383
self.context.new_function_pointer_type(None, return_type, &arguments_type, is_c_variadic)

compiler/rustc_codegen_llvm/src/abi.rs

Lines changed: 137 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
use std::cmp;
1+
use std::borrow::Borrow;
2+
use std::{cmp, iter};
23

34
use libc::c_uint;
45
use rustc_abi::{
@@ -303,8 +304,37 @@ impl<'ll, 'tcx> ArgAbiBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
303304
}
304305
}
305306

307+
pub(crate) enum FunctionSignature<'ll> {
308+
/// The signature is obtained directly from LLVM, and **may not match the Rust signature**
309+
Intrinsic(&'ll Type),
310+
/// The name starts with `llvm.`, but can't obtain the intrinsic ID. May be invalid or upgradable
311+
MaybeInvalidIntrinsic(&'ll Type),
312+
/// Just the Rust signature
313+
Rust(&'ll Type),
314+
}
315+
316+
impl<'ll> FunctionSignature<'ll> {
317+
pub(crate) fn fn_ty(&self) -> &'ll Type {
318+
match self {
319+
FunctionSignature::Intrinsic(fn_ty)
320+
| FunctionSignature::MaybeInvalidIntrinsic(fn_ty)
321+
| FunctionSignature::Rust(fn_ty) => fn_ty,
322+
}
323+
}
324+
}
325+
306326
pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> {
307-
fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
327+
fn llvm_return_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
328+
fn llvm_argument_types(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<&'ll Type>;
329+
fn llvm_type(
330+
&self,
331+
cx: &CodegenCx<'ll, 'tcx>,
332+
name: &[u8],
333+
do_verify: bool,
334+
) -> FunctionSignature<'ll>;
335+
/// **If this function is an LLVM intrinsic** checks if the LLVM signature provided matches with this
336+
fn verify_intrinsic_signature(&self, cx: &CodegenCx<'ll, 'tcx>, llvm_ty: &'ll Type) -> bool;
337+
308338
fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
309339
fn llvm_cconv(&self, cx: &CodegenCx<'ll, 'tcx>) -> llvm::CallConv;
310340

@@ -317,30 +347,39 @@ pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> {
317347
);
318348

319349
/// Apply attributes to a function call.
320-
fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value);
350+
fn apply_attrs_callsite(
351+
&self,
352+
bx: &mut Builder<'_, 'll, 'tcx>,
353+
callsite: &'ll Value,
354+
llfn: &'ll Value,
355+
);
321356
}
322357

323358
impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
324-
fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
359+
fn llvm_return_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
360+
match &self.ret.mode {
361+
PassMode::Ignore => cx.type_void(),
362+
PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx),
363+
PassMode::Cast { cast, pad_i32: _ } => cast.llvm_type(cx),
364+
PassMode::Indirect { .. } => cx.type_void(),
365+
}
366+
}
367+
368+
fn llvm_argument_types(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<&'ll Type> {
369+
let indirect_return = matches!(self.ret.mode, PassMode::Indirect { .. });
370+
325371
// Ignore "extra" args from the call site for C variadic functions.
326372
// Only the "fixed" args are part of the LLVM function signature.
327373
let args =
328374
if self.c_variadic { &self.args[..self.fixed_count as usize] } else { &self.args };
329375

330376
// This capacity calculation is approximate.
331-
let mut llargument_tys = Vec::with_capacity(
332-
self.args.len() + if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 },
333-
);
377+
let mut llargument_tys =
378+
Vec::with_capacity(args.len() + if indirect_return { 1 } else { 0 });
334379

335-
let llreturn_ty = match &self.ret.mode {
336-
PassMode::Ignore => cx.type_void(),
337-
PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx),
338-
PassMode::Cast { cast, pad_i32: _ } => cast.llvm_type(cx),
339-
PassMode::Indirect { .. } => {
340-
llargument_tys.push(cx.type_ptr());
341-
cx.type_void()
342-
}
343-
};
380+
if indirect_return {
381+
llargument_tys.push(cx.type_ptr());
382+
}
344383

345384
for arg in args {
346385
// Note that the exact number of arguments pushed here is carefully synchronized with
@@ -387,10 +426,72 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
387426
llargument_tys.push(llarg_ty);
388427
}
389428

390-
if self.c_variadic {
391-
cx.type_variadic_func(&llargument_tys, llreturn_ty)
429+
llargument_tys
430+
}
431+
432+
fn verify_intrinsic_signature(&self, cx: &CodegenCx<'ll, 'tcx>, llvm_fn_ty: &'ll Type) -> bool {
433+
let rust_return_ty = self.llvm_return_type(cx);
434+
let rust_argument_tys = self.llvm_argument_types(cx);
435+
436+
let llvm_return_ty = cx.get_return_type(llvm_fn_ty);
437+
let llvm_argument_tys = cx.func_params_types(llvm_fn_ty);
438+
let llvm_is_variadic = cx.func_is_variadic(llvm_fn_ty);
439+
440+
if self.c_variadic != llvm_is_variadic || rust_argument_tys.len() != llvm_argument_tys.len()
441+
{
442+
return false;
443+
}
444+
445+
iter::once((rust_return_ty, llvm_return_ty))
446+
.chain(iter::zip(rust_argument_tys, llvm_argument_tys))
447+
.all(|(rust_ty, llvm_ty)| rust_ty == llvm_ty)
448+
}
449+
450+
fn llvm_type(
451+
&self,
452+
cx: &CodegenCx<'ll, 'tcx>,
453+
name: &[u8],
454+
do_verify: bool,
455+
) -> FunctionSignature<'ll> {
456+
let mut maybe_invalid = false;
457+
458+
if name.starts_with(b"llvm.") {
459+
if let Some(intrinsic) = llvm::Intrinsic::lookup(name) {
460+
if !intrinsic.is_overloaded() {
461+
// FIXME: also do this for overloaded intrinsics
462+
let llvm_fn_ty = intrinsic.get_type(cx.llcx, &[]);
463+
if do_verify {
464+
if !self.verify_intrinsic_signature(cx, llvm_fn_ty) {
465+
cx.tcx.dcx().fatal(format!(
466+
"Intrinsic signature mismatch for `{}`: expected signature `{llvm_fn_ty:?}`",
467+
str::from_utf8(name).unwrap()
468+
));
469+
}
470+
}
471+
return FunctionSignature::Intrinsic(llvm_fn_ty);
472+
}
473+
} else {
474+
// it's one of 2 cases,
475+
// - either the base name is invalid
476+
// - it has been superseded by something else, so the intrinsic was removed entirely
477+
// to check for upgrades, we need the `llfn`, so we defer it for now
478+
maybe_invalid = true;
479+
}
480+
}
481+
482+
let return_ty = self.llvm_return_type(cx);
483+
let argument_tys = self.llvm_argument_types(cx);
484+
485+
let fn_ty = if self.c_variadic {
486+
cx.type_variadic_func(&argument_tys, return_ty)
392487
} else {
393-
cx.type_func(&llargument_tys, llreturn_ty)
488+
cx.type_func(&argument_tys, return_ty)
489+
};
490+
491+
if maybe_invalid {
492+
FunctionSignature::MaybeInvalidIntrinsic(fn_ty)
493+
} else {
494+
FunctionSignature::Rust(fn_ty)
394495
}
395496
}
396497

@@ -548,7 +649,23 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
548649
}
549650
}
550651

551-
fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value) {
652+
fn apply_attrs_callsite(
653+
&self,
654+
bx: &mut Builder<'_, 'll, 'tcx>,
655+
callsite: &'ll Value,
656+
llfn: &'ll Value,
657+
) {
658+
// if we are using the LLVM signature, use the LLVM attributes otherwise it might be problematic
659+
let name = llvm::get_value_name(llfn);
660+
if name.starts_with(b"llvm.")
661+
&& let Some(intrinsic) = llvm::Intrinsic::lookup(&name)
662+
{
663+
// FIXME: also do this for overloaded intrinsics
664+
if !intrinsic.is_overloaded() {
665+
return;
666+
}
667+
}
668+
552669
let mut func_attrs = SmallVec::<[_; 2]>::new();
553670
if self.ret.layout.is_uninhabited() {
554671
func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(bx.cx.llcx));

compiler/rustc_codegen_llvm/src/builder.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -445,7 +445,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
445445
)
446446
};
447447
if let Some(fn_abi) = fn_abi {
448-
fn_abi.apply_attrs_callsite(self, invoke);
448+
fn_abi.apply_attrs_callsite(self, invoke, llfn);
449449
}
450450
invoke
451451
}
@@ -1447,7 +1447,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
14471447
}
14481448

14491449
if let Some(fn_abi) = fn_abi {
1450-
fn_abi.apply_attrs_callsite(self, call);
1450+
fn_abi.apply_attrs_callsite(self, call, llfn);
14511451
}
14521452
call
14531453
}
@@ -1787,7 +1787,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
17871787
)
17881788
};
17891789
if let Some(fn_abi) = fn_abi {
1790-
fn_abi.apply_attrs_callsite(self, callbr);
1790+
fn_abi.apply_attrs_callsite(self, callbr, llfn);
17911791
}
17921792
callbr
17931793
}

compiler/rustc_codegen_llvm/src/declare.rs

Lines changed: 29 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ use rustc_target::callconv::FnAbi;
2222
use smallvec::SmallVec;
2323
use tracing::debug;
2424

25-
use crate::abi::FnAbiLlvmExt;
25+
use crate::abi::{FnAbiLlvmExt, FunctionSignature};
2626
use crate::common::AsCCharPtr;
2727
use crate::context::{CodegenCx, GenericCx, SCx, SimpleCx};
2828
use crate::llvm::AttributePlace::Function;
@@ -150,17 +150,34 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
150150
) -> &'ll Value {
151151
debug!("declare_rust_fn(name={:?}, fn_abi={:?})", name, fn_abi);
152152

153-
// Function addresses in Rust are never significant, allowing functions to
154-
// be merged.
155-
let llfn = declare_raw_fn(
156-
self,
157-
name,
158-
fn_abi.llvm_cconv(self),
159-
llvm::UnnamedAddr::Global,
160-
llvm::Visibility::Default,
161-
fn_abi.llvm_type(self),
162-
);
163-
fn_abi.apply_attrs_llfn(self, llfn, instance);
153+
let signature = fn_abi.llvm_type(self, name.as_bytes(), true);
154+
let llfn;
155+
156+
if let FunctionSignature::Intrinsic(fn_ty) = signature {
157+
// intrinsics have a specified set of attributes, so we don't use the `FnAbi` set for them
158+
llfn = declare_simple_fn(
159+
self,
160+
name,
161+
fn_abi.llvm_cconv(self),
162+
llvm::UnnamedAddr::Global,
163+
llvm::Visibility::Default,
164+
fn_ty,
165+
);
166+
} else {
167+
// Function addresses in Rust are never significant, allowing functions to
168+
// be merged.
169+
llfn = declare_raw_fn(
170+
self,
171+
name,
172+
fn_abi.llvm_cconv(self),
173+
llvm::UnnamedAddr::Global,
174+
llvm::Visibility::Default,
175+
signature.fn_ty(),
176+
);
177+
fn_abi.apply_attrs_llfn(self, llfn, instance);
178+
}
179+
180+
// todo: check for upgrades, and emit error if not upgradable
164181

165182
if self.tcx.sess.is_sanitizer_cfi_enabled() {
166183
if let Some(instance) = instance {

compiler/rustc_codegen_llvm/src/intrinsic.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1075,7 +1075,7 @@ fn gen_fn<'a, 'll, 'tcx>(
10751075
codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
10761076
) -> (&'ll Type, &'ll Value) {
10771077
let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
1078-
let llty = fn_abi.llvm_type(cx);
1078+
let llty = fn_abi.llvm_type(cx, name.as_bytes(), true).fn_ty();
10791079
let llfn = cx.declare_fn(name, fn_abi, None);
10801080
cx.set_frame_pointer_type(llfn);
10811081
cx.apply_target_cpu_attr(llfn);

compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,6 @@ unsafe extern "C" {
7373
pub(crate) fn LLVMDumpModule(M: &Module);
7474
pub(crate) fn LLVMDumpValue(V: &Value);
7575
pub(crate) fn LLVMGetFunctionCallConv(F: &Value) -> c_uint;
76-
pub(crate) fn LLVMGetReturnType(T: &Type) -> &Type;
7776
pub(crate) fn LLVMGetParams(Fnc: &Value, params: *mut &Value);
7877
pub(crate) fn LLVMGetNamedFunction(M: &Module, Name: *const c_char) -> Option<&Value>;
7978
}

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -973,6 +973,8 @@ unsafe extern "C" {
973973
) -> &'a Type;
974974
pub(crate) fn LLVMCountParamTypes(FunctionTy: &Type) -> c_uint;
975975
pub(crate) fn LLVMGetParamTypes<'a>(FunctionTy: &'a Type, Dest: *mut &'a Type);
976+
pub(crate) fn LLVMGetReturnType(FunctionTy: &Type) -> &Type;
977+
pub(crate) fn LLVMIsFunctionVarArg(FunctionTy: &Type) -> Bool;
976978

977979
// Operations on struct types
978980
pub(crate) fn LLVMStructTypeInContext<'a>(
@@ -1120,6 +1122,13 @@ unsafe extern "C" {
11201122

11211123
// Operations about llvm intrinsics
11221124
pub(crate) fn LLVMLookupIntrinsicID(Name: *const c_char, NameLen: size_t) -> c_uint;
1125+
pub(crate) fn LLVMIntrinsicIsOverloaded(ID: NonZero<c_uint>) -> Bool;
1126+
pub(crate) fn LLVMIntrinsicGetType<'a>(
1127+
C: &'a Context,
1128+
ID: NonZero<c_uint>,
1129+
ParamTypes: *const &'a Type,
1130+
ParamCount: size_t,
1131+
) -> &'a Type;
11231132
pub(crate) fn LLVMGetIntrinsicDeclaration<'a>(
11241133
Mod: &'a Module,
11251134
ID: NonZero<c_uint>,

compiler/rustc_codegen_llvm/src/llvm/mod.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,14 @@ impl Intrinsic {
311311
NonZero::new(id).map(|id| Self { id })
312312
}
313313

314+
pub(crate) fn is_overloaded(self) -> bool {
315+
unsafe { LLVMIntrinsicIsOverloaded(self.id).is_true() }
316+
}
317+
318+
pub(crate) fn get_type<'ll>(self, llcx: &'ll Context, type_params: &[&'ll Type]) -> &'ll Type {
319+
unsafe { LLVMIntrinsicGetType(llcx, self.id, type_params.as_ptr(), type_params.len()) }
320+
}
321+
314322
pub(crate) fn get_declaration<'ll>(
315323
self,
316324
llmod: &'ll Module,

compiler/rustc_codegen_llvm/src/type_.rs

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,10 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
7070
unsafe { llvm::LLVMVectorType(ty, len as c_uint) }
7171
}
7272

73+
pub(crate) fn get_return_type(&self, ty: &'ll Type) -> &'ll Type {
74+
unsafe { llvm::LLVMGetReturnType(ty) }
75+
}
76+
7377
pub(crate) fn func_params_types(&self, ty: &'ll Type) -> Vec<&'ll Type> {
7478
unsafe {
7579
let n_args = llvm::LLVMCountParamTypes(ty) as usize;
@@ -79,6 +83,10 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
7983
args
8084
}
8185
}
86+
87+
pub(crate) fn func_is_variadic(&self, ty: &'ll Type) -> bool {
88+
unsafe { llvm::LLVMIsFunctionVarArg(ty).is_true() }
89+
}
8290
}
8391
impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
8492
pub(crate) fn type_bool(&self) -> &'ll Type {
@@ -288,8 +296,12 @@ impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
288296
fn cast_backend_type(&self, ty: &CastTarget) -> &'ll Type {
289297
ty.llvm_type(self)
290298
}
291-
fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type {
292-
fn_abi.llvm_type(self)
299+
fn fn_decl_backend_type(
300+
&self,
301+
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
302+
fn_ptr: &'ll Value,
303+
) -> &'ll Type {
304+
fn_abi.llvm_type(self, &llvm::get_value_name(fn_ptr), false).fn_ty()
293305
}
294306
fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type {
295307
fn_abi.ptr_to_llvm_type(self)

0 commit comments

Comments
 (0)