Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions compiler/rustc_codegen_llvm/src/context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ use smallvec::SmallVec;
use crate::back::write::to_llvm_code_model;
use crate::callee::get_fn;
use crate::debuginfo::metadata::apply_vcall_visibility_metadata;
use crate::llvm::Metadata;
use crate::llvm::{Metadata, MetadataKindId};
use crate::type_::Type;
use crate::value::Value;
use crate::{attributes, common, coverageinfo, debuginfo, llvm, llvm_util};
Expand Down Expand Up @@ -1006,11 +1006,11 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
pub(crate) fn set_metadata<'a>(
&self,
val: &'a Value,
kind_id: impl Into<llvm::MetadataKindId>,
kind_id: MetadataKindId,
md: &'ll Metadata,
) {
let node = self.get_metadata_value(md);
llvm::LLVMSetMetadata(val, kind_id.into(), node);
llvm::LLVMSetMetadata(val, kind_id, node);
}
}

Expand Down
8 changes: 2 additions & 6 deletions compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1611,16 +1611,12 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>(
let v = [llvm::LLVMValueAsMetadata(cx.const_usize(0)), typeid];
llvm::LLVMRustGlobalAddMetadata(
vtable,
llvm::MD_type as c_uint,
llvm::MD_type,
llvm::LLVMMDNodeInContext2(cx.llcx, v.as_ptr(), v.len()),
);
let vcall_visibility = llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64));
let vcall_visibility_metadata = llvm::LLVMMDNodeInContext2(cx.llcx, &vcall_visibility, 1);
llvm::LLVMGlobalSetMetadata(
vtable,
llvm::MetadataType::MD_vcall_visibility as c_uint,
vcall_visibility_metadata,
);
llvm::LLVMGlobalSetMetadata(vtable, llvm::MD_vcall_visibility, vcall_visibility_metadata);
}
}

Expand Down
44 changes: 7 additions & 37 deletions compiler/rustc_codegen_llvm/src/llvm/ffi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ use super::debuginfo::{
DITemplateTypeParameter, DIType, DebugEmissionKind, DebugNameTableKind,
};
use crate::llvm;
use crate::llvm::MetadataKindId;

/// In the LLVM-C API, boolean values are passed as `typedef int LLVMBool`,
/// which has a different ABI from Rust or C++ `bool`.
Expand Down Expand Up @@ -513,31 +514,6 @@ pub(crate) enum FileType {
ObjectFile,
}

/// LLVMMetadataType
#[derive(Copy, Clone)]
#[repr(C)]
#[expect(dead_code, reason = "Some variants are unused, but are kept to match LLVM-C")]
pub(crate) enum MetadataType {
MD_dbg = 0,
MD_tbaa = 1,
MD_prof = 2,
MD_fpmath = 3,
MD_range = 4,
MD_tbaa_struct = 5,
MD_invariant_load = 6,
MD_alias_scope = 7,
MD_noalias = 8,
MD_nontemporal = 9,
MD_mem_parallel_loop_access = 10,
MD_nonnull = 11,
MD_unpredictable = 15,
MD_align = 17,
MD_type = 19,
MD_vcall_visibility = 28,
MD_noundef = 29,
MD_kcfi_type = 36,
}

/// Must match the layout of `LLVMInlineAsmDialect`.
#[derive(Copy, Clone, PartialEq)]
#[repr(C)]
Expand Down Expand Up @@ -1035,16 +1011,6 @@ pub(crate) type GetSymbolsCallback =
unsafe extern "C" fn(*mut c_void, *const c_char) -> *mut c_void;
pub(crate) type GetSymbolsErrorCallback = unsafe extern "C" fn(*const c_char) -> *mut c_void;

#[derive(Copy, Clone)]
#[repr(transparent)]
pub(crate) struct MetadataKindId(c_uint);

impl From<MetadataType> for MetadataKindId {
fn from(value: MetadataType) -> Self {
Self(value as c_uint)
}
}

unsafe extern "C" {
// Create and destroy contexts.
pub(crate) fn LLVMContextDispose(C: &'static mut Context);
Expand Down Expand Up @@ -1139,7 +1105,11 @@ unsafe extern "C" {
pub(crate) fn LLVMSetValueName2(Val: &Value, Name: *const c_char, NameLen: size_t);
pub(crate) fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value);
pub(crate) safe fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: MetadataKindId, Node: &'a Value);
pub(crate) fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata);
pub(crate) fn LLVMGlobalSetMetadata<'a>(
Val: &'a Value,
KindID: MetadataKindId,
Metadata: &'a Metadata,
);
pub(crate) safe fn LLVMValueAsMetadata(Node: &Value) -> &Metadata;

// Operations on constants of any type
Expand Down Expand Up @@ -2059,7 +2029,7 @@ unsafe extern "C" {
// Operations on all values
pub(crate) fn LLVMRustGlobalAddMetadata<'a>(
Val: &'a Value,
KindID: c_uint,
KindID: MetadataKindId,
Metadata: &'a Metadata,
);
pub(crate) fn LLVMRustIsNonGVFunctionPointerTy(Val: &Value) -> bool;
Expand Down
71 changes: 71 additions & 0 deletions compiler/rustc_codegen_llvm/src/llvm/metadata_kind.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
use libc::c_uint;

pub(crate) use self::fixed_kinds::*;

#[derive(Copy, Clone)]
#[repr(transparent)]
pub(crate) struct MetadataKindId(c_uint);

macro_rules! declare_fixed_metadata_kinds {
(
$(
FIXED_MD_KIND($variant:ident, $value:literal)
)*
) => {
// Use a submodule to group all declarations into one `#[expect(..)]`.
#[expect(dead_code)]
mod fixed_kinds {
use super::MetadataKindId;
$(
#[expect(non_upper_case_globals)]
pub(crate) const $variant: MetadataKindId = MetadataKindId($value);
)*
}
};
}

// Must be kept in sync with the corresponding static assertions in `RustWrapper.cpp`.
declare_fixed_metadata_kinds! {
FIXED_MD_KIND(MD_dbg, 0)
FIXED_MD_KIND(MD_tbaa, 1)
FIXED_MD_KIND(MD_prof, 2)
FIXED_MD_KIND(MD_fpmath, 3)
FIXED_MD_KIND(MD_range, 4)
FIXED_MD_KIND(MD_tbaa_struct, 5)
FIXED_MD_KIND(MD_invariant_load, 6)
FIXED_MD_KIND(MD_alias_scope, 7)
FIXED_MD_KIND(MD_noalias, 8)
FIXED_MD_KIND(MD_nontemporal, 9)
FIXED_MD_KIND(MD_mem_parallel_loop_access, 10)
FIXED_MD_KIND(MD_nonnull, 11)
FIXED_MD_KIND(MD_dereferenceable, 12)
FIXED_MD_KIND(MD_dereferenceable_or_null, 13)
FIXED_MD_KIND(MD_make_implicit, 14)
FIXED_MD_KIND(MD_unpredictable, 15)
FIXED_MD_KIND(MD_invariant_group, 16)
FIXED_MD_KIND(MD_align, 17)
FIXED_MD_KIND(MD_loop, 18)
FIXED_MD_KIND(MD_type, 19)
FIXED_MD_KIND(MD_section_prefix, 20)
FIXED_MD_KIND(MD_absolute_symbol, 21)
FIXED_MD_KIND(MD_associated, 22)
FIXED_MD_KIND(MD_callees, 23)
FIXED_MD_KIND(MD_irr_loop, 24)
FIXED_MD_KIND(MD_access_group, 25)
FIXED_MD_KIND(MD_callback, 26)
FIXED_MD_KIND(MD_preserve_access_index, 27)
FIXED_MD_KIND(MD_vcall_visibility, 28)
FIXED_MD_KIND(MD_noundef, 29)
FIXED_MD_KIND(MD_annotation, 30)
FIXED_MD_KIND(MD_nosanitize, 31)
FIXED_MD_KIND(MD_func_sanitize, 32)
FIXED_MD_KIND(MD_exclude, 33)
FIXED_MD_KIND(MD_memprof, 34)
FIXED_MD_KIND(MD_callsite, 35)
FIXED_MD_KIND(MD_kcfi_type, 36)
FIXED_MD_KIND(MD_pcsections, 37)
FIXED_MD_KIND(MD_DIAssignID, 38)
FIXED_MD_KIND(MD_coro_outside_frame, 39)
FIXED_MD_KIND(MD_mmra, 40)
FIXED_MD_KIND(MD_noalias_addrspace, 41)
}
3 changes: 2 additions & 1 deletion compiler/rustc_codegen_llvm/src/llvm/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,14 @@ use rustc_llvm::RustString;

pub(crate) use self::CallConv::*;
pub(crate) use self::CodeGenOptSize::*;
pub(crate) use self::MetadataType::*;
pub(crate) use self::ffi::*;
pub(crate) use self::metadata_kind::*;
use crate::common::AsCCharPtr;

pub(crate) mod diagnostic;
pub(crate) mod enzyme_ffi;
mod ffi;
mod metadata_kind;

pub(crate) use self::enzyme_ffi::*;

Expand Down
8 changes: 4 additions & 4 deletions compiler/rustc_codegen_llvm/src/type_.rs
Original file line number Diff line number Diff line change
Expand Up @@ -306,7 +306,7 @@ impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata];
llvm::LLVMRustGlobalAddMetadata(
function,
llvm::MD_type as c_uint,
llvm::MD_type,
llvm::LLVMMDNodeInContext2(self.llcx, v.as_ptr(), v.len()),
)
}
Expand All @@ -318,7 +318,7 @@ impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata];
llvm::LLVMGlobalSetMetadata(
function,
llvm::MD_type as c_uint,
llvm::MD_type,
llvm::LLVMMDNodeInContext2(self.llcx, v.as_ptr(), v.len()),
)
}
Expand All @@ -333,7 +333,7 @@ impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
unsafe {
llvm::LLVMRustGlobalAddMetadata(
function,
llvm::MD_kcfi_type as c_uint,
llvm::MD_kcfi_type,
llvm::LLVMMDNodeInContext2(
self.llcx,
&llvm::LLVMValueAsMetadata(kcfi_type_metadata),
Expand All @@ -348,7 +348,7 @@ impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
unsafe {
llvm::LLVMGlobalSetMetadata(
function,
llvm::MD_kcfi_type as c_uint,
llvm::MD_kcfi_type,
llvm::LLVMMDNodeInContext2(
self.llcx,
&llvm::LLVMValueAsMetadata(kcfi_type_metadata),
Expand Down
52 changes: 52 additions & 0 deletions compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1824,3 +1824,55 @@ extern "C" size_t LLVMRustEnzymeGetMaxTypeDepth() {
return 6; // Default fallback depth
}
#endif

// Statically assert that the fixed metadata kind IDs declared in
// `metadata_kind.rs` match the ones actually used by LLVM.
#define FIXED_MD_KIND(VARIANT, VALUE) \
static_assert(::llvm::LLVMContext::VARIANT == VALUE);
// Must be kept in sync with the corresponding list in `metadata_kind.rs`.
FIXED_MD_KIND(MD_dbg, 0)
FIXED_MD_KIND(MD_tbaa, 1)
FIXED_MD_KIND(MD_prof, 2)
FIXED_MD_KIND(MD_fpmath, 3)
FIXED_MD_KIND(MD_range, 4)
FIXED_MD_KIND(MD_tbaa_struct, 5)
FIXED_MD_KIND(MD_invariant_load, 6)
FIXED_MD_KIND(MD_alias_scope, 7)
FIXED_MD_KIND(MD_noalias, 8)
FIXED_MD_KIND(MD_nontemporal, 9)
FIXED_MD_KIND(MD_mem_parallel_loop_access, 10)
FIXED_MD_KIND(MD_nonnull, 11)
FIXED_MD_KIND(MD_dereferenceable, 12)
FIXED_MD_KIND(MD_dereferenceable_or_null, 13)
FIXED_MD_KIND(MD_make_implicit, 14)
FIXED_MD_KIND(MD_unpredictable, 15)
FIXED_MD_KIND(MD_invariant_group, 16)
FIXED_MD_KIND(MD_align, 17)
FIXED_MD_KIND(MD_loop, 18)
FIXED_MD_KIND(MD_type, 19)
FIXED_MD_KIND(MD_section_prefix, 20)
FIXED_MD_KIND(MD_absolute_symbol, 21)
FIXED_MD_KIND(MD_associated, 22)
FIXED_MD_KIND(MD_callees, 23)
FIXED_MD_KIND(MD_irr_loop, 24)
FIXED_MD_KIND(MD_access_group, 25)
FIXED_MD_KIND(MD_callback, 26)
FIXED_MD_KIND(MD_preserve_access_index, 27)
FIXED_MD_KIND(MD_vcall_visibility, 28)
FIXED_MD_KIND(MD_noundef, 29)
FIXED_MD_KIND(MD_annotation, 30)
FIXED_MD_KIND(MD_nosanitize, 31)
FIXED_MD_KIND(MD_func_sanitize, 32)
FIXED_MD_KIND(MD_exclude, 33)
FIXED_MD_KIND(MD_memprof, 34)
FIXED_MD_KIND(MD_callsite, 35)
FIXED_MD_KIND(MD_kcfi_type, 36)
FIXED_MD_KIND(MD_pcsections, 37)
FIXED_MD_KIND(MD_DIAssignID, 38)
FIXED_MD_KIND(MD_coro_outside_frame, 39)
FIXED_MD_KIND(MD_mmra, 40)
FIXED_MD_KIND(MD_noalias_addrspace, 41)
// If some fixed metadata kinds are not present and consistent in all supported
// LLVM versions, it's fine to omit them from this list; in that case Rust-side
// code cannot declare them as fixed IDs and must look them up by name instead.
#undef FIXED_MD_KIND
4 changes: 3 additions & 1 deletion library/alloc/src/rc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,9 @@ use crate::vec::Vec;
// This is repr(C) to future-proof against possible field-reordering, which
// would interfere with otherwise safe [into|from]_raw() of transmutable
// inner types.
#[repr(C)]
// repr(align(2)) (forcing alignment to at least 2) is required because usize
// has 1-byte alignment on AVR.
#[repr(C, align(2))]
struct RcInner<T: ?Sized> {
strong: Cell<usize>,
weak: Cell<usize>,
Expand Down
16 changes: 9 additions & 7 deletions library/alloc/src/sync.rs
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,7 @@ pub struct Weak<
// but it is not necessarily a valid pointer.
// `Weak::new` sets this to `usize::MAX` so that it doesn’t need
// to allocate space on the heap. That's not a value a real pointer
// will ever have because RcInner has alignment at least 2.
// will ever have because ArcInner has alignment at least 2.
ptr: NonNull<ArcInner<T>>,
alloc: A,
}
Expand All @@ -366,7 +366,9 @@ impl<T: ?Sized, A: Allocator> fmt::Debug for Weak<T, A> {
// This is repr(C) to future-proof against possible field-reordering, which
// would interfere with otherwise safe [into|from]_raw() of transmutable
// inner types.
#[repr(C)]
// Unlike RcInner, repr(align(2)) is not strictly required because atomic types
// have the alignment same as its size, but we use it for consistency and clarity.
#[repr(C, align(2))]
struct ArcInner<T: ?Sized> {
strong: Atomic<usize>,

Expand Down Expand Up @@ -1613,9 +1615,9 @@ impl<T: ?Sized, A: Allocator> Arc<T, A> {
pub fn as_ptr(this: &Self) -> *const T {
let ptr: *mut ArcInner<T> = NonNull::as_ptr(this.ptr);

// SAFETY: This cannot go through Deref::deref or RcInnerPtr::inner because
// SAFETY: This cannot go through Deref::deref or ArcInnerPtr::inner because
// this is required to retain raw/mut provenance such that e.g. `get_mut` can
// write through the pointer after the Rc is recovered through `from_raw`.
// write through the pointer after the Arc is recovered through `from_raw`.
unsafe { &raw mut (*ptr).data }
}

Expand Down Expand Up @@ -2450,7 +2452,7 @@ impl<T: ?Sized, A: Allocator> Arc<T, A> {
/// If any other `Arc` or [`Weak`] pointers to the same allocation exist, then
/// they must not be dereferenced or have active borrows for the duration
/// of the returned borrow, and their inner type must be exactly the same as the
/// inner type of this Rc (including lifetimes). This is trivially the case if no
/// inner type of this Arc (including lifetimes). This is trivially the case if no
/// such pointers exist, for example immediately after `Arc::new`.
///
/// # Examples
Expand Down Expand Up @@ -3022,7 +3024,7 @@ impl<T: ?Sized, A: Allocator> Weak<T, A> {
// Otherwise, we're guaranteed the pointer came from a nondangling Weak.
// SAFETY: data_offset is safe to call, as ptr references a real (potentially dropped) T.
let offset = unsafe { data_offset(ptr) };
// Thus, we reverse the offset to get the whole RcInner.
// Thus, we reverse the offset to get the whole ArcInner.
// SAFETY: the pointer originated from a Weak, so this offset is safe.
unsafe { ptr.byte_sub(offset) as *mut ArcInner<T> }
};
Expand Down Expand Up @@ -4015,7 +4017,7 @@ impl<T: ?Sized, A: Allocator> Unpin for Arc<T, A> {}
/// valid instance of T, but the T is allowed to be dropped.
unsafe fn data_offset<T: ?Sized>(ptr: *const T) -> usize {
// Align the unsized value to the end of the ArcInner.
// Because RcInner is repr(C), it will always be the last field in memory.
// Because ArcInner is repr(C), it will always be the last field in memory.
// SAFETY: since the only unsized types possible are slices, trait objects,
// and extern types, the input safety requirement is currently enough to
// satisfy the requirements of align_of_val_raw; this is an implementation
Expand Down
Loading
Loading