diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 680ad98593e7e..e03c2868b0f1c 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -39,13 +39,11 @@ trait ArgAttributesExt { const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] = [(ArgAttribute::InReg, llvm::AttributeKind::InReg)]; -const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 6] = [ +const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 4] = [ (ArgAttribute::NoAlias, llvm::AttributeKind::NoAlias), - (ArgAttribute::CapturesAddress, llvm::AttributeKind::CapturesAddress), (ArgAttribute::NonNull, llvm::AttributeKind::NonNull), (ArgAttribute::ReadOnly, llvm::AttributeKind::ReadOnly), (ArgAttribute::NoUndef, llvm::AttributeKind::NoUndef), - (ArgAttribute::CapturesReadOnly, llvm::AttributeKind::CapturesReadOnly), ]; fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 8]> { @@ -81,15 +79,23 @@ fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&' } for (attr, llattr) in OPTIMIZATION_ATTRIBUTES { if regular.contains(attr) { - // captures(...) is only available since LLVM 21. - if (attr == ArgAttribute::CapturesReadOnly || attr == ArgAttribute::CapturesAddress) - && llvm_util::get_version() < (21, 0, 0) - { - continue; - } attrs.push(llattr.create_attr(cx.llcx)); } } + // captures(...) is only available since LLVM 21. + if (21, 0, 0) <= llvm_util::get_version() { + const CAPTURES_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 3] = [ + (ArgAttribute::CapturesNone, llvm::AttributeKind::CapturesNone), + (ArgAttribute::CapturesAddress, llvm::AttributeKind::CapturesAddress), + (ArgAttribute::CapturesReadOnly, llvm::AttributeKind::CapturesReadOnly), + ]; + for (attr, llattr) in CAPTURES_ATTRIBUTES { + if regular.contains(attr) { + attrs.push(llattr.create_attr(cx.llcx)); + break; + } + } + } } else if cx.tcx.sess.opts.unstable_opts.sanitizer.contains(SanitizerSet::MEMORY) { // If we're not optimising, *but* memory sanitizer is on, emit noundef, since it affects // memory sanitizer's behavior. diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 2456ed2e46d67..53f0f9ff9d01b 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -289,6 +289,7 @@ pub(crate) enum AttributeKind { DeadOnUnwind = 43, DeadOnReturn = 44, CapturesReadOnly = 45, + CapturesNone = 46, } /// LLVMIntPredicate diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 6b4f8a6dba79f..ad459986826a5 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -245,6 +245,7 @@ enum class LLVMRustAttributeKind { DeadOnUnwind = 43, DeadOnReturn = 44, CapturesReadOnly = 45, + CapturesNone = 46, }; static Attribute::AttrKind fromRust(LLVMRustAttributeKind Kind) { @@ -339,6 +340,7 @@ static Attribute::AttrKind fromRust(LLVMRustAttributeKind Kind) { #endif case LLVMRustAttributeKind::CapturesAddress: case LLVMRustAttributeKind::CapturesReadOnly: + case LLVMRustAttributeKind::CapturesNone: report_fatal_error("Should be handled separately"); } report_fatal_error("bad LLVMRustAttributeKind"); @@ -390,6 +392,9 @@ extern "C" void LLVMRustEraseInstFromParent(LLVMValueRef Instr) { extern "C" LLVMAttributeRef LLVMRustCreateAttrNoValue(LLVMContextRef C, LLVMRustAttributeKind RustAttr) { #if LLVM_VERSION_GE(21, 0) + if (RustAttr == LLVMRustAttributeKind::CapturesNone) { + return wrap(Attribute::getWithCaptureInfo(*unwrap(C), CaptureInfo::none())); + } if (RustAttr == LLVMRustAttributeKind::CapturesAddress) { return wrap(Attribute::getWithCaptureInfo( *unwrap(C), CaptureInfo(CaptureComponents::Address))); diff --git a/compiler/rustc_middle/src/middle/deduced_param_attrs.rs b/compiler/rustc_middle/src/middle/deduced_param_attrs.rs index 68d1c852f0e27..5e9e625f4c746 100644 --- a/compiler/rustc_middle/src/middle/deduced_param_attrs.rs +++ b/compiler/rustc_middle/src/middle/deduced_param_attrs.rs @@ -2,19 +2,21 @@ use rustc_macros::{Decodable, Encodable, HashStable}; use crate::ty::{Ty, TyCtxt, TypingEnv}; -/// Flags that dictate how a parameter is mutated. If the flags are empty, the param is -/// read-only. If non-empty, it is read-only if *all* flags' conditions are met. +/// Summarizes how a parameter (a return place or an argument) is used inside a MIR body. #[derive(Clone, Copy, PartialEq, Debug, Decodable, Encodable, HashStable)] -pub struct DeducedReadOnlyParam(u8); +pub struct UsageSummary(u8); bitflags::bitflags! { - impl DeducedReadOnlyParam: u8 { - /// This parameter is dropped. It is read-only if `!needs_drop`. - const IF_NO_DROP = 1 << 0; - /// This parameter is borrowed. It is read-only if `Freeze`. - const IF_FREEZE = 1 << 1; - /// This parameter is mutated. It is never read-only. - const MUTATED = 1 << 2; + impl UsageSummary: u8 { + /// This parameter is dropped when it `needs_drop`. + const DROP = 1 << 0; + /// There is a shared borrow to this parameter. + /// It allows for mutation unless parameter is `Freeze`. + const SHARED_BORROW = 1 << 1; + /// This parameter is mutated (excluding through a drop or a shared borrow). + const MUTATE = 1 << 2; + /// This parameter is captured (excluding through a drop). + const CAPTURE = 1 << 3; } } @@ -24,43 +26,53 @@ bitflags::bitflags! { /// These can be useful for optimization purposes when a function is directly called. We compute /// them and store them into the crate metadata so that downstream crates can make use of them. /// -/// Right now, we only have `read_only`, but `no_capture` and `no_alias` might be useful in the +/// Right now, we have `readonly` and `captures(none)`, but `no_alias` might be useful in the /// future. #[derive(Clone, Copy, PartialEq, Debug, Decodable, Encodable, HashStable)] pub struct DeducedParamAttrs { - /// The parameter is marked immutable in the function. - pub read_only: DeducedReadOnlyParam, -} - -// By default, consider the parameters to be mutated. -impl Default for DeducedParamAttrs { - #[inline] - fn default() -> DeducedParamAttrs { - DeducedParamAttrs { read_only: DeducedReadOnlyParam::MUTATED } - } + pub usage: UsageSummary, } impl DeducedParamAttrs { + /// Returns true if no attributes have been deduced. #[inline] pub fn is_default(self) -> bool { - self.read_only.contains(DeducedReadOnlyParam::MUTATED) + self.usage.contains(UsageSummary::MUTATE | UsageSummary::CAPTURE) } + /// For parameters passed indirectly, returns true if pointer is never written through. pub fn read_only<'tcx>( &self, tcx: TyCtxt<'tcx>, typing_env: TypingEnv<'tcx>, ty: Ty<'tcx>, ) -> bool { - let read_only = self.read_only; - // We have to check *all* set bits; only if all checks pass is this truly read-only. - if read_only.contains(DeducedReadOnlyParam::MUTATED) { + // Only if all checks pass is this truly read-only. + if self.usage.contains(UsageSummary::MUTATE) { + return false; + } + if self.usage.contains(UsageSummary::DROP) && ty.needs_drop(tcx, typing_env) { + return false; + } + if self.usage.contains(UsageSummary::SHARED_BORROW) && !ty.is_freeze(tcx, typing_env) { return false; } - if read_only.contains(DeducedReadOnlyParam::IF_NO_DROP) && ty.needs_drop(tcx, typing_env) { + true + } + + /// For parameters passed indirectly, returns true if pointer is not captured, i.e., its + /// address is not captured, and pointer is used neither for reads nor writes after function + /// returns. + pub fn captures_none<'tcx>( + &self, + tcx: TyCtxt<'tcx>, + typing_env: TypingEnv<'tcx>, + ty: Ty<'tcx>, + ) -> bool { + if self.usage.contains(UsageSummary::CAPTURE) { return false; } - if read_only.contains(DeducedReadOnlyParam::IF_FREEZE) && !ty.is_freeze(tcx, typing_env) { + if self.usage.contains(UsageSummary::DROP) && ty.needs_drop(tcx, typing_env) { return false; } true diff --git a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs index e421e47bd1615..21762c8dd68c9 100644 --- a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs +++ b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs @@ -11,65 +11,91 @@ use rustc_hir::def_id::LocalDefId; use rustc_index::IndexVec; -use rustc_middle::middle::deduced_param_attrs::{DeducedParamAttrs, DeducedReadOnlyParam}; +use rustc_middle::middle::deduced_param_attrs::{DeducedParamAttrs, UsageSummary}; use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_session::config::OptLevel; -/// A visitor that determines which arguments have been mutated. We can't use the mutability field -/// on LocalDecl for this because it has no meaning post-optimization. -struct DeduceReadOnly { - /// Each bit is indexed by argument number, starting at zero (so 0 corresponds to local decl - /// 1). The bit is false if the argument may have been mutated or true if we know it hasn't - /// been up to the point we're at. - read_only: IndexVec, +/// A visitor that determines how a return place and arguments are used inside MIR body. +/// To determine whether a local is mutated we can't use the mutability field on LocalDecl +/// because it has no meaning post-optimization. +struct DeduceParamAttrs { + /// Summarizes how a return place and arguments are used inside MIR body. + usage: IndexVec, } -impl DeduceReadOnly { - /// Returns a new DeduceReadOnly instance. - fn new(arg_count: usize) -> Self { - Self { read_only: IndexVec::from_elem_n(DeducedReadOnlyParam::empty(), arg_count) } +impl DeduceParamAttrs { + /// Returns a new DeduceParamAttrs instance. + fn new(body: &Body<'_>) -> Self { + let mut this = + Self { usage: IndexVec::from_elem_n(UsageSummary::empty(), body.arg_count + 1) }; + // Code generation indicates that a return place is writable. To avoid setting both + // `readonly` and `writable` attributes, when return place is never written to, mark it as + // mutated. + this.usage[RETURN_PLACE] |= UsageSummary::MUTATE; + this } - /// Returns whether the given local is a parameter and its index. - fn as_param(&self, local: Local) -> Option { - // Locals and parameters are shifted by `RETURN_PLACE`. - let param_index = local.as_usize().checked_sub(1)?; - if param_index < self.read_only.len() { Some(param_index) } else { None } + /// Returns whether a local is the return place or an argument and returns its index. + fn as_param(&self, local: Local) -> Option { + if local.index() < self.usage.len() { Some(local) } else { None } } } -impl<'tcx> Visitor<'tcx> for DeduceReadOnly { +impl<'tcx> Visitor<'tcx> for DeduceParamAttrs { fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, _location: Location) { - // We're only interested in arguments. - let Some(param_index) = self.as_param(place.local) else { return }; + // We're only interested in the return place or an argument. + let Some(i) = self.as_param(place.local) else { return }; match context { - // Not mutating, so it's fine. + // Not actually using the local. PlaceContext::NonUse(..) => {} - // Dereference is not a mutation. + // Neither mutated nor captured. _ if place.is_indirect_first_projection() => {} // This is a `Drop`. It could disappear at monomorphization, so mark it specially. PlaceContext::MutatingUse(MutatingUseContext::Drop) // Projection changes the place's type, so `needs_drop(local.ty)` is not // `needs_drop(place.ty)`. if place.projection.is_empty() => { - self.read_only[param_index] |= DeducedReadOnlyParam::IF_NO_DROP; + self.usage[i] |= UsageSummary::DROP; + } + PlaceContext::MutatingUse( + MutatingUseContext::Call + | MutatingUseContext::Yield + | MutatingUseContext::Drop + | MutatingUseContext::Borrow + | MutatingUseContext::RawBorrow) => { + self.usage[i] |= UsageSummary::MUTATE; + self.usage[i] |= UsageSummary::CAPTURE; + } + PlaceContext::MutatingUse( + MutatingUseContext::Store + | MutatingUseContext::SetDiscriminant + | MutatingUseContext::AsmOutput + | MutatingUseContext::Projection + | MutatingUseContext::Retag) => { + self.usage[i] |= UsageSummary::MUTATE; } - // This is a mutation, so mark it as such. - PlaceContext::MutatingUse(..) - // Whether mutating though a `&raw const` is allowed is still undecided, so we - // disable any sketchy `readonly` optimizations for now. | PlaceContext::NonMutatingUse(NonMutatingUseContext::RawBorrow) => { - self.read_only[param_index] |= DeducedReadOnlyParam::MUTATED; + // Whether mutating though a `&raw const` is allowed is still undecided, so we + // disable any sketchy `readonly` optimizations for now. + self.usage[i] |= UsageSummary::MUTATE; + self.usage[i] |= UsageSummary::CAPTURE; } - // Not mutating if the parameter is `Freeze`. PlaceContext::NonMutatingUse(NonMutatingUseContext::SharedBorrow) => { - self.read_only[param_index] |= DeducedReadOnlyParam::IF_FREEZE; + // Not mutating if the parameter is `Freeze`. + self.usage[i] |= UsageSummary::SHARED_BORROW; + self.usage[i] |= UsageSummary::CAPTURE; } // Not mutating, so it's fine. - PlaceContext::NonMutatingUse(..) => {} + PlaceContext::NonMutatingUse( + NonMutatingUseContext::Inspect + | NonMutatingUseContext::Copy + | NonMutatingUseContext::Move + | NonMutatingUseContext::FakeBorrow + | NonMutatingUseContext::PlaceMention + | NonMutatingUseContext::Projection) => {} } } @@ -98,11 +124,11 @@ impl<'tcx> Visitor<'tcx> for DeduceReadOnly { if let TerminatorKind::Call { ref args, .. } = terminator.kind { for arg in args { if let Operand::Move(place) = arg.node - // We're only interested in arguments. - && let Some(param_index) = self.as_param(place.local) && !place.is_indirect_first_projection() + && let Some(i) = self.as_param(place.local) { - self.read_only[param_index] |= DeducedReadOnlyParam::MUTATED; + self.usage[i] |= UsageSummary::MUTATE; + self.usage[i] |= UsageSummary::CAPTURE; } } }; @@ -154,10 +180,9 @@ pub(super) fn deduced_param_attrs<'tcx>( if matches!(fn_ty.kind(), ty::FnDef(..)) && fn_ty .fn_sig(tcx) - .inputs() + .inputs_and_output() .skip_binder() .iter() - .cloned() .all(type_will_always_be_passed_directly) { return &[]; @@ -170,13 +195,13 @@ pub(super) fn deduced_param_attrs<'tcx>( // Grab the optimized MIR. Analyze it to determine which arguments have been mutated. let body: &Body<'tcx> = tcx.optimized_mir(def_id); - let mut deduce_read_only = DeduceReadOnly::new(body.arg_count); - deduce_read_only.visit_body(body); - tracing::trace!(?deduce_read_only.read_only); + let mut deduce = DeduceParamAttrs::new(body); + deduce.visit_body(body); + tracing::trace!(?deduce.usage); - let mut deduced_param_attrs: &[_] = tcx.arena.alloc_from_iter( - deduce_read_only.read_only.into_iter().map(|read_only| DeducedParamAttrs { read_only }), - ); + let mut deduced_param_attrs: &[_] = tcx + .arena + .alloc_from_iter(deduce.usage.into_iter().map(|usage| DeducedParamAttrs { usage })); // Trailing parameters past the size of the `deduced_param_attrs` array are assumed to have the // default set of attributes, so we don't have to store them explicitly. Pop them off to save a diff --git a/compiler/rustc_target/src/callconv/mod.rs b/compiler/rustc_target/src/callconv/mod.rs index 3f7382ee0e2b5..a33c246c88c67 100644 --- a/compiler/rustc_target/src/callconv/mod.rs +++ b/compiler/rustc_target/src/callconv/mod.rs @@ -113,13 +113,14 @@ mod attr_impl { pub struct ArgAttribute(u8); bitflags::bitflags! { impl ArgAttribute: u8 { - const NoAlias = 1 << 1; - const CapturesAddress = 1 << 2; - const NonNull = 1 << 3; - const ReadOnly = 1 << 4; - const InReg = 1 << 5; - const NoUndef = 1 << 6; - const CapturesReadOnly = 1 << 7; + const CapturesNone = 0b111; + const CapturesAddress = 0b110; + const CapturesReadOnly = 0b100; + const NoAlias = 1 << 3; + const NonNull = 1 << 4; + const ReadOnly = 1 << 5; + const InReg = 1 << 6; + const NoUndef = 1 << 7; } } rustc_data_structures::external_bitflags_debug! { ArgAttribute } diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index ed5289c6850e8..0f09e548f0e2b 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -5,6 +5,7 @@ use rustc_abi::{BackendRepr, ExternAbi, PointerKind, Scalar, Size}; use rustc_hir as hir; use rustc_hir::lang_items::LangItem; use rustc_middle::bug; +use rustc_middle::middle::deduced_param_attrs::DeducedParamAttrs; use rustc_middle::query::Providers; use rustc_middle::ty::layout::{ FnAbiError, HasTyCtxt, HasTypingEnv, LayoutCx, LayoutOf, TyAndLayout, fn_can_unwind, @@ -614,37 +615,19 @@ fn fn_abi_adjust_for_abi<'tcx>( if abi.is_rustic_abi() { fn_abi.adjust_for_rust_abi(cx); - // Look up the deduced parameter attributes for this function, if we have its def ID and // we're optimizing in non-incremental mode. We'll tag its parameters with those attributes // as appropriate. - let deduced_param_attrs = + let deduced = if tcx.sess.opts.optimize != OptLevel::No && tcx.sess.opts.incremental.is_none() { fn_def_id.map(|fn_def_id| tcx.deduced_param_attrs(fn_def_id)).unwrap_or_default() } else { &[] }; - - for (arg_idx, arg) in fn_abi.args.iter_mut().enumerate() { - if arg.is_ignore() { - continue; - } - - // If we deduced that this parameter was read-only, add that to the attribute list now. - // - // The `readonly` parameter only applies to pointers, so we can only do this if the - // argument was passed indirectly. (If the argument is passed directly, it's an SSA - // value, so it's implicitly immutable.) - if let &mut PassMode::Indirect { ref mut attrs, .. } = &mut arg.mode { - // The `deduced_param_attrs` list could be empty if this is a type of function - // we can't deduce any parameters for, so make sure the argument index is in - // bounds. - if let Some(deduced_param_attrs) = deduced_param_attrs.get(arg_idx) - && deduced_param_attrs.read_only(tcx, cx.typing_env, arg.layout.ty) - { - debug!("added deduced read-only attribute"); - attrs.regular.insert(ArgAttribute::ReadOnly); - } + if !deduced.is_empty() { + apply_deduced_attributes(cx, deduced, 0, &mut fn_abi.ret); + for (arg_idx, arg) in fn_abi.args.iter_mut().enumerate() { + apply_deduced_attributes(cx, deduced, arg_idx + 1, arg); } } } else { @@ -652,6 +635,34 @@ fn fn_abi_adjust_for_abi<'tcx>( } } +/// Apply deduced optimization attributes to a parameter using an indirect pass mode. +/// +/// `deduced` is a possibly truncated list of deduced attributes for a return place and arguments. +/// `idx` the index of the parameter on the list (0 for a return place, and 1.. for arguments). +fn apply_deduced_attributes<'tcx>( + cx: &LayoutCx<'tcx>, + deduced: &[DeducedParamAttrs], + idx: usize, + arg: &mut ArgAbi<'tcx, Ty<'tcx>>, +) { + // Deduction is performed under the assumption of the indirection pass mode. + let PassMode::Indirect { ref mut attrs, .. } = arg.mode else { + return; + }; + // The default values at the tail of the list are not encoded. + let Some(deduced) = deduced.get(idx) else { + return; + }; + if deduced.read_only(cx.tcx(), cx.typing_env, arg.layout.ty) { + debug!("added deduced ReadOnly attribute"); + attrs.regular.insert(ArgAttribute::ReadOnly); + } + if deduced.captures_none(cx.tcx(), cx.typing_env, arg.layout.ty) { + debug!("added deduced CapturesNone attribute"); + attrs.regular.insert(ArgAttribute::CapturesNone); + } +} + #[tracing::instrument(level = "debug", skip(cx))] fn make_thin_self_ptr<'tcx>( cx: &(impl HasTyCtxt<'tcx> + HasTypingEnv<'tcx>), diff --git a/tests/codegen-llvm/addr-of-mutate.rs b/tests/codegen-llvm/addr-of-mutate.rs index 36d6bf555d156..d59d85af62a91 100644 --- a/tests/codegen-llvm/addr-of-mutate.rs +++ b/tests/codegen-llvm/addr-of-mutate.rs @@ -24,7 +24,7 @@ pub unsafe fn second(a_ptr_and_b: (*mut (i32, bool), (i64, bool))) -> bool { } // If going through a deref (and there are no other mutating accesses), then `readonly` is fine. -// CHECK: i1 @third(ptr{{( dead_on_return)?}} noalias noundef readonly align {{[0-9]+}}{{( captures\(address\))?}} dereferenceable({{[0-9]+}}) %a_ptr_and_b) +// CHECK: i1 @third(ptr{{( dead_on_return)?}} noalias noundef readonly align {{[0-9]+}}{{( captures\(none\))?}} dereferenceable({{[0-9]+}}) %a_ptr_and_b) #[no_mangle] pub unsafe fn third(a_ptr_and_b: (*mut (i32, bool), (i64, bool))) -> bool { let b_bool_ptr = core::ptr::addr_of!((*a_ptr_and_b.0).1).cast_mut(); diff --git a/tests/codegen-llvm/deduced-param-attrs.rs b/tests/codegen-llvm/deduced-param-attrs.rs index f99615cbe6d42..8062419ed5c6e 100644 --- a/tests/codegen-llvm/deduced-param-attrs.rs +++ b/tests/codegen-llvm/deduced-param-attrs.rs @@ -1,81 +1,172 @@ //@ compile-flags: -Copt-level=3 -Cno-prepopulate-passes - +//@ compile-flags: -Cpanic=abort -Csymbol-mangling-version=v0 +//@ revisions: LLVM21 LLVM20 +//@ [LLVM21] min-llvm-version: 21 +//@ [LLVM20] max-llvm-major-version: 20 +#![feature(custom_mir, core_intrinsics)] #![crate_type = "lib"] -#![allow(internal_features)] -#![feature(unsized_fn_params)] - +extern crate core; +use core::intrinsics::mir::*; use std::cell::Cell; -use std::hint; +use std::hint::black_box; +use std::mem::ManuallyDrop; -// Check to make sure that we can deduce the `readonly` attribute from function bodies for -// parameters passed indirectly. +pub struct Big { + pub blah: [i32; 1024], +} -pub struct BigStruct { - blah: [i32; 1024], +pub struct BigCell { + pub blah: [Cell; 1024], } -pub struct BigCellContainer { - blah: [Cell; 1024], +pub struct BigDrop { + pub blah: [u8; 1024], } -// The by-value parameter for this big struct can be marked readonly. -// -// CHECK: @use_big_struct_immutably({{.*}} readonly {{.*}} %big_struct) -#[no_mangle] -pub fn use_big_struct_immutably(big_struct: BigStruct) { - hint::black_box(&big_struct); +impl Drop for BigDrop { + #[inline(never)] + fn drop(&mut self) {} } -// The by-value parameter for this big struct can't be marked readonly, because we mutate it. -// -// CHECK: @use_big_struct_mutably( +// CHECK-LABEL: @mutate( // CHECK-NOT: readonly -// CHECK-SAME: %big_struct) -#[no_mangle] -pub fn use_big_struct_mutably(mut big_struct: BigStruct) { - big_struct.blah[987] = 654; - hint::black_box(&big_struct); +// CHECK-SAME: %b) +#[unsafe(no_mangle)] +pub fn mutate(mut b: Big) { + b.blah[987] = 654; + black_box(&b); } -// The by-value parameter for this big struct can't be marked readonly, because it contains -// UnsafeCell. -// -// CHECK: @use_big_cell_container( -// CHECK-NOT: readonly -// CHECK-SAME: %big_cell_container) -#[no_mangle] -pub fn use_big_cell_container(big_cell_container: BigCellContainer) { - hint::black_box(&big_cell_container); +// LLVM21-LABEL: @deref_mut({{.*}}readonly {{.*}}captures(none) {{.*}}%c) +// LLVM20-LABEL: @deref_mut({{.*}}readonly {{.*}}%c) +#[unsafe(no_mangle)] +pub fn deref_mut(c: (BigCell, &mut usize)) { + *c.1 = 42; +} + +// LLVM21-LABEL: @call_copy_arg(ptr {{.*}}readonly {{.*}}captures(none){{.*}}) +// LLVM20-LABEL: @call_copy_arg(ptr {{.*}}readonly {{.*}}) +#[unsafe(no_mangle)] +#[custom_mir(dialect = "runtime", phase = "optimized")] +pub fn call_copy_arg(a: Big) { + mir! { + { + Call(RET = call_copy_arg(a), ReturnTo(bb1), UnwindUnreachable()) + } + bb1 = { + Return() + } + } +} + +// CHECK-LABEL: @call_move_arg( +// CHECK-NOT: readonly +// LLVM21-SAME: captures(address) +// CHECK-SAME: ) +#[unsafe(no_mangle)] +#[custom_mir(dialect = "runtime", phase = "optimized")] +pub fn call_move_arg(a: Big) { + mir! { + { + Call(RET = call_move_arg(Move(a)), ReturnTo(bb1), UnwindUnreachable()) + } + bb1 = { + Return() + } + } +} + +fn shared_borrow(a: T) { + black_box(&a); } -// Make sure that we don't mistakenly mark a big struct as `readonly` when passed through a generic -// type parameter if it contains UnsafeCell. +// Freeze parameter cannot be mutated through a shared borrow. // -// CHECK: @use_something( -// CHECK-NOT: readonly -// CHECK-SAME: %something) -#[no_mangle] +// CHECK-LABEL: ; deduced_param_attrs::shared_borrow:: +// CHECK-NEXT: ; +// LLVM21-NEXT: (ptr {{.*}}readonly {{.*}}captures(address) {{.*}}%a) +// LLVM20-NEXT: (ptr {{.*}}readonly {{.*}}%a) +pub static A0: fn(Big) = shared_borrow; + +// !Freeze parameter can be mutated through a shared borrow. +// +// CHECK-LABEL: ; deduced_param_attrs::shared_borrow:: +// CHECK-NEXT: ; +// CHECK-NOT: readonly +// CHECK-NEXT: %a) +pub static A1: fn(BigCell) = shared_borrow; + +// The parameter can be mutated through a raw const borrow. +// +// CHECK-LABEL: ; deduced_param_attrs::raw_const_borrow +// CHECK-NOT: readonly +// CHECK-NEXT : %a) #[inline(never)] -pub fn use_something(something: T) { - hint::black_box(&something); +pub fn raw_const_borrow(a: Big) { + black_box(&raw const a); } -// Make sure that we still mark a big `Freeze` struct as `readonly` when passed through a generic -// type parameter. +fn consume(_: T) {} + +// The parameter doesn't need to be dropped. // -// CHECK: @use_something_freeze( -// CHECK-SAME: readonly -// CHECK-SAME: %x) -#[no_mangle] -#[inline(never)] -pub fn use_something_freeze(x: T) {} +// CHECK-LABEL: ; deduced_param_attrs::consume:: +// CHECK-NEXT: ; +// LLVM21-NEXT: (ptr {{.*}}readonly {{.*}}captures(none) {{.*}}) +// LLVM20-NEXT: (ptr {{.*}}readonly {{.*}}) +pub static B0: fn(BigCell) = consume; + +// The parameter needs to be dropped. +// +// CHECK-LABEL: ; deduced_param_attrs::consume:: +// CHECK-NEXT: ; +// LLVM21-NEXT: (ptr {{.*}}captures(address) {{.*}}) +// LLVM20-NEXT: (ptr {{.*}}) +pub static B1: fn(BigDrop) = consume; + +fn consume_parts(t: (T, T)) { + let (_t0, ..) = t; +} + +// In principle it would be possible to deduce readonly here. +// +// CHECK-LABEL: ; deduced_param_attrs::consume_parts::<[u8; 40]> +// CHECK-NEXT: ; +// CHECK-NOT: readonly +// CHECK-NEXT: %t) +pub static C1: fn(([u8; 40], [u8; 40])) = consume_parts; + +// The inner field of ManuallyDrop needs to be dropped. +// +// CHECK-LABEL: @manually_drop_field( +// CHECK-NOT: readonly +// CHECK-SAME: %b) +#[unsafe(no_mangle)] +pub fn manually_drop_field(a: fn() -> BigDrop, mut b: ManuallyDrop) { + // FIXME(tmiasko) replace with custom MIR, instead of expecting MIR optimizations to turn this + // into: drop((_2.0: BigDrop)) + *b = a(); + unsafe { core::intrinsics::unreachable() } +} + +// `readonly` is omitted from the return place, even when applicable. +// +// CHECK-LABEL: @never_returns( +// CHECK-NOT: readonly +// CHECK-SAME: %_0) +#[unsafe(no_mangle)] +pub fn never_returns() -> [u8; 80] { + loop {} +} -#[no_mangle] -pub fn forward_big_cell_container(big_cell_container: BigCellContainer) { - use_something(big_cell_container) +// LLVM21-LABEL: @not_captured_return_place(ptr{{.*}} captures(none) {{.*}}%_0) +#[unsafe(no_mangle)] +pub fn not_captured_return_place() -> [u8; 80] { + [0u8; 80] } -#[no_mangle] -pub fn forward_big_container(big_struct: BigStruct) { - use_something_freeze(big_struct) +// LLVM21-LABEL: @captured_return_place(ptr{{.*}} captures(address) {{.*}}%_0) +#[unsafe(no_mangle)] +pub fn captured_return_place() -> [u8; 80] { + black_box([0u8; 80]) } diff --git a/tests/codegen-llvm/function-arguments.rs b/tests/codegen-llvm/function-arguments.rs index a0744e44c61ec..aaa1d57592a8f 100644 --- a/tests/codegen-llvm/function-arguments.rs +++ b/tests/codegen-llvm/function-arguments.rs @@ -134,7 +134,7 @@ pub fn mutable_notunpin_borrow(_: &mut NotUnpin) {} #[no_mangle] pub fn notunpin_borrow(_: &NotUnpin) {} -// CHECK: @indirect_struct(ptr{{( dead_on_return)?}} noalias noundef readonly align 4{{( captures\(address\))?}} dereferenceable(32) %_1) +// CHECK: @indirect_struct(ptr{{( dead_on_return)?}} noalias noundef readonly align 4{{( captures\(none\))?}} dereferenceable(32) %_1) #[no_mangle] pub fn indirect_struct(_: S) {} @@ -197,7 +197,7 @@ pub fn notunpin_box(x: Box) -> Box { x } -// CHECK: @struct_return(ptr{{( dead_on_unwind)?}} noalias noundef{{( writable)?}} sret([32 x i8]) align 4{{( captures\(address\))?}} dereferenceable(32){{( %_0)?}}) +// CHECK: @struct_return(ptr{{( dead_on_unwind)?}} noalias noundef{{( writable)?}} sret([32 x i8]) align 4{{( captures\(none\))?}} dereferenceable(32){{( %_0)?}}) #[no_mangle] pub fn struct_return() -> S { S { _field: [0, 0, 0, 0, 0, 0, 0, 0] } diff --git a/tests/ui/abi/c-zst.powerpc-linux.stderr b/tests/ui/abi/c-zst.powerpc-linux.stderr index e79b3fcec8b97..816addd795761 100644 --- a/tests/ui/abi/c-zst.powerpc-linux.stderr +++ b/tests/ui/abi/c-zst.powerpc-linux.stderr @@ -27,7 +27,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | CapturesAddress | NonNull | NoUndef, + regular: CapturesAddress | NoAlias | NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some( diff --git a/tests/ui/abi/c-zst.s390x-linux.stderr b/tests/ui/abi/c-zst.s390x-linux.stderr index e79b3fcec8b97..816addd795761 100644 --- a/tests/ui/abi/c-zst.s390x-linux.stderr +++ b/tests/ui/abi/c-zst.s390x-linux.stderr @@ -27,7 +27,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | CapturesAddress | NonNull | NoUndef, + regular: CapturesAddress | NoAlias | NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some( diff --git a/tests/ui/abi/c-zst.sparc64-linux.stderr b/tests/ui/abi/c-zst.sparc64-linux.stderr index e79b3fcec8b97..816addd795761 100644 --- a/tests/ui/abi/c-zst.sparc64-linux.stderr +++ b/tests/ui/abi/c-zst.sparc64-linux.stderr @@ -27,7 +27,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | CapturesAddress | NonNull | NoUndef, + regular: CapturesAddress | NoAlias | NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some( diff --git a/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr b/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr index e79b3fcec8b97..816addd795761 100644 --- a/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr +++ b/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr @@ -27,7 +27,7 @@ error: fn_abi_of(pass_zst) = FnAbi { }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | CapturesAddress | NonNull | NoUndef, + regular: CapturesAddress | NoAlias | NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some( diff --git a/tests/ui/abi/debug.generic.stderr b/tests/ui/abi/debug.generic.stderr index fd4c43591425c..04d6f50872a39 100644 --- a/tests/ui/abi/debug.generic.stderr +++ b/tests/ui/abi/debug.generic.stderr @@ -454,7 +454,7 @@ error: ABIs are not compatible }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | CapturesAddress | NonNull | NoUndef, + regular: CapturesAddress | NoAlias | NonNull | NoUndef, arg_ext: None, pointee_size: Size(32 bytes), pointee_align: Some( @@ -527,7 +527,7 @@ error: ABIs are not compatible }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | CapturesAddress | NonNull | NoUndef, + regular: CapturesAddress | NoAlias | NonNull | NoUndef, arg_ext: None, pointee_size: Size(128 bytes), pointee_align: Some( @@ -939,7 +939,7 @@ error: fn_abi_of(assoc_test) = FnAbi { }, mode: Direct( ArgAttributes { - regular: NoAlias | NonNull | ReadOnly | NoUndef | CapturesReadOnly, + regular: CapturesReadOnly | NoAlias | NonNull | ReadOnly | NoUndef, arg_ext: None, pointee_size: Size(2 bytes), pointee_align: Some( diff --git a/tests/ui/abi/debug.loongarch64.stderr b/tests/ui/abi/debug.loongarch64.stderr index 95351b42092a4..85c888c4fae08 100644 --- a/tests/ui/abi/debug.loongarch64.stderr +++ b/tests/ui/abi/debug.loongarch64.stderr @@ -454,7 +454,7 @@ error: ABIs are not compatible }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | CapturesAddress | NonNull | NoUndef, + regular: CapturesAddress | NoAlias | NonNull | NoUndef, arg_ext: None, pointee_size: Size(32 bytes), pointee_align: Some( @@ -527,7 +527,7 @@ error: ABIs are not compatible }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | CapturesAddress | NonNull | NoUndef, + regular: CapturesAddress | NoAlias | NonNull | NoUndef, arg_ext: None, pointee_size: Size(128 bytes), pointee_align: Some( @@ -939,7 +939,7 @@ error: fn_abi_of(assoc_test) = FnAbi { }, mode: Direct( ArgAttributes { - regular: NoAlias | NonNull | ReadOnly | NoUndef | CapturesReadOnly, + regular: CapturesReadOnly | NoAlias | NonNull | ReadOnly | NoUndef, arg_ext: None, pointee_size: Size(2 bytes), pointee_align: Some( diff --git a/tests/ui/abi/debug.riscv64.stderr b/tests/ui/abi/debug.riscv64.stderr index 95351b42092a4..85c888c4fae08 100644 --- a/tests/ui/abi/debug.riscv64.stderr +++ b/tests/ui/abi/debug.riscv64.stderr @@ -454,7 +454,7 @@ error: ABIs are not compatible }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | CapturesAddress | NonNull | NoUndef, + regular: CapturesAddress | NoAlias | NonNull | NoUndef, arg_ext: None, pointee_size: Size(32 bytes), pointee_align: Some( @@ -527,7 +527,7 @@ error: ABIs are not compatible }, mode: Indirect { attrs: ArgAttributes { - regular: NoAlias | CapturesAddress | NonNull | NoUndef, + regular: CapturesAddress | NoAlias | NonNull | NoUndef, arg_ext: None, pointee_size: Size(128 bytes), pointee_align: Some( @@ -939,7 +939,7 @@ error: fn_abi_of(assoc_test) = FnAbi { }, mode: Direct( ArgAttributes { - regular: NoAlias | NonNull | ReadOnly | NoUndef | CapturesReadOnly, + regular: CapturesReadOnly | NoAlias | NonNull | ReadOnly | NoUndef, arg_ext: None, pointee_size: Size(2 bytes), pointee_align: Some(