From 5ead46377d791284efa1019d437345f51842b9ec Mon Sep 17 00:00:00 2001 From: Camille Gillot Date: Sun, 7 Sep 2025 13:47:17 +0000 Subject: [PATCH 1/3] Only decide once whether to use a complex projection. --- compiler/rustc_mir_transform/src/gvn.rs | 27 +++++++++++-------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index bd8ea7dddad83..8d7a498bbf820 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -971,7 +971,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { match self.compute_place_value(*place, location) { Ok(value) => { - if let Some(new_place) = self.try_as_place(value, location, true) + if let Some(new_place) = self.try_as_place(value, location, false) && (new_place.local != place.local || new_place.projection.len() < place.projection.len()) { @@ -1014,7 +1014,6 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { #[instrument(level = "trace", skip(self), ret)] fn simplify_rvalue( &mut self, - lhs: &Place<'tcx>, rvalue: &mut Rvalue<'tcx>, location: Location, ) -> Option { @@ -1034,7 +1033,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { Value::Repeat(op, amount) } Rvalue::NullaryOp(op, ty) => Value::NullaryOp(op, ty), - Rvalue::Aggregate(..) => return self.simplify_aggregate(lhs, rvalue, location), + Rvalue::Aggregate(..) => return self.simplify_aggregate(rvalue, location), Rvalue::Ref(_, borrow_kind, ref mut place) => { self.simplify_place_projection(place, location); return self.new_pointer(*place, AddressKind::Ref(borrow_kind)); @@ -1142,7 +1141,6 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { fn simplify_aggregate( &mut self, - lhs: &Place<'tcx>, rvalue: &mut Rvalue<'tcx>, location: Location, ) -> Option { @@ -1225,15 +1223,6 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { } if let Some(value) = self.simplify_aggregate_to_copy(ty, variant_index, &fields) { - // Allow introducing places with non-constant offsets, as those are still better than - // reconstructing an aggregate. But avoid creating `*a = copy (*b)`, as they might be - // aliases resulting in overlapping assignments. - let allow_complex_projection = - lhs.projection[..].iter().all(PlaceElem::is_stable_offset); - if let Some(place) = self.try_as_place(value, location, allow_complex_projection) { - self.reused_locals.insert(place.local); - *rvalue = Rvalue::Use(Operand::Copy(place)); - } return Some(value); } @@ -1879,13 +1868,21 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, '_, 'tcx> { ) { self.simplify_place_projection(lhs, location); - let value = self.simplify_rvalue(lhs, rvalue, location); + let value = self.simplify_rvalue(rvalue, location); if let Some(value) = value { + // Allow introducing places with non-constant offsets, as those are still better than + // reconstructing an aggregate. But avoid creating `*a = copy (*b)`, as they might be + // aliases resulting in overlapping assignments. + let allow_complex_projection = + lhs.projection[..].iter().all(PlaceElem::is_stable_offset); + if let Some(const_) = self.try_as_constant(value) { *rvalue = Rvalue::Use(Operand::Constant(Box::new(const_))); - } else if let Some(place) = self.try_as_place(value, location, false) + } else if let Some(place) = self.try_as_place(value, location, allow_complex_projection) && *rvalue != Rvalue::Use(Operand::Move(place)) && *rvalue != Rvalue::Use(Operand::Copy(place)) + // Avoid introducing overlapping assignments to the same local. + && place.local != lhs.local { *rvalue = Rvalue::Use(Operand::Copy(place)); self.reused_locals.insert(place.local); From e9f9832273412b128e490dac2f64eebf72dc6cad Mon Sep 17 00:00:00 2001 From: Camille Gillot Date: Sun, 24 Aug 2025 02:56:14 +0000 Subject: [PATCH 2/3] Simplify project. --- compiler/rustc_mir_transform/src/gvn.rs | 137 ++++++++++++------------ 1 file changed, 68 insertions(+), 69 deletions(-) diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index 8d7a498bbf820..fe6c8eb054463 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -800,81 +800,80 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { proj: ProjectionElem>, ) -> Option<(PlaceTy<'tcx>, VnIndex)> { let projection_ty = place_ty.projection_ty(self.tcx, proj); - let proj = match proj { - ProjectionElem::Deref => { - if let Some(Mutability::Not) = place_ty.ty.ref_mutability() + let proj = proj.try_map(Some, |_| ())?; + + if let ProjectionElem::Deref = proj + && !( + // An immutable borrow `_x` always points to the same value for the + // lifetime of the borrow, so we can merge all instances of `*_x`. + place_ty.ty.ref_mutability() == Some(Mutability::Not) && projection_ty.ty.is_freeze(self.tcx, self.typing_env()) - { - if let Value::Address { base, projection, .. } = self.get(value) - && let Some(value) = self.dereference_address(base, projection) - { - return Some((projection_ty, value)); - } + ) + { + return None; + } - // An immutable borrow `_x` always points to the same value for the - // lifetime of the borrow, so we can merge all instances of `*_x`. - return Some((projection_ty, self.insert_deref(projection_ty.ty, value))); - } else { - return None; - } + match (proj, self.get(value)) { + (ProjectionElem::Deref, Value::Address { base, projection, .. }) + if let Some(deref) = self.dereference_address(base, projection) => + { + return Some((projection_ty, deref)); } - ProjectionElem::Downcast(name, index) => ProjectionElem::Downcast(name, index), - ProjectionElem::Field(f, _) => match self.get(value) { - Value::Aggregate(_, fields) => return Some((projection_ty, fields[f.as_usize()])), - Value::Union(active, field) if active == f => return Some((projection_ty, field)), - Value::Projection(outer_value, ProjectionElem::Downcast(_, read_variant)) - if let Value::Aggregate(written_variant, fields) = self.get(outer_value) - // This pass is not aware of control-flow, so we do not know whether the - // replacement we are doing is actually reachable. We could be in any arm of - // ``` - // match Some(x) { - // Some(y) => /* stuff */, - // None => /* other */, - // } - // ``` - // - // In surface rust, the current statement would be unreachable. - // - // However, from the reference chapter on enums and RFC 2195, - // accessing the wrong variant is not UB if the enum has repr. - // So it's not impossible for a series of MIR opts to generate - // a downcast to an inactive variant. - && written_variant == read_variant => - { - return Some((projection_ty, fields[f.as_usize()])); - } - _ => ProjectionElem::Field(f, ()), - }, - ProjectionElem::Index(idx) => { - if let Value::Repeat(inner, _) = self.get(value) { - return Some((projection_ty, inner)); - } - ProjectionElem::Index(idx) + (ProjectionElem::Deref, _) => { + return Some((projection_ty, self.insert_deref(projection_ty.ty, value))); } - ProjectionElem::ConstantIndex { offset, min_length, from_end } => { - match self.get(value) { - Value::Repeat(inner, _) => { - return Some((projection_ty, inner)); - } - Value::Aggregate(_, operands) => { - let offset = if from_end { - operands.len() - offset as usize - } else { - offset as usize - }; - let value = operands.get(offset).copied()?; - return Some((projection_ty, value)); - } - _ => {} - }; - ProjectionElem::ConstantIndex { offset, min_length, from_end } + (ProjectionElem::Field(f, _), Value::Aggregate(_, fields)) => { + return Some((projection_ty, fields[f.as_usize()])); } - ProjectionElem::Subslice { from, to, from_end } => { - ProjectionElem::Subslice { from, to, from_end } + (ProjectionElem::Field(f, _), Value::Union(active, field)) if active == f => { + return Some((projection_ty, field)); } - ProjectionElem::OpaqueCast(_) => ProjectionElem::OpaqueCast(()), - ProjectionElem::UnwrapUnsafeBinder(_) => ProjectionElem::UnwrapUnsafeBinder(()), - }; + ( + ProjectionElem::Field(f, _), + Value::Projection(outer_value, ProjectionElem::Downcast(_, read_variant)), + ) if let Value::Aggregate(written_variant, fields) = self.get(outer_value) + // This pass is not aware of control-flow, so we do not know whether the + // replacement we are doing is actually reachable. We could be in any arm of + // ``` + // match Some(x) { + // Some(y) => /* stuff */, + // None => /* other */, + // } + // ``` + // + // In surface rust, the current statement would be unreachable. + // + // However, from the reference chapter on enums and RFC 2195, + // accessing the wrong variant is not UB if the enum has repr. + // So it's not impossible for a series of MIR opts to generate + // a downcast to an inactive variant. + && written_variant == read_variant => + { + return Some((projection_ty, fields[f.as_usize()])); + } + (ProjectionElem::Index(_), Value::Repeat(inner, _)) => { + return Some((projection_ty, inner)); + } + (ProjectionElem::ConstantIndex { .. }, Value::Repeat(inner, _)) => { + return Some((projection_ty, inner)); + } + ( + ProjectionElem::ConstantIndex { offset, from_end: false, .. }, + Value::Aggregate(_, operands), + ) => { + let value = operands.get(offset as usize).copied()?; + return Some((projection_ty, value)); + } + ( + ProjectionElem::ConstantIndex { offset, from_end: true, .. }, + Value::Aggregate(_, operands), + ) => { + let offset = operands.len().checked_sub(offset as usize)?; + let value = operands[offset]; + return Some((projection_ty, value)); + } + _ => {} + } let value = self.insert(projection_ty.ty, Value::Projection(value, proj)); Some((projection_ty, value)) From b75587b3b0a7bba14f81049968c4792ee1d39dc0 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Mon, 23 Jun 2025 09:46:51 +0000 Subject: [PATCH 3/3] Extend GVN to perform local value numbering. --- compiler/rustc_mir_transform/src/gvn.rs | 175 ++++++++++++------ compiler/rustc_mir_transform/src/ssa.rs | 4 + tests/coverage/closure.cov-map | 56 +++--- .../boxes.main.GVN.panic-abort.diff | 15 +- .../boxes.main.GVN.panic-unwind.diff | 15 +- .../discriminant.main.GVN.32bit.diff | 9 +- .../discriminant.main.GVN.64bit.diff | 9 +- tests/mir-opt/const_prop/discriminant.rs | 2 +- .../const_prop/mutable_variable.main.GVN.diff | 3 +- tests/mir-opt/const_prop/mutable_variable.rs | 2 +- .../mutable_variable_no_prop.main.GVN.diff | 12 +- .../const_prop/mutable_variable_no_prop.rs | 4 +- .../gvn.fn_pointers.GVN.panic-abort.diff | 18 +- .../gvn.fn_pointers.GVN.panic-unwind.diff | 18 +- tests/mir-opt/gvn.rs | 19 +- ...xpression_elimination.GVN.panic-abort.diff | 12 +- ...pression_elimination.GVN.panic-unwind.diff | 12 +- ...py_aggregate.all_copy_use_changed.GVN.diff | 14 +- ..._aggregate.all_copy_use_changed_2.GVN.diff | 14 +- tests/mir-opt/gvn_copy_moves.fn0.GVN.diff | 2 +- tests/mir-opt/gvn_copy_moves.fn2.GVN.diff | 27 +++ tests/mir-opt/gvn_copy_moves.rs | 34 +++- .../mir-opt/gvn_repeat.repeat_local.GVN.diff | 3 +- .../mir-opt/gvn_repeat.repeat_place.GVN.diff | 3 +- tests/mir-opt/gvn_repeat.rs | 6 +- .../loops.vec_move.PreCodegen.after.mir | 2 +- ...ated_loop.PreCodegen.after.panic-abort.mir | 12 +- 27 files changed, 331 insertions(+), 171 deletions(-) create mode 100644 tests/mir-opt/gvn_copy_moves.fn2.GVN.diff diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index fe6c8eb054463..fc2f6847e5b91 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -1,4 +1,4 @@ -//! Global value numbering. +//! Value numbering. //! //! MIR may contain repeated and/or redundant computations. The objective of this pass is to detect //! such redundancies and re-use the already-computed result when possible. @@ -8,15 +8,23 @@ //! //! We traverse all assignments `x = rvalue` and operands. //! -//! For each SSA one, we compute a symbolic representation of values that are assigned to SSA +//! For each assignment, we compute a symbolic representation of values that are assigned to SSA //! locals. This symbolic representation is defined by the `Value` enum. Each produced instance of //! `Value` is interned as a `VnIndex`, which allows us to cheaply compute identical values. //! -//! For each non-SSA -//! one, we compute the `VnIndex` of the rvalue. If this `VnIndex` is associated to a constant, we -//! replace the rvalue/operand by that constant. Otherwise, if there is an SSA local `y` -//! associated to this `VnIndex`, and if its definition location strictly dominates the assignment -//! to `x`, we replace the assignment by `x = y`. +//! If the local is SSA, we append it into the mapping `rev_locals_ssa[value]` for later reuse. +//! That mapping accumulates the knowledge across basic blocks. +//! +//! If the local is non-SSA, we remove it from `rev_locals_non_ssa[old_value]` and append it to +//! `rev_locals_non_ssa[new_value]`. That mapping is cleared at the beginning of each basic block, +//! to ensure we do not carry information across blocks. +//! +//! If the computed `VnIndex` is associated to a constant, we replace the rvalue/operand by that +//! constant. Otherwise, if there is a local `y` associated to this `VnIndex`: +//! - if `y` is SSA and its definition location strictly dominates the assignment to `x`, we +//! replace the assignment by `x = y`; +//! - if `y` is not SSA, then the assignment happens earlier in the same block, so we replace the +//! assignment by `x = y`. //! //! By opportunity, this pass simplifies some `Rvalue`s based on the accumulated knowledge. //! @@ -97,7 +105,7 @@ use rustc_const_eval::interpret::{ ImmTy, Immediate, InterpCx, MemPlaceMeta, MemoryKind, OpTy, Projectable, Scalar, intern_const_alloc_for_constprop, }; -use rustc_data_structures::fx::FxHasher; +use rustc_data_structures::fx::{FxHashMap, FxHasher}; use rustc_data_structures::graph::dominators::Dominators; use rustc_hir::def::DefKind; use rustc_index::bit_set::DenseBitSet; @@ -358,10 +366,15 @@ struct VnState<'body, 'a, 'tcx> { local_decls: &'body LocalDecls<'tcx>, is_coroutine: bool, /// Value stored in each local. - locals: IndexVec>, + locals_ssa: FxHashMap, + // Keep two separate maps to efficiently clear non-SSA locals. + locals_non_ssa: FxHashMap, /// Locals that are assigned that value. - // This vector does not hold all the values of `VnIndex` that we create. - rev_locals: IndexVec>, + // This vector holds the locals that are SSA. + rev_locals_ssa: IndexVec>, + // This map holds the locals that are not SSA. This map is cleared at the end of each block. + // Therefore, we do not need a location, the local always appears before the current location. + rev_locals_non_ssa: FxHashMap>, values: ValueSet<'a, 'tcx>, /// Values evaluated as constants if possible. evaluated: IndexVec>>, @@ -395,8 +408,13 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { ecx: InterpCx::new(tcx, DUMMY_SP, typing_env, DummyMachine), local_decls, is_coroutine: body.coroutine.is_some(), - locals: IndexVec::from_elem(None, local_decls), - rev_locals: IndexVec::with_capacity(num_values), + locals_ssa: FxHashMap::with_capacity_and_hasher(local_decls.len(), Default::default()), + locals_non_ssa: FxHashMap::with_capacity_and_hasher( + local_decls.len(), + Default::default(), + ), + rev_locals_ssa: IndexVec::with_capacity(num_values), + rev_locals_non_ssa: FxHashMap::default(), values: ValueSet::new(num_values), evaluated: IndexVec::with_capacity(num_values), derefs: Vec::new(), @@ -415,11 +433,11 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { fn insert(&mut self, ty: Ty<'tcx>, value: Value<'a, 'tcx>) -> VnIndex { let (index, new) = self.values.insert(ty, value); if new { - // Grow `evaluated` and `rev_locals` here to amortize the allocations. + // Grow `evaluated` and `rev_locals_ssa` here to amortize the allocations. let evaluated = self.eval_to_const(index); let _index = self.evaluated.push(evaluated); debug_assert_eq!(index, _index); - let _index = self.rev_locals.push(SmallVec::new()); + let _index = self.rev_locals_ssa.push(Default::default()); debug_assert_eq!(index, _index); } index @@ -432,7 +450,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { let index = self.values.insert_unique(ty, Value::Opaque); let _index = self.evaluated.push(None); debug_assert_eq!(index, _index); - let _index = self.rev_locals.push(SmallVec::new()); + let _index = self.rev_locals_ssa.push(SmallVec::new()); debug_assert_eq!(index, _index); index } @@ -450,17 +468,19 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { let mut projection = place.projection.iter(); let base = if place.is_indirect_first_projection() { - let base = self.locals[place.local]?; + let base = self.local(place.local); // Skip the initial `Deref`. projection.next(); AddressBase::Deref(base) } else { AddressBase::Local(place.local) }; + let arena = self.arena; + // Do not try evaluating inside `Index`, this has been done by `simplify_place_projection`. let projection = - projection.map(|proj| proj.try_map(|index| self.locals[index], |ty| ty).ok_or(())); - let projection = self.arena.try_alloc_from_iter(projection).ok()?; + projection.map(|proj| proj.try_map(|index| Some(self.local(index)), |ty| ty).ok_or(())); + let projection = arena.try_alloc_from_iter(projection).ok()?; let index = self.values.insert_unique(ty, |provenance| Value::Address { base, @@ -471,7 +491,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { let evaluated = self.eval_to_const(index); let _index = self.evaluated.push(evaluated); debug_assert_eq!(index, _index); - let _index = self.rev_locals.push(SmallVec::new()); + let _index = self.rev_locals_ssa.push(SmallVec::new()); debug_assert_eq!(index, _index); Some(index) @@ -496,7 +516,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { let evaluated = self.eval_to_const(index); let _index = self.evaluated.push(evaluated); debug_assert_eq!(index, _index); - let _index = self.rev_locals.push(SmallVec::new()); + let _index = self.rev_locals_ssa.push(SmallVec::new()); debug_assert_eq!(index, _index); } index @@ -512,12 +532,55 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { self.values.ty(index) } - /// Record that `local` is assigned `value`. `local` must be SSA. + /// Record that `local` is assigned `value`. #[instrument(level = "trace", skip(self))] fn assign(&mut self, local: Local, value: VnIndex) { - debug_assert!(self.ssa.is_ssa(local)); - self.locals[local] = Some(value); - self.rev_locals[value].push(local); + if self.ssa.is_ssa(local) { + self.locals_ssa.insert(local, value); + self.rev_locals_ssa[value].push(local); + } else { + self.locals_non_ssa.insert(local, value); + self.rev_locals_non_ssa.entry(value).or_default().push(local); + } + } + + /// Return the value assigned to a local, or assign an opaque value and return it. + #[instrument(level = "trace", skip(self), ret)] + fn local(&mut self, local: Local) -> VnIndex { + if let Some(value) = self.locals_ssa.get(&local) { + return *value; + } + if let Some(value) = self.locals_non_ssa.get(&local) { + return *value; + } + let value = self.new_opaque(self.local_decls[local].ty); + // For SSA locals, the assignment dominates all uses. + // If we are here, this means the locals is not SSA. + self.locals_non_ssa.insert(local, value); + self.rev_locals_non_ssa.entry(value).or_default().push(local); + value + } + + #[instrument(level = "trace", skip(self))] + fn discard_place(&mut self, place: Place<'tcx>) { + let discard_local = |this: &mut Self, local| { + if this.ssa.is_ssa(local) { + return; + } + if let Some(value) = this.locals_non_ssa.remove(&local) { + this.rev_locals_non_ssa.entry(value).or_default().retain(|l| *l != local); + } + }; + if place.is_indirect_first_projection() { + // Non-local mutation maybe invalidate deref. + self.invalidate_derefs(); + // Remove stored value from borrowed locals. + for local in self.ssa.borrowed_locals().iter() { + discard_local(self, local); + } + } else { + discard_local(self, place.local); + } } fn insert_bool(&mut self, flag: bool) -> VnIndex { @@ -776,7 +839,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { let (mut place_ty, mut value) = match base { // The base is a local, so we take the local's value and project from it. AddressBase::Local(local) => { - let local = self.locals[local]?; + let local = self.local(local); let place_ty = PlaceTy::from_ty(self.ty(local)); (place_ty, local) } @@ -885,7 +948,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { // If the projection is indirect, we treat the local as a value, so can replace it with // another local. if place.is_indirect_first_projection() - && let Some(base) = self.locals[place.local] + && let base = self.local(place.local) && let Some(new_local) = self.try_as_local(base, location) && place.local != new_local { @@ -897,9 +960,8 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { for i in 0..projection.len() { let elem = projection[i]; - if let ProjectionElem::Index(idx_local) = elem - && let Some(idx) = self.locals[idx_local] - { + if let ProjectionElem::Index(idx_local) = elem { + let idx = self.local(idx_local); if let Some(offset) = self.evaluated[idx].as_ref() && let Some(offset) = self.ecx.read_target_usize(offset).discard_err() && let Some(min_length) = offset.checked_add(1) @@ -935,7 +997,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { let mut place_ref = place.as_ref(); // Invariant: `value` holds the value up-to the `index`th projection excluded. - let Some(mut value) = self.locals[place.local] else { return Err(place_ref) }; + let mut value = self.local(place.local); // Invariant: `value` has type `place_ty`, with optional downcast variant if needed. let mut place_ty = PlaceTy::from_ty(self.local_decls[place.local].ty); for (index, proj) in place.projection.iter().enumerate() { @@ -946,7 +1008,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { place_ref = PlaceRef { local, projection: &place.projection[index..] }; } - let Some(proj) = proj.try_map(|value| self.locals[value], |ty| ty) else { + let Some(proj) = proj.try_map(|value| Some(self.local(value)), |ty| ty) else { return Err(place_ref); }; let Some(ty_and_value) = self.project(place_ty, value, proj) else { @@ -1832,11 +1894,17 @@ impl<'tcx> VnState<'_, '_, 'tcx> { /// If there is a local which is assigned `index`, and its assignment strictly dominates `loc`, /// return it. If you used this local, add it to `reused_locals` to remove storage statements. fn try_as_local(&mut self, index: VnIndex, loc: Location) -> Option { - let other = self.rev_locals.get(index)?; - other - .iter() - .find(|&&other| self.ssa.assignment_dominates(&self.dominators, other, loc)) - .copied() + if let Some(ssa) = self.rev_locals_ssa.get(index) + && let Some(other) = ssa + .iter() + .find(|&&other| self.ssa.assignment_dominates(&self.dominators, other, loc)) + { + Some(*other) + } else if let Some(non_ssa) = self.rev_locals_non_ssa.get(&index) { + non_ssa.first().copied() + } else { + None + } } } @@ -1845,11 +1913,16 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, '_, 'tcx> { self.tcx } + fn visit_basic_block_data(&mut self, block: BasicBlock, bbdata: &mut BasicBlockData<'tcx>) { + self.locals_non_ssa.clear(); + self.rev_locals_non_ssa.clear(); + self.super_basic_block_data(block, bbdata); + } + fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, location: Location) { self.simplify_place_projection(place, location); - if context.is_mutating_use() && place.is_indirect() { - // Non-local mutation maybe invalidate deref. - self.invalidate_derefs(); + if context.is_mutating_use() { + self.discard_place(*place); } self.super_place(place, context, location); } @@ -1888,13 +1961,9 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, '_, 'tcx> { } } - if lhs.is_indirect() { - // Non-local mutation maybe invalidate deref. - self.invalidate_derefs(); - } + self.discard_place(*lhs); if let Some(local) = lhs.as_local() - && self.ssa.is_ssa(local) && let rvalue_ty = rvalue.ty(self.local_decls, self.tcx) // FIXME(#112651) `rvalue` may have a subtype to `local`. We can only mark // `local` as reusable if we have an exact type match. @@ -1906,14 +1975,13 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, '_, 'tcx> { } fn visit_terminator(&mut self, terminator: &mut Terminator<'tcx>, location: Location) { - if let Terminator { kind: TerminatorKind::Call { destination, .. }, .. } = terminator { - if let Some(local) = destination.as_local() - && self.ssa.is_ssa(local) - { - let ty = self.local_decls[local].ty; - let opaque = self.new_opaque(ty); - self.assign(local, opaque); - } + self.super_terminator(terminator, location); + if let Terminator { kind: TerminatorKind::Call { destination, .. }, .. } = terminator + && let Some(local) = destination.as_local() + { + let ty = self.local_decls[local].ty; + let opaque = self.new_opaque(ty); + self.assign(local, opaque); } // Function calls and ASM may invalidate (nested) derefs. We must handle them carefully. // Currently, only preserving derefs for trivial terminators like SwitchInt and Goto. @@ -1924,7 +1992,6 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, '_, 'tcx> { if !safe_to_preserve_derefs { self.invalidate_derefs(); } - self.super_terminator(terminator, location); } } diff --git a/compiler/rustc_mir_transform/src/ssa.rs b/compiler/rustc_mir_transform/src/ssa.rs index 73c249a3c8cdb..0be26d2e3be52 100644 --- a/compiler/rustc_mir_transform/src/ssa.rs +++ b/compiler/rustc_mir_transform/src/ssa.rs @@ -99,19 +99,23 @@ impl SsaLocals { ssa } + #[inline] pub(super) fn num_locals(&self) -> usize { self.assignments.len() } + #[inline] pub(super) fn locals(&self) -> impl Iterator { self.assignments.indices() } + #[inline] pub(super) fn is_ssa(&self, local: Local) -> bool { matches!(self.assignments[local], Set1::One(_)) } /// Return the number of uses if a local that are not "Deref". + #[inline] pub(super) fn num_direct_uses(&self, local: Local) -> u32 { self.direct_uses[local] } diff --git a/tests/coverage/closure.cov-map b/tests/coverage/closure.cov-map index ee8934f0a846f..608792564245c 100644 --- a/tests/coverage/closure.cov-map +++ b/tests/coverage/closure.cov-map @@ -74,24 +74,22 @@ Number of file 0 mappings: 66 - Code(Counter(0)) at (prev + 1, 1) to (start + 0, 2) Highest counter ID seen: c1 -Function name: closure::main::{closure#0} -Raw bytes (51): 0x[01, 01, 01, 01, 05, 09, 01, 28, 05, 00, 06, 01, 01, 0d, 00, 1a, 01, 00, 1d, 00, 1e, 01, 01, 0c, 00, 14, 05, 00, 15, 02, 0a, 02, 02, 09, 00, 0a, 01, 01, 09, 00, 17, 01, 00, 18, 00, 20, 01, 01, 05, 00, 06] +Function name: closure::main::{closure#0} (unused) +Raw bytes (49): 0x[01, 01, 00, 09, 00, 28, 05, 00, 06, 00, 01, 0d, 00, 1a, 00, 00, 1d, 00, 1e, 00, 01, 0c, 00, 14, 00, 00, 15, 02, 0a, 00, 02, 09, 00, 0a, 00, 01, 09, 00, 17, 00, 00, 18, 00, 20, 00, 01, 05, 00, 06] Number of files: 1 - file 0 => $DIR/closure.rs -Number of expressions: 1 -- expression 0 operands: lhs = Counter(0), rhs = Counter(1) +Number of expressions: 0 Number of file 0 mappings: 9 -- Code(Counter(0)) at (prev + 40, 5) to (start + 0, 6) -- Code(Counter(0)) at (prev + 1, 13) to (start + 0, 26) -- Code(Counter(0)) at (prev + 0, 29) to (start + 0, 30) -- Code(Counter(0)) at (prev + 1, 12) to (start + 0, 20) -- Code(Counter(1)) at (prev + 0, 21) to (start + 2, 10) -- Code(Expression(0, Sub)) at (prev + 2, 9) to (start + 0, 10) - = (c0 - c1) -- Code(Counter(0)) at (prev + 1, 9) to (start + 0, 23) -- Code(Counter(0)) at (prev + 0, 24) to (start + 0, 32) -- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 6) -Highest counter ID seen: c1 +- Code(Zero) at (prev + 40, 5) to (start + 0, 6) +- Code(Zero) at (prev + 1, 13) to (start + 0, 26) +- Code(Zero) at (prev + 0, 29) to (start + 0, 30) +- Code(Zero) at (prev + 1, 12) to (start + 0, 20) +- Code(Zero) at (prev + 0, 21) to (start + 2, 10) +- Code(Zero) at (prev + 2, 9) to (start + 0, 10) +- Code(Zero) at (prev + 1, 9) to (start + 0, 23) +- Code(Zero) at (prev + 0, 24) to (start + 0, 32) +- Code(Zero) at (prev + 1, 5) to (start + 0, 6) +Highest counter ID seen: (none) Function name: closure::main::{closure#10} (unused) Raw bytes (25): 0x[01, 01, 00, 04, 00, 9b, 01, 07, 00, 08, 00, 00, 09, 00, 11, 00, 00, 12, 00, 1e, 00, 00, 20, 00, 21] @@ -199,24 +197,22 @@ Number of file 0 mappings: 7 - Code(Counter(0)) at (prev + 2, 9) to (start + 0, 10) Highest counter ID seen: c1 -Function name: closure::main::{closure#18} -Raw bytes (51): 0x[01, 01, 01, 01, 05, 09, 01, 19, 0d, 00, 0e, 01, 01, 15, 00, 22, 01, 00, 25, 00, 26, 01, 01, 14, 00, 1c, 05, 00, 1d, 02, 12, 02, 02, 11, 00, 12, 01, 01, 11, 00, 1f, 01, 00, 20, 00, 28, 01, 01, 0d, 00, 0e] +Function name: closure::main::{closure#18} (unused) +Raw bytes (49): 0x[01, 01, 00, 09, 00, 19, 0d, 00, 0e, 00, 01, 15, 00, 22, 00, 00, 25, 00, 26, 00, 01, 14, 00, 1c, 00, 00, 1d, 02, 12, 00, 02, 11, 00, 12, 00, 01, 11, 00, 1f, 00, 00, 20, 00, 28, 00, 01, 0d, 00, 0e] Number of files: 1 - file 0 => $DIR/closure.rs -Number of expressions: 1 -- expression 0 operands: lhs = Counter(0), rhs = Counter(1) +Number of expressions: 0 Number of file 0 mappings: 9 -- Code(Counter(0)) at (prev + 25, 13) to (start + 0, 14) -- Code(Counter(0)) at (prev + 1, 21) to (start + 0, 34) -- Code(Counter(0)) at (prev + 0, 37) to (start + 0, 38) -- Code(Counter(0)) at (prev + 1, 20) to (start + 0, 28) -- Code(Counter(1)) at (prev + 0, 29) to (start + 2, 18) -- Code(Expression(0, Sub)) at (prev + 2, 17) to (start + 0, 18) - = (c0 - c1) -- Code(Counter(0)) at (prev + 1, 17) to (start + 0, 31) -- Code(Counter(0)) at (prev + 0, 32) to (start + 0, 40) -- Code(Counter(0)) at (prev + 1, 13) to (start + 0, 14) -Highest counter ID seen: c1 +- Code(Zero) at (prev + 25, 13) to (start + 0, 14) +- Code(Zero) at (prev + 1, 21) to (start + 0, 34) +- Code(Zero) at (prev + 0, 37) to (start + 0, 38) +- Code(Zero) at (prev + 1, 20) to (start + 0, 28) +- Code(Zero) at (prev + 0, 29) to (start + 2, 18) +- Code(Zero) at (prev + 2, 17) to (start + 0, 18) +- Code(Zero) at (prev + 1, 17) to (start + 0, 31) +- Code(Zero) at (prev + 0, 32) to (start + 0, 40) +- Code(Zero) at (prev + 1, 13) to (start + 0, 14) +Highest counter ID seen: (none) Function name: closure::main::{closure#19} Raw bytes (51): 0x[01, 01, 01, 01, 05, 09, 01, 43, 0d, 00, 0e, 01, 01, 15, 00, 22, 01, 00, 25, 00, 26, 01, 01, 14, 00, 1c, 05, 00, 1d, 02, 12, 02, 02, 11, 00, 12, 01, 01, 11, 00, 1f, 01, 00, 20, 00, 28, 01, 01, 0d, 00, 0e] diff --git a/tests/mir-opt/const_prop/boxes.main.GVN.panic-abort.diff b/tests/mir-opt/const_prop/boxes.main.GVN.panic-abort.diff index f43c0cca9ad28..6f33229ce7a27 100644 --- a/tests/mir-opt/const_prop/boxes.main.GVN.panic-abort.diff +++ b/tests/mir-opt/const_prop/boxes.main.GVN.panic-abort.diff @@ -30,16 +30,21 @@ } bb1: { - StorageLive(_7); +- StorageLive(_7); ++ nop; _7 = ShallowInitBox(move _6, i32); _8 = copy ((_7.0: std::ptr::Unique).0: std::ptr::NonNull) as *const i32 (Transmute); (*_8) = const 42_i32; - _3 = move _7; - StorageDead(_7); - _9 = copy ((_3.0: std::ptr::Unique).0: std::ptr::NonNull) as *const i32 (Transmute); - _2 = copy (*_9); +- _3 = move _7; +- StorageDead(_7); +- _9 = copy ((_3.0: std::ptr::Unique).0: std::ptr::NonNull) as *const i32 (Transmute); +- _2 = copy (*_9); - _1 = Add(move _2, const 0_i32); - StorageDead(_2); ++ _3 = copy _7; ++ nop; ++ _9 = copy _8; ++ _2 = copy (*_8); + _1 = copy _2; + nop; drop(_3) -> [return: bb2, unwind unreachable]; diff --git a/tests/mir-opt/const_prop/boxes.main.GVN.panic-unwind.diff b/tests/mir-opt/const_prop/boxes.main.GVN.panic-unwind.diff index 2c903b6d85349..008f3d96962f9 100644 --- a/tests/mir-opt/const_prop/boxes.main.GVN.panic-unwind.diff +++ b/tests/mir-opt/const_prop/boxes.main.GVN.panic-unwind.diff @@ -30,16 +30,21 @@ } bb1: { - StorageLive(_7); +- StorageLive(_7); ++ nop; _7 = ShallowInitBox(move _6, i32); _8 = copy ((_7.0: std::ptr::Unique).0: std::ptr::NonNull) as *const i32 (Transmute); (*_8) = const 42_i32; - _3 = move _7; - StorageDead(_7); - _9 = copy ((_3.0: std::ptr::Unique).0: std::ptr::NonNull) as *const i32 (Transmute); - _2 = copy (*_9); +- _3 = move _7; +- StorageDead(_7); +- _9 = copy ((_3.0: std::ptr::Unique).0: std::ptr::NonNull) as *const i32 (Transmute); +- _2 = copy (*_9); - _1 = Add(move _2, const 0_i32); - StorageDead(_2); ++ _3 = copy _7; ++ nop; ++ _9 = copy _8; ++ _2 = copy (*_8); + _1 = copy _2; + nop; drop(_3) -> [return: bb2, unwind: bb3]; diff --git a/tests/mir-opt/const_prop/discriminant.main.GVN.32bit.diff b/tests/mir-opt/const_prop/discriminant.main.GVN.32bit.diff index 2543cf6257d43..57ef2724feb7a 100644 --- a/tests/mir-opt/const_prop/discriminant.main.GVN.32bit.diff +++ b/tests/mir-opt/const_prop/discriminant.main.GVN.32bit.diff @@ -15,7 +15,8 @@ bb0: { StorageLive(_1); - StorageLive(_2); +- StorageLive(_2); ++ nop; StorageLive(_3); - _3 = Option::::Some(const true); - _4 = discriminant(_3); @@ -41,8 +42,10 @@ } bb4: { - _1 = Add(move _2, const 0_i32); - StorageDead(_2); +- _1 = Add(move _2, const 0_i32); +- StorageDead(_2); ++ _1 = copy _2; ++ nop; StorageDead(_3); _0 = const (); StorageDead(_1); diff --git a/tests/mir-opt/const_prop/discriminant.main.GVN.64bit.diff b/tests/mir-opt/const_prop/discriminant.main.GVN.64bit.diff index 2543cf6257d43..57ef2724feb7a 100644 --- a/tests/mir-opt/const_prop/discriminant.main.GVN.64bit.diff +++ b/tests/mir-opt/const_prop/discriminant.main.GVN.64bit.diff @@ -15,7 +15,8 @@ bb0: { StorageLive(_1); - StorageLive(_2); +- StorageLive(_2); ++ nop; StorageLive(_3); - _3 = Option::::Some(const true); - _4 = discriminant(_3); @@ -41,8 +42,10 @@ } bb4: { - _1 = Add(move _2, const 0_i32); - StorageDead(_2); +- _1 = Add(move _2, const 0_i32); +- StorageDead(_2); ++ _1 = copy _2; ++ nop; StorageDead(_3); _0 = const (); StorageDead(_1); diff --git a/tests/mir-opt/const_prop/discriminant.rs b/tests/mir-opt/const_prop/discriminant.rs index aa169eb4e3699..e9c1e50794343 100644 --- a/tests/mir-opt/const_prop/discriminant.rs +++ b/tests/mir-opt/const_prop/discriminant.rs @@ -20,6 +20,6 @@ fn main() { // CHECK: [[tmp]] = const 10_i32; // CHECK: goto -> bb4; // CHECK: bb4: { - // CHECK: {{_.*}} = Add(move [[tmp]], const 0_i32); + // CHECK: {{_.*}} = copy [[tmp]]; let x = (if let Some(true) = Some(true) { 42 } else { 10 }) + 0; } diff --git a/tests/mir-opt/const_prop/mutable_variable.main.GVN.diff b/tests/mir-opt/const_prop/mutable_variable.main.GVN.diff index cc25b6caf90d9..009b71044bd44 100644 --- a/tests/mir-opt/const_prop/mutable_variable.main.GVN.diff +++ b/tests/mir-opt/const_prop/mutable_variable.main.GVN.diff @@ -17,7 +17,8 @@ _1 = const 42_i32; _1 = const 99_i32; StorageLive(_2); - _2 = copy _1; +- _2 = copy _1; ++ _2 = const 99_i32; _0 = const (); StorageDead(_2); StorageDead(_1); diff --git a/tests/mir-opt/const_prop/mutable_variable.rs b/tests/mir-opt/const_prop/mutable_variable.rs index 3aa1b1bcd2181..e86048e90b990 100644 --- a/tests/mir-opt/const_prop/mutable_variable.rs +++ b/tests/mir-opt/const_prop/mutable_variable.rs @@ -7,7 +7,7 @@ fn main() { // CHECK: debug y => [[y:_.*]]; // CHECK: [[x]] = const 42_i32; // CHECK: [[x]] = const 99_i32; - // CHECK: [[y]] = copy [[x]]; + // CHECK: [[y]] = const 99_i32; let mut x = 42; x = 99; let y = x; diff --git a/tests/mir-opt/const_prop/mutable_variable_no_prop.main.GVN.diff b/tests/mir-opt/const_prop/mutable_variable_no_prop.main.GVN.diff index 3f8dc12f4ae57..5feecdbeff29f 100644 --- a/tests/mir-opt/const_prop/mutable_variable_no_prop.main.GVN.diff +++ b/tests/mir-opt/const_prop/mutable_variable_no_prop.main.GVN.diff @@ -19,17 +19,21 @@ StorageLive(_1); _1 = const 42_u32; StorageLive(_2); - StorageLive(_3); +- StorageLive(_3); ++ nop; StorageLive(_4); _4 = const {ALLOC0: *mut u32}; _3 = copy (*_4); - _1 = move _3; - StorageDead(_3); +- _1 = move _3; +- StorageDead(_3); ++ _1 = copy _3; ++ nop; StorageDead(_4); _2 = const (); StorageDead(_2); StorageLive(_5); - _5 = copy _1; +- _5 = copy _1; ++ _5 = copy _3; _0 = const (); StorageDead(_5); StorageDead(_1); diff --git a/tests/mir-opt/const_prop/mutable_variable_no_prop.rs b/tests/mir-opt/const_prop/mutable_variable_no_prop.rs index 66af5bf1d5d40..8f3ba8ec846da 100644 --- a/tests/mir-opt/const_prop/mutable_variable_no_prop.rs +++ b/tests/mir-opt/const_prop/mutable_variable_no_prop.rs @@ -10,8 +10,8 @@ fn main() { // CHECK: debug y => [[y:_.*]]; // CHECK: [[x]] = const 42_u32; // CHECK: [[tmp:_.*]] = copy (*{{_.*}}); - // CHECK: [[x]] = move [[tmp]]; - // CHECK: [[y]] = copy [[x]]; + // CHECK: [[x]] = copy [[tmp]]; + // CHECK: [[y]] = copy [[tmp]]; let mut x = 42; unsafe { x = STATIC; diff --git a/tests/mir-opt/gvn.fn_pointers.GVN.panic-abort.diff b/tests/mir-opt/gvn.fn_pointers.GVN.panic-abort.diff index f3f631956374d..3cce35d34e90d 100644 --- a/tests/mir-opt/gvn.fn_pointers.GVN.panic-abort.diff +++ b/tests/mir-opt/gvn.fn_pointers.GVN.panic-abort.diff @@ -8,10 +8,10 @@ let mut _3: fn(u8) -> u8; let _5: (); let mut _6: fn(u8) -> u8; - let mut _9: {closure@$DIR/gvn.rs:617:19: 617:21}; + let mut _9: {closure@$DIR/gvn.rs:620:19: 620:21}; let _10: (); let mut _11: fn(); - let mut _13: {closure@$DIR/gvn.rs:617:19: 617:21}; + let mut _13: {closure@$DIR/gvn.rs:620:19: 620:21}; let _14: (); let mut _15: fn(); scope 1 { @@ -19,7 +19,7 @@ let _4: fn(u8) -> u8; scope 2 { debug g => _4; - let _7: {closure@$DIR/gvn.rs:617:19: 617:21}; + let _7: {closure@$DIR/gvn.rs:620:19: 620:21}; scope 3 { debug closure => _7; let _8: fn(); @@ -62,16 +62,16 @@ StorageDead(_6); StorageDead(_5); - StorageLive(_7); -- _7 = {closure@$DIR/gvn.rs:617:19: 617:21}; +- _7 = {closure@$DIR/gvn.rs:620:19: 620:21}; - StorageLive(_8); + nop; -+ _7 = const ZeroSized: {closure@$DIR/gvn.rs:617:19: 617:21}; ++ _7 = const ZeroSized: {closure@$DIR/gvn.rs:620:19: 620:21}; + nop; StorageLive(_9); - _9 = copy _7; - _8 = move _9 as fn() (PointerCoercion(ClosureFnPointer(Safe), AsCast)); -+ _9 = const ZeroSized: {closure@$DIR/gvn.rs:617:19: 617:21}; -+ _8 = const ZeroSized: {closure@$DIR/gvn.rs:617:19: 617:21} as fn() (PointerCoercion(ClosureFnPointer(Safe), AsCast)); ++ _9 = const ZeroSized: {closure@$DIR/gvn.rs:620:19: 620:21}; ++ _8 = const ZeroSized: {closure@$DIR/gvn.rs:620:19: 620:21} as fn() (PointerCoercion(ClosureFnPointer(Safe), AsCast)); StorageDead(_9); StorageLive(_10); StorageLive(_11); @@ -88,8 +88,8 @@ StorageLive(_13); - _13 = copy _7; - _12 = move _13 as fn() (PointerCoercion(ClosureFnPointer(Safe), AsCast)); -+ _13 = const ZeroSized: {closure@$DIR/gvn.rs:617:19: 617:21}; -+ _12 = const ZeroSized: {closure@$DIR/gvn.rs:617:19: 617:21} as fn() (PointerCoercion(ClosureFnPointer(Safe), AsCast)); ++ _13 = const ZeroSized: {closure@$DIR/gvn.rs:620:19: 620:21}; ++ _12 = const ZeroSized: {closure@$DIR/gvn.rs:620:19: 620:21} as fn() (PointerCoercion(ClosureFnPointer(Safe), AsCast)); StorageDead(_13); StorageLive(_14); StorageLive(_15); diff --git a/tests/mir-opt/gvn.fn_pointers.GVN.panic-unwind.diff b/tests/mir-opt/gvn.fn_pointers.GVN.panic-unwind.diff index 029e736a97952..d85aca040fe67 100644 --- a/tests/mir-opt/gvn.fn_pointers.GVN.panic-unwind.diff +++ b/tests/mir-opt/gvn.fn_pointers.GVN.panic-unwind.diff @@ -8,10 +8,10 @@ let mut _3: fn(u8) -> u8; let _5: (); let mut _6: fn(u8) -> u8; - let mut _9: {closure@$DIR/gvn.rs:617:19: 617:21}; + let mut _9: {closure@$DIR/gvn.rs:620:19: 620:21}; let _10: (); let mut _11: fn(); - let mut _13: {closure@$DIR/gvn.rs:617:19: 617:21}; + let mut _13: {closure@$DIR/gvn.rs:620:19: 620:21}; let _14: (); let mut _15: fn(); scope 1 { @@ -19,7 +19,7 @@ let _4: fn(u8) -> u8; scope 2 { debug g => _4; - let _7: {closure@$DIR/gvn.rs:617:19: 617:21}; + let _7: {closure@$DIR/gvn.rs:620:19: 620:21}; scope 3 { debug closure => _7; let _8: fn(); @@ -62,16 +62,16 @@ StorageDead(_6); StorageDead(_5); - StorageLive(_7); -- _7 = {closure@$DIR/gvn.rs:617:19: 617:21}; +- _7 = {closure@$DIR/gvn.rs:620:19: 620:21}; - StorageLive(_8); + nop; -+ _7 = const ZeroSized: {closure@$DIR/gvn.rs:617:19: 617:21}; ++ _7 = const ZeroSized: {closure@$DIR/gvn.rs:620:19: 620:21}; + nop; StorageLive(_9); - _9 = copy _7; - _8 = move _9 as fn() (PointerCoercion(ClosureFnPointer(Safe), AsCast)); -+ _9 = const ZeroSized: {closure@$DIR/gvn.rs:617:19: 617:21}; -+ _8 = const ZeroSized: {closure@$DIR/gvn.rs:617:19: 617:21} as fn() (PointerCoercion(ClosureFnPointer(Safe), AsCast)); ++ _9 = const ZeroSized: {closure@$DIR/gvn.rs:620:19: 620:21}; ++ _8 = const ZeroSized: {closure@$DIR/gvn.rs:620:19: 620:21} as fn() (PointerCoercion(ClosureFnPointer(Safe), AsCast)); StorageDead(_9); StorageLive(_10); StorageLive(_11); @@ -88,8 +88,8 @@ StorageLive(_13); - _13 = copy _7; - _12 = move _13 as fn() (PointerCoercion(ClosureFnPointer(Safe), AsCast)); -+ _13 = const ZeroSized: {closure@$DIR/gvn.rs:617:19: 617:21}; -+ _12 = const ZeroSized: {closure@$DIR/gvn.rs:617:19: 617:21} as fn() (PointerCoercion(ClosureFnPointer(Safe), AsCast)); ++ _13 = const ZeroSized: {closure@$DIR/gvn.rs:620:19: 620:21}; ++ _12 = const ZeroSized: {closure@$DIR/gvn.rs:620:19: 620:21} as fn() (PointerCoercion(ClosureFnPointer(Safe), AsCast)); StorageDead(_13); StorageLive(_14); StorageLive(_15); diff --git a/tests/mir-opt/gvn.rs b/tests/mir-opt/gvn.rs index 3c3241fefe22e..eb60e996adfcd 100644 --- a/tests/mir-opt/gvn.rs +++ b/tests/mir-opt/gvn.rs @@ -99,12 +99,14 @@ fn subexpression_elimination(x: u64, y: u64, mut z: u64) { opaque((x * y) - y); opaque((x * y) - y); - // We cannot substitute through an immutable reference. + // We can substitute through an immutable reference. + // But we prefer using a SSA local rather than a borrowed one. // CHECK: [[ref:_.*]] = &_3; - // CHECK: [[deref:_.*]] = copy (*[[ref]]); + // CHECK: [[deref:_.*]] = copy _3; // CHECK: [[addref:_.*]] = Add(move [[deref]], copy _1); // CHECK: opaque::(move [[addref]]) - // CHECK: [[deref2:_.*]] = copy (*[[ref]]); + // But `_3` is not SSA so we cannot merge the values in two different blocks. + // CHECK: [[deref2:_.*]] = copy _3; // CHECK: [[addref2:_.*]] = Add(move [[deref2]], copy _1); // CHECK: opaque::(move [[addref2]]) let a = &z; @@ -141,13 +143,14 @@ fn subexpression_elimination(x: u64, y: u64, mut z: u64) { // We still cannot substitute again, and never with the earlier computations. // Important: `e` is not `a`! - // CHECK: [[ref2:_.*]] = &_3; - // CHECK: [[deref2:_.*]] = copy (*[[ref2]]); - // CHECK: [[addref2:_.*]] = Add(move [[deref2]], copy _1); - // CHECK: opaque::(move [[addref2]]) - // CHECK: [[deref3:_.*]] = copy (*[[ref2]]); + // CHECK: [[ref3:_.*]] = &_3; + // CHECK: [[deref3:_.*]] = copy _3; // CHECK: [[addref3:_.*]] = Add(move [[deref3]], copy _1); // CHECK: opaque::(move [[addref3]]) + // And `_3` is not SSA so we cannot merge the values in two different blocks. + // CHECK: [[deref4:_.*]] = copy _3; + // CHECK: [[addref4:_.*]] = Add(move [[deref4]], copy _1); + // CHECK: opaque::(move [[addref4]]) let e = &z; opaque(*e + x); opaque(*e + x); diff --git a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff index e872e011542b3..0a17eba07087b 100644 --- a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff +++ b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff @@ -760,7 +760,8 @@ StorageLive(_127); StorageLive(_128); StorageLive(_129); - _129 = copy (*_126); +- _129 = copy (*_126); ++ _129 = copy _3; StorageLive(_130); _130 = copy _1; - _128 = Add(move _129, move _130); @@ -776,7 +777,8 @@ StorageLive(_131); StorageLive(_132); StorageLive(_133); - _133 = copy (*_126); +- _133 = copy (*_126); ++ _133 = copy _3; StorageLive(_134); _134 = copy _1; - _132 = Add(move _133, move _134); @@ -901,7 +903,8 @@ StorageLive(_164); StorageLive(_165); StorageLive(_166); - _166 = copy (*_163); +- _166 = copy (*_163); ++ _166 = copy _3; StorageLive(_167); _167 = copy _1; - _165 = Add(move _166, move _167); @@ -917,7 +920,8 @@ StorageLive(_168); StorageLive(_169); StorageLive(_170); - _170 = copy (*_163); +- _170 = copy (*_163); ++ _170 = copy _3; StorageLive(_171); _171 = copy _1; - _169 = Add(move _170, move _171); diff --git a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff index 3996dab27a343..d12a8aaca419d 100644 --- a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff +++ b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff @@ -760,7 +760,8 @@ StorageLive(_127); StorageLive(_128); StorageLive(_129); - _129 = copy (*_126); +- _129 = copy (*_126); ++ _129 = copy _3; StorageLive(_130); _130 = copy _1; - _128 = Add(move _129, move _130); @@ -776,7 +777,8 @@ StorageLive(_131); StorageLive(_132); StorageLive(_133); - _133 = copy (*_126); +- _133 = copy (*_126); ++ _133 = copy _3; StorageLive(_134); _134 = copy _1; - _132 = Add(move _133, move _134); @@ -901,7 +903,8 @@ StorageLive(_164); StorageLive(_165); StorageLive(_166); - _166 = copy (*_163); +- _166 = copy (*_163); ++ _166 = copy _3; StorageLive(_167); _167 = copy _1; - _165 = Add(move _166, move _167); @@ -917,7 +920,8 @@ StorageLive(_168); StorageLive(_169); StorageLive(_170); - _170 = copy (*_163); +- _170 = copy (*_163); ++ _170 = copy _3; StorageLive(_171); _171 = copy _1; - _169 = Add(move _170, move _171); diff --git a/tests/mir-opt/gvn_copy_aggregate.all_copy_use_changed.GVN.diff b/tests/mir-opt/gvn_copy_aggregate.all_copy_use_changed.GVN.diff index dc65cccb7bd6e..4368fa5b93d50 100644 --- a/tests/mir-opt/gvn_copy_aggregate.all_copy_use_changed.GVN.diff +++ b/tests/mir-opt/gvn_copy_aggregate.all_copy_use_changed.GVN.diff @@ -25,24 +25,28 @@ StorageLive(_2); _2 = copy ((*_1).0: i32); ((*_1).0: i32) = const 1_i32; - StorageLive(_3); +- StorageLive(_3); ++ nop; _3 = copy ((*_1).0: i32); - _2 = move _3; - StorageDead(_3); +- _2 = move _3; +- StorageDead(_3); - StorageLive(_4); ++ _2 = copy _3; ++ nop; + nop; _4 = copy ((*_1).1: u64); - StorageLive(_5); + nop; _5 = copy ((*_1).2: [i8; 3]); StorageLive(_6); - _6 = copy _2; +- _6 = copy _2; ++ _6 = copy _3; StorageLive(_7); _7 = copy _4; StorageLive(_8); _8 = copy _5; - _0 = AllCopy { a: move _6, b: move _7, c: move _8 }; -+ _0 = AllCopy { a: move _6, b: copy _4, c: copy _5 }; ++ _0 = AllCopy { a: copy _3, b: copy _4, c: copy _5 }; StorageDead(_8); StorageDead(_7); StorageDead(_6); diff --git a/tests/mir-opt/gvn_copy_aggregate.all_copy_use_changed_2.GVN.diff b/tests/mir-opt/gvn_copy_aggregate.all_copy_use_changed_2.GVN.diff index 08a4a078adcb4..4b7c2b7d7fc42 100644 --- a/tests/mir-opt/gvn_copy_aggregate.all_copy_use_changed_2.GVN.diff +++ b/tests/mir-opt/gvn_copy_aggregate.all_copy_use_changed_2.GVN.diff @@ -31,18 +31,22 @@ + nop; _4 = copy ((*_1).2: [i8; 3]); ((*_1).0: i32) = const 1_i32; - StorageLive(_5); +- StorageLive(_5); ++ nop; _5 = copy ((*_1).0: i32); - _2 = move _5; - StorageDead(_5); +- _2 = move _5; +- StorageDead(_5); ++ _2 = copy _5; ++ nop; StorageLive(_6); - _6 = copy _2; +- _6 = copy _2; ++ _6 = copy _5; StorageLive(_7); _7 = copy _3; StorageLive(_8); _8 = copy _4; - _0 = AllCopy { a: move _6, b: move _7, c: move _8 }; -+ _0 = AllCopy { a: move _6, b: copy _3, c: copy _4 }; ++ _0 = AllCopy { a: copy _5, b: copy _3, c: copy _4 }; StorageDead(_8); StorageDead(_7); StorageDead(_6); diff --git a/tests/mir-opt/gvn_copy_moves.fn0.GVN.diff b/tests/mir-opt/gvn_copy_moves.fn0.GVN.diff index c364da7a3cc68..c882f617f3836 100644 --- a/tests/mir-opt/gvn_copy_moves.fn0.GVN.diff +++ b/tests/mir-opt/gvn_copy_moves.fn0.GVN.diff @@ -17,7 +17,7 @@ _3 = (copy _2,); _4 = copy _3; - _5 = fn1(move (_3.0: [u128; 6]), copy _4) -> [return: bb1, unwind unreachable]; -+ _5 = fn1(copy (_3.0: [u128; 6]), copy _3) -> [return: bb1, unwind unreachable]; ++ _5 = fn1(copy _2, copy _3) -> [return: bb1, unwind unreachable]; } bb1: { diff --git a/tests/mir-opt/gvn_copy_moves.fn2.GVN.diff b/tests/mir-opt/gvn_copy_moves.fn2.GVN.diff new file mode 100644 index 0000000000000..37b471418fbb2 --- /dev/null +++ b/tests/mir-opt/gvn_copy_moves.fn2.GVN.diff @@ -0,0 +1,27 @@ +- // MIR for `fn2` before GVN ++ // MIR for `fn2` after GVN + + fn fn2() -> () { + let mut _0: (); + let mut _1: usize; + let mut _2: [u128; 6]; + let mut _3: ([u128; 6],); + let mut _4: ([u128; 6],); + let mut _5: (); + + bb0: { + _1 = const 1_usize; + _2 = [const 42_u128; 6]; + _3 = (copy _2,); +- _2[_1] = const 1_u128; ++ _2[1 of 2] = const 1_u128; + _4 = copy _3; +- _5 = fn1(move (_3.0: [u128; 6]), copy _4) -> [return: bb1, unwind unreachable]; ++ _5 = fn1(copy (_3.0: [u128; 6]), copy _3) -> [return: bb1, unwind unreachable]; + } + + bb1: { + return; + } + } + diff --git a/tests/mir-opt/gvn_copy_moves.rs b/tests/mir-opt/gvn_copy_moves.rs index b1fe2caf3a143..c72c1709f69c4 100644 --- a/tests/mir-opt/gvn_copy_moves.rs +++ b/tests/mir-opt/gvn_copy_moves.rs @@ -20,7 +20,7 @@ fn fn0() { // CHECK-NEXT: _2[1 of 2] = const 1_u128; // CHECK-NEXT: _3 = (copy _2,); // CHECK-NEXT: _4 = copy _3; - // CHECK-NEXT: _5 = fn1(copy (_3.0: [u128; 6]), copy _3) + // CHECK-NEXT: _5 = fn1(copy _2, copy _3) a = 1_usize; b = [42; 6]; b[a] = 1; @@ -39,8 +39,40 @@ fn fn1(a: [u128; 6], mut b: ([u128; 6],)) { b.0 = [0; 6]; } +// Like `fn0` but we assign to `b` after copying the value to `c`. +#[custom_mir(dialect = "runtime", phase = "initial")] +fn fn2() { + // CHECK-LABEL: fn fn2( + mir! { + let a: usize; + let b: [u128; 6]; + let c: ([u128; 6],); + let d: ([u128; 6],); + let x: (); + { + // CHECK: bb0: { + // CHECK-NEXT: _1 = const 1_usize; + // CHECK-NEXT: _2 = [const 42_u128; 6]; + // CHECK-NEXT: _3 = (copy _2,); + // CHECK-NEXT: _2[1 of 2] = const 1_u128; + // CHECK-NEXT: _4 = copy _3; + // CHECK-NEXT: _5 = fn1(copy (_3.0: [u128; 6]), copy _3) + a = 1_usize; + b = [42; 6]; + c = (b,); + b[a] = 1; + d = c; + Call(x = fn1(Move(c.0), d), ReturnTo(bb1), UnwindUnreachable()) + } + bb1 = { + Return() + } + } +} + fn main() { fn0(); } // EMIT_MIR gvn_copy_moves.fn0.GVN.diff +// EMIT_MIR gvn_copy_moves.fn2.GVN.diff diff --git a/tests/mir-opt/gvn_repeat.repeat_local.GVN.diff b/tests/mir-opt/gvn_repeat.repeat_local.GVN.diff index eb3f885b8665b..fd04782528117 100644 --- a/tests/mir-opt/gvn_repeat.repeat_local.GVN.diff +++ b/tests/mir-opt/gvn_repeat.repeat_local.GVN.diff @@ -10,7 +10,8 @@ _4 = [copy _3; 5]; _5 = &_4[_1]; _1 = copy _2; - _0 = copy (*_5); +- _0 = copy (*_5); ++ _0 = copy _3; return; } } diff --git a/tests/mir-opt/gvn_repeat.repeat_place.GVN.diff b/tests/mir-opt/gvn_repeat.repeat_place.GVN.diff index e490925bc119e..4cb7d3d9aee43 100644 --- a/tests/mir-opt/gvn_repeat.repeat_place.GVN.diff +++ b/tests/mir-opt/gvn_repeat.repeat_place.GVN.diff @@ -10,7 +10,8 @@ _4 = [copy (*_3); 5]; _5 = &_4[_1]; _1 = copy _2; - _0 = copy (*_5); +- _0 = copy (*_5); ++ _0 = copy (*_3); return; } } diff --git a/tests/mir-opt/gvn_repeat.rs b/tests/mir-opt/gvn_repeat.rs index 49364c6bfd232..730378d825983 100644 --- a/tests/mir-opt/gvn_repeat.rs +++ b/tests/mir-opt/gvn_repeat.rs @@ -27,8 +27,7 @@ pub fn index_place(mut idx1: usize, idx2: usize, array: [i32; 5]) -> i32 { #[custom_mir(dialect = "runtime")] pub fn repeat_place(mut idx1: usize, idx2: usize, val: &i32) -> i32 { // CHECK-LABEL: fn repeat_place( - // CHECK: let mut [[ELEM:.*]]: &i32; - // CHECK: _0 = copy (*[[ELEM]]) + // CHECK: _0 = copy (*_3) mir! { let array; let elem; @@ -46,8 +45,7 @@ pub fn repeat_place(mut idx1: usize, idx2: usize, val: &i32) -> i32 { #[custom_mir(dialect = "runtime")] pub fn repeat_local(mut idx1: usize, idx2: usize, val: i32) -> i32 { // CHECK-LABEL: fn repeat_local( - // CHECK: let mut [[ELEM:.*]]: &i32; - // CHECK: _0 = copy (*[[ELEM]]); + // CHECK: _0 = copy _3; mir! { let array; let elem; diff --git a/tests/mir-opt/pre-codegen/loops.vec_move.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/loops.vec_move.PreCodegen.after.mir index 66eb1bcfaa665..87c9af72d3b58 100644 --- a/tests/mir-opt/pre-codegen/loops.vec_move.PreCodegen.after.mir +++ b/tests/mir-opt/pre-codegen/loops.vec_move.PreCodegen.after.mir @@ -184,7 +184,7 @@ fn vec_move(_1: Vec) -> () { StorageDead(_3); // DBG: _32 = &_2; // DBG: _31 = &((_2.0: std::vec::Vec).0: alloc::raw_vec::RawVec); - _4 = copy (((((_2.0: std::vec::Vec).0: alloc::raw_vec::RawVec).0: alloc::raw_vec::RawVecInner).0: std::ptr::Unique).0: std::ptr::NonNull); + _4 = copy ((((_1.0: alloc::raw_vec::RawVec).0: alloc::raw_vec::RawVecInner).0: std::ptr::Unique).0: std::ptr::NonNull); _5 = copy _4 as *const impl Sized (Transmute); _6 = NonNull:: { pointer: copy _5 }; _7 = copy _4 as *mut impl Sized (Transmute); diff --git a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir index 104987b0fdda9..51f5becd4696c 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir @@ -169,8 +169,6 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } bb4: { - StorageLive(_28); - StorageLive(_25); StorageLive(_26); StorageLive(_23); StorageLive(_11); @@ -238,8 +236,6 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { bb10: { StorageDead(_23); StorageDead(_26); - StorageDead(_25); - StorageDead(_28); StorageDead(_10); drop(_2) -> [return: bb11, unwind unreachable]; } @@ -269,8 +265,8 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { _24 = copy ((_23 as Some).0: &T); StorageDead(_23); _25 = copy _10; - _26 = AddWithOverflow(copy _10, const 1_usize); - assert(!move (_26.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _10, const 1_usize) -> [success: bb14, unwind unreachable]; + _26 = AddWithOverflow(copy _25, const 1_usize); + assert(!move (_26.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _25, const 1_usize) -> [success: bb14, unwind unreachable]; } bb14: { @@ -280,20 +276,18 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { _28 = Option::<(usize, &T)>::Some(move _27); StorageDead(_27); StorageDead(_26); - StorageDead(_25); _29 = copy (((_28 as Some).0: (usize, &T)).0: usize); _30 = copy (((_28 as Some).0: (usize, &T)).1: &T); StorageLive(_31); _31 = &_2; StorageLive(_32); - _32 = (copy _29, copy _30); + _32 = copy ((_28 as Some).0: (usize, &T)); _33 = >::call(move _31, move _32) -> [return: bb15, unwind unreachable]; } bb15: { StorageDead(_32); StorageDead(_31); - StorageDead(_28); goto -> bb4; } }