Make Value Copy by arena-allocating vectors.

Author: cjgillot

compiler/rustc_mir_transform/src/gvn.rs

@@ -88,6 +88,7 @@ use std::borrow::Cow;
 
 use either::Either;
 use rustc_abi::{self as abi, BackendRepr, FIRST_VARIANT, FieldIdx, Primitive, Size, VariantIdx};
+use rustc_arena::DroplessArena;
 use rustc_const_eval::const_eval::DummyMachine;
 use rustc_const_eval::interpret::{
     ImmTy, Immediate, InterpCx, MemPlaceMeta, MemoryKind, OpTy, Projectable, Scalar,
@@ -126,7 +127,9 @@ impl<'tcx> crate::MirPass<'tcx> for GVN {
         // Clone dominators because we need them while mutating the body.
         let dominators = body.basic_blocks.dominators().clone();
 
-        let mut state = VnState::new(tcx, body, typing_env, &ssa, dominators, &body.local_decls);
+        let arena = DroplessArena::default();
+        let mut state =
+            VnState::new(tcx, body, typing_env, &ssa, dominators, &body.local_decls, &arena);
 
         for local in body.args_iter().filter(|&local| ssa.is_ssa(local)) {
             let opaque = state.new_opaque(body.local_decls[local].ty);
@@ -160,8 +163,8 @@ enum AddressKind {
     Address(RawPtrKind),
 }
 
-#[derive(Debug, PartialEq, Eq, Hash)]
-enum Value<'tcx> {
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+enum Value<'a, 'tcx> {
     // Root values.
     /// Used to represent values we know nothing about.
     /// The `usize` is a counter incremented by `new_opaque`.
@@ -176,7 +179,7 @@ enum Value<'tcx> {
     },
     /// An aggregate value, either tuple/closure/struct/enum.
     /// This does not contain unions, as we cannot reason with the value.
-    Aggregate(VariantIdx, Vec<VnIndex>),
+    Aggregate(VariantIdx, &'a [VnIndex]),
     /// A raw pointer aggregate built from a thin pointer and metadata.
     RawPtr {
         /// Thin pointer component. This is field 0 in MIR.
@@ -212,7 +215,7 @@ enum Value<'tcx> {
     },
 }
 
-struct VnState<'body, 'tcx> {
+struct VnState<'body, 'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     ecx: InterpCx<'tcx, DummyMachine>,
     local_decls: &'body LocalDecls<'tcx>,
@@ -222,7 +225,7 @@ struct VnState<'body, 'tcx> {
     /// Locals that are assigned that value.
     // This vector does not hold all the values of `VnIndex` that we create.
     rev_locals: IndexVec<VnIndex, SmallVec<[Local; 1]>>,
-    values: FxIndexSet<(Value<'tcx>, Ty<'tcx>)>,
+    values: FxIndexSet<(Value<'a, 'tcx>, Ty<'tcx>)>,
     /// Values evaluated as constants if possible.
     evaluated: IndexVec<VnIndex, Option<OpTy<'tcx>>>,
     /// Counter to generate different values.
@@ -232,16 +235,18 @@ struct VnState<'body, 'tcx> {
     ssa: &'body SsaLocals,
     dominators: Dominators<BasicBlock>,
     reused_locals: DenseBitSet<Local>,
+    arena: &'a DroplessArena,
 }
 
-impl<'body, 'tcx> VnState<'body, 'tcx> {
+impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
     fn new(
         tcx: TyCtxt<'tcx>,
         body: &Body<'tcx>,
         typing_env: ty::TypingEnv<'tcx>,
         ssa: &'body SsaLocals,
         dominators: Dominators<BasicBlock>,
         local_decls: &'body LocalDecls<'tcx>,
+        arena: &'a DroplessArena,
     ) -> Self {
         // Compute a rough estimate of the number of values in the body from the number of
         // statements. This is meant to reduce the number of allocations, but it's all right if
@@ -264,6 +269,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             ssa,
             dominators,
             reused_locals: DenseBitSet::new_empty(local_decls.len()),
+            arena,
         }
     }
 
@@ -272,7 +278,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
     }
 
     #[instrument(level = "trace", skip(self), ret)]
-    fn insert(&mut self, ty: Ty<'tcx>, value: Value<'tcx>) -> VnIndex {
+    fn insert(&mut self, ty: Ty<'tcx>, value: Value<'a, 'tcx>) -> VnIndex {
         let (index, new) = self.values.insert_full((value, ty));
         let index = VnIndex::from_usize(index);
         if new {
@@ -315,8 +321,8 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
     }
 
     #[inline]
-    fn get(&self, index: VnIndex) -> &Value<'tcx> {
-        &self.values.get_index(index.as_usize()).unwrap().0
+    fn get(&self, index: VnIndex) -> Value<'a, 'tcx> {
+        self.values.get_index(index.as_usize()).unwrap().0
     }
 
     #[inline]
@@ -361,8 +367,8 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         self.insert(ty, Value::Constant { value, disambiguator: 0 })
     }
 
-    fn insert_tuple(&mut self, ty: Ty<'tcx>, values: Vec<VnIndex>) -> VnIndex {
-        self.insert(ty, Value::Aggregate(VariantIdx::ZERO, values))
+    fn insert_tuple(&mut self, ty: Ty<'tcx>, values: &[VnIndex]) -> VnIndex {
+        self.insert(ty, Value::Aggregate(VariantIdx::ZERO, self.arena.alloc_slice(values)))
     }
 
     fn insert_deref(&mut self, ty: Ty<'tcx>, value: VnIndex) -> VnIndex {
@@ -388,7 +394,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         } else {
             return None;
         };
-        let op = match *self.get(value) {
+        let op = match self.get(value) {
             _ if ty.is_zst() => ImmTy::uninit(ty).into(),
 
             Opaque(_) => return None,
@@ -608,7 +614,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                 if let Value::Aggregate(_, fields) = self.get(value) {
                     return Some((projection_ty, fields[f.as_usize()]));
                 } else if let Value::Projection(outer_value, ProjectionElem::Downcast(_, read_variant)) = self.get(value)
-                    && let Value::Aggregate(written_variant, fields) = self.get(*outer_value)
+                    && let Value::Aggregate(written_variant, fields) = self.get(outer_value)
                     // This pass is not aware of control-flow, so we do not know whether the
                     // replacement we are doing is actually reachable. We could be in any arm of
                     // ```
@@ -633,15 +639,15 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             ProjectionElem::Index(idx) => {
                 if let Value::Repeat(inner, _) = self.get(value) {
                     *from_non_ssa_index |= self.locals[idx].is_none();
-                    return Some((projection_ty, *inner));
+                    return Some((projection_ty, inner));
                 }
                 let idx = self.locals[idx]?;
                 ProjectionElem::Index(idx)
             }
             ProjectionElem::ConstantIndex { offset, min_length, from_end } => {
                 match self.get(value) {
                     Value::Repeat(inner, _) => {
-                        return Some((projection_ty, *inner));
+                        return Some((projection_ty, inner));
                     }
                     Value::Aggregate(_, operands) => {
                         let offset = if from_end {
@@ -731,8 +737,8 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         let mut place_ty = PlaceTy::from_ty(self.local_decls[place.local].ty);
         let mut from_non_ssa_index = false;
         for (index, proj) in place.projection.iter().enumerate() {
-            if let Value::Projection(pointer, ProjectionElem::Deref) = *self.get(value)
-                && let Value::Address { place: mut pointee, kind, .. } = *self.get(pointer)
+            if let Value::Projection(pointer, ProjectionElem::Deref) = self.get(value)
+                && let Value::Address { place: mut pointee, kind, .. } = self.get(pointer)
                 && let AddressKind::Ref(BorrowKind::Shared) = kind
                 && let Some(v) = self.simplify_place_value(&mut pointee, location)
             {
@@ -755,8 +761,8 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             (place_ty, value) = self.project(place_ty, value, proj, &mut from_non_ssa_index)?;
         }
 
-        if let Value::Projection(pointer, ProjectionElem::Deref) = *self.get(value)
-            && let Value::Address { place: mut pointee, kind, .. } = *self.get(pointer)
+        if let Value::Projection(pointer, ProjectionElem::Deref) = self.get(value)
+            && let Value::Address { place: mut pointee, kind, .. } = self.get(pointer)
             && let AddressKind::Ref(BorrowKind::Shared) = kind
             && let Some(v) = self.simplify_place_value(&mut pointee, location)
         {
@@ -868,7 +874,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
     fn simplify_discriminant(&mut self, place: VnIndex) -> Option<VnIndex> {
         let enum_ty = self.ty(place);
         if enum_ty.is_enum()
-            && let Value::Aggregate(variant, _) = *self.get(place)
+            && let Value::Aggregate(variant, _) = self.get(place)
         {
             let discr = self.ecx.discriminant_for_variant(enum_ty, variant).discard_err()?;
             return Some(self.insert_scalar(discr.layout.ty, discr.to_scalar()));
@@ -904,12 +910,12 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         let Some(&first_field) = fields.first() else {
             return None;
         };
-        let Value::Projection(copy_from_value, _) = *self.get(first_field) else {
+        let Value::Projection(copy_from_value, _) = self.get(first_field) else {
             return None;
         };
         // All fields must correspond one-to-one and come from the same aggregate value.
         if fields.iter().enumerate().any(|(index, &v)| {
-            if let Value::Projection(pointer, ProjectionElem::Field(from_index, _)) = *self.get(v)
+            if let Value::Projection(pointer, ProjectionElem::Field(from_index, _)) = self.get(v)
                 && copy_from_value == pointer
                 && from_index.index() == index
             {
@@ -921,7 +927,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         }
 
         let mut copy_from_local_value = copy_from_value;
-        if let Value::Projection(pointer, proj) = *self.get(copy_from_value)
+        if let Value::Projection(pointer, proj) = self.get(copy_from_value)
             && let ProjectionElem::Downcast(_, read_variant) = proj
         {
             if variant_index == read_variant {
@@ -979,13 +985,10 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             }
         }
 
-        let fields: Vec<_> = field_ops
-            .iter_mut()
-            .map(|op| {
-                self.simplify_operand(op, location)
-                    .unwrap_or_else(|| self.new_opaque(op.ty(self.local_decls, self.tcx)))
-            })
-            .collect();
+        let fields = self.arena.alloc_from_iter(field_ops.iter_mut().map(|op| {
+            self.simplify_operand(op, location)
+                .unwrap_or_else(|| self.new_opaque(op.ty(self.local_decls, self.tcx)))
+        }));
 
         let variant_index = match *kind {
             AggregateKind::Array(..) | AggregateKind::Tuple => {
@@ -1006,12 +1009,12 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                 let mut was_updated = false;
                 while let Value::Cast { kind: CastKind::PtrToPtr, value: cast_value } =
                     self.get(pointer)
-                    && let ty::RawPtr(from_pointee_ty, from_mtbl) = self.ty(*cast_value).kind()
+                    && let ty::RawPtr(from_pointee_ty, from_mtbl) = self.ty(cast_value).kind()
                     && let ty::RawPtr(_, output_mtbl) = ty.kind()
                     && from_mtbl == output_mtbl
                     && from_pointee_ty.is_sized(self.tcx, self.typing_env())
                 {
-                    pointer = *cast_value;
+                    pointer = cast_value;
                     was_updated = true;
                 }
 
@@ -1069,9 +1072,9 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                     // To allow things like `*mut (?A, ?T)` <-> `*mut (?B, ?T)`,
                     // it's fine to get a projection as the type.
                     Value::Cast { kind: CastKind::PtrToPtr, value: inner }
-                        if self.pointers_have_same_metadata(self.ty(*inner), arg_ty) =>
+                        if self.pointers_have_same_metadata(self.ty(inner), arg_ty) =>
                     {
-                        arg_index = *inner;
+                        arg_index = inner;
                         was_updated = true;
                         continue;
                     }
@@ -1098,15 +1101,15 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         }
 
         let value = match (op, self.get(arg_index)) {
-            (UnOp::Not, Value::UnaryOp(UnOp::Not, inner)) => return Some(*inner),
-            (UnOp::Neg, Value::UnaryOp(UnOp::Neg, inner)) => return Some(*inner),
+            (UnOp::Not, Value::UnaryOp(UnOp::Not, inner)) => return Some(inner),
+            (UnOp::Neg, Value::UnaryOp(UnOp::Neg, inner)) => return Some(inner),
             (UnOp::Not, Value::BinaryOp(BinOp::Eq, lhs, rhs)) => {
-                Value::BinaryOp(BinOp::Ne, *lhs, *rhs)
+                Value::BinaryOp(BinOp::Ne, lhs, rhs)
             }
             (UnOp::Not, Value::BinaryOp(BinOp::Ne, lhs, rhs)) => {
-                Value::BinaryOp(BinOp::Eq, *lhs, *rhs)
+                Value::BinaryOp(BinOp::Eq, lhs, rhs)
             }
-            (UnOp::PtrMetadata, Value::RawPtr { metadata, .. }) => return Some(*metadata),
+            (UnOp::PtrMetadata, Value::RawPtr { metadata, .. }) => return Some(metadata),
             // We have an unsizing cast, which assigns the length to wide pointer metadata.
             (
                 UnOp::PtrMetadata,
@@ -1115,7 +1118,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                     value: inner,
                 },
             ) if let ty::Slice(..) = arg_ty.builtin_deref(true).unwrap().kind()
-                && let ty::Array(_, len) = self.ty(*inner).builtin_deref(true).unwrap().kind() =>
+                && let ty::Array(_, len) = self.ty(inner).builtin_deref(true).unwrap().kind() =>
             {
                 return Some(self.insert_constant(Const::Ty(self.tcx.types.usize, *len)));
             }
@@ -1148,12 +1151,12 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             && lhs_ty.is_any_ptr()
             && let Value::Cast { kind: CastKind::PtrToPtr, value: lhs_value } = self.get(lhs)
             && let Value::Cast { kind: CastKind::PtrToPtr, value: rhs_value } = self.get(rhs)
-            && let lhs_from = self.ty(*lhs_value)
-            && lhs_from == self.ty(*rhs_value)
+            && let lhs_from = self.ty(lhs_value)
+            && lhs_from == self.ty(rhs_value)
             && self.pointers_have_same_metadata(lhs_from, lhs_ty)
         {
-            lhs = *lhs_value;
-            rhs = *rhs_value;
+            lhs = lhs_value;
+            rhs = rhs_value;
             if let Some(lhs_op) = self.try_as_operand(lhs, location)
                 && let Some(rhs_op) = self.try_as_operand(rhs, location)
             {
@@ -1287,7 +1290,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         if op.is_overflowing() {
             let ty = Ty::new_tup(self.tcx, &[self.ty(result), self.tcx.types.bool]);
             let false_val = self.insert_bool(false);
-            Some(self.insert_tuple(ty, vec![result, false_val]))
+            Some(self.insert_tuple(ty, &[result, false_val]))
         } else {
             Some(result)
         }
@@ -1340,11 +1343,11 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                 && let ty::RawPtr(to_pointee, _) = to.kind()
                 && to_pointee.is_sized(self.tcx, self.typing_env())
             {
-                from = self.ty(*pointer);
-                value = *pointer;
+                from = self.ty(pointer);
+                value = pointer;
                 was_updated_this_iteration = true;
                 if from == to {
-                    return Some(*pointer);
+                    return Some(pointer);
                 }
             }
 
@@ -1353,7 +1356,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             if let Transmute = kind
                 && let Value::Aggregate(variant_idx, field_values) = self.get(value)
                 && let Some((field_idx, field_ty)) =
-                    self.value_is_all_in_one_field(from, *variant_idx)
+                    self.value_is_all_in_one_field(from, variant_idx)
             {
                 from = field_ty;
                 value = field_values[field_idx.as_usize()];
@@ -1364,7 +1367,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             }
 
             // Various cast-then-cast cases can be simplified.
-            if let Value::Cast { kind: inner_kind, value: inner_value } = *self.get(value) {
+            if let Value::Cast { kind: inner_kind, value: inner_value } = self.get(value) {
                 let inner_from = self.ty(inner_value);
                 let new_kind = match (inner_kind, kind) {
                     // Even if there's a narrowing cast in here that's fine, because
@@ -1438,7 +1441,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         // We have an unsizing cast, which assigns the length to wide pointer metadata.
         if let Value::Cast { kind, value: from } = self.get(inner)
             && let CastKind::PointerCoercion(ty::adjustment::PointerCoercion::Unsize, _) = kind
-            && let Some(from) = self.ty(*from).builtin_deref(true)
+            && let Some(from) = self.ty(from).builtin_deref(true)
             && let ty::Array(_, len) = from.kind()
             && let Some(to) = self.ty(inner).builtin_deref(true)
             && let ty::Slice(..) = to.kind()
@@ -1596,7 +1599,7 @@ fn op_to_prop_const<'tcx>(
     None
 }
 
-impl<'tcx> VnState<'_, 'tcx> {
+impl<'tcx> VnState<'_, '_, 'tcx> {
     /// If either [`Self::try_as_constant`] as [`Self::try_as_place`] succeeds,
     /// returns that result as an [`Operand`].
     fn try_as_operand(&mut self, index: VnIndex, location: Location) -> Option<Operand<'tcx>> {
@@ -1615,7 +1618,7 @@ impl<'tcx> VnState<'_, 'tcx> {
         // This was already constant in MIR, do not change it. If the constant is not
         // deterministic, adding an additional mention of it in MIR will not give the same value as
         // the former mention.
-        if let Value::Constant { value, disambiguator: 0 } = *self.get(index) {
+        if let Value::Constant { value, disambiguator: 0 } = self.get(index) {
             debug_assert!(value.is_deterministic());
             return Some(ConstOperand { span: DUMMY_SP, user_ty: None, const_: value });
         }
@@ -1654,7 +1657,7 @@ impl<'tcx> VnState<'_, 'tcx> {
                 let place =
                     Place { local, projection: self.tcx.mk_place_elems(projection.as_slice()) };
                 return Some(place);
-            } else if let Value::Projection(pointer, proj) = *self.get(index)
+            } else if let Value::Projection(pointer, proj) = self.get(index)
                 && (allow_complex_projection || proj.is_stable_offset())
                 && let Some(proj) = self.try_as_place_elem(self.ty(index), proj, loc)
             {
@@ -1677,7 +1680,7 @@ impl<'tcx> VnState<'_, 'tcx> {
     }
 }
 
-impl<'tcx> MutVisitor<'tcx> for VnState<'_, 'tcx> {
+impl<'tcx> MutVisitor<'tcx> for VnState<'_, '_, 'tcx> {
     fn tcx(&self) -> TyCtxt<'tcx> {
         self.tcx
     }