Skip to content

GVN: Evaluate constants lazily. #146869

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 8, 2025
+45 −39
Merged

GVN: Evaluate constants lazily. #146869

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 45 additions & 39 deletions compiler/rustc_mir_transform/src/gvn.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,10 @@ struct VnState<'body, 'a, 'tcx> {
rev_locals: IndexVec<VnIndex, SmallVec<[Local; 1]>>,
values: ValueSet<'a, 'tcx>,
/// Values evaluated as constants if possible.
evaluated: IndexVec<VnIndex, Option<OpTy<'tcx>>>,
/// - `None` are values not computed yet;
/// - `Some(None)` are values for which computation has failed;
/// - `Some(Some(op))` are successful computations.
evaluated: IndexVec<VnIndex, Option<Option<&'a OpTy<'tcx>>>>,
/// Cache the deref values.
derefs: Vec<VnIndex>,
ssa: &'body SsaLocals,
Expand Down Expand Up @@ -416,8 +419,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
let (index, new) = self.values.insert(ty, value);
if new {
// Grow `evaluated` and `rev_locals` here to amortize the allocations.
let evaluated = self.eval_to_const(index);
let _index = self.evaluated.push(evaluated);
let _index = self.evaluated.push(None);
debug_assert_eq!(index, _index);
let _index = self.rev_locals.push(SmallVec::new());
debug_assert_eq!(index, _index);
Expand All @@ -430,7 +432,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
#[instrument(level = "trace", skip(self), ret)]
fn new_opaque(&mut self, ty: Ty<'tcx>) -> VnIndex {
let index = self.values.insert_unique(ty, Value::Opaque);
let _index = self.evaluated.push(None);
let _index = self.evaluated.push(Some(None));
debug_assert_eq!(index, _index);
let _index = self.rev_locals.push(SmallVec::new());
debug_assert_eq!(index, _index);
Expand Down Expand Up @@ -468,8 +470,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
kind,
provenance,
});
let evaluated = self.eval_to_const(index);
let _index = self.evaluated.push(evaluated);
let _index = self.evaluated.push(None);
debug_assert_eq!(index, _index);
let _index = self.rev_locals.push(SmallVec::new());
debug_assert_eq!(index, _index);
Expand All @@ -493,8 +494,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
(index, true)
};
if new {
let evaluated = self.eval_to_const(index);
let _index = self.evaluated.push(evaluated);
let _index = self.evaluated.push(None);
debug_assert_eq!(index, _index);
let _index = self.rev_locals.push(SmallVec::new());
debug_assert_eq!(index, _index);
Expand Down Expand Up @@ -551,7 +551,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
}

#[instrument(level = "trace", skip(self), ret)]
fn eval_to_const(&mut self, value: VnIndex) -> Option<OpTy<'tcx>> {
fn eval_to_const_inner(&mut self, value: VnIndex) -> Option<OpTy<'tcx>> {
use Value::*;
let ty = self.ty(value);
// Avoid computing layouts inside a coroutine, as that can cause cycles.
Expand All @@ -571,10 +571,8 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
self.ecx.eval_mir_constant(value, DUMMY_SP, None).discard_err()?
}
Aggregate(variant, ref fields) => {
let fields = fields
.iter()
.map(|&f| self.evaluated[f].as_ref())
.collect::<Option<Vec<_>>>()?;
let fields =
fields.iter().map(|&f| self.eval_to_const(f)).collect::<Option<Vec<_>>>()?;
let variant = if ty.ty.is_enum() { Some(variant) } else { None };
if matches!(ty.backend_repr, BackendRepr::Scalar(..) | BackendRepr::ScalarPair(..))
{
Expand Down Expand Up @@ -603,7 +601,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
}
}
Union(active_field, field) => {
let field = self.evaluated[field].as_ref()?;
let field = self.eval_to_const(field)?;
if matches!(ty.backend_repr, BackendRepr::Scalar(..) | BackendRepr::ScalarPair(..))
{
let dest = self.ecx.allocate(ty, MemoryKind::Stack).discard_err()?;
Expand All @@ -618,8 +616,8 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
}
}
RawPtr { pointer, metadata } => {
let pointer = self.evaluated[pointer].as_ref()?;
let metadata = self.evaluated[metadata].as_ref()?;
let pointer = self.eval_to_const(pointer)?;
let metadata = self.eval_to_const(metadata)?;

// Pointers don't have fields, so don't `project_field` them.
let data = self.ecx.read_pointer(pointer).discard_err()?;
Expand All @@ -633,7 +631,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
}

Projection(base, elem) => {
let base = self.evaluated[base].as_ref()?;
let base = self.eval_to_const(base)?;
// `Index` by constants should have been replaced by `ConstantIndex` by
// `simplify_place_projection`.
let elem = elem.try_map(|_| None, |()| ty.ty)?;
Expand All @@ -642,7 +640,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
Address { base, projection, .. } => {
debug_assert!(!projection.contains(&ProjectionElem::Deref));
let pointer = match base {
AddressBase::Deref(pointer) => self.evaluated[pointer].as_ref()?,
AddressBase::Deref(pointer) => self.eval_to_const(pointer)?,
// We have no stack to point to.
AddressBase::Local(_) => return None,
};
Expand All @@ -658,7 +656,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
}

Discriminant(base) => {
let base = self.evaluated[base].as_ref()?;
let base = self.eval_to_const(base)?;
let variant = self.ecx.read_discriminant(base).discard_err()?;
let discr_value =
self.ecx.discriminant_for_variant(base.layout.ty, variant).discard_err()?;
Expand All @@ -675,7 +673,6 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
NullOp::SizeOf => arg_layout.size.bytes(),
NullOp::AlignOf => arg_layout.align.bytes(),
NullOp::OffsetOf(fields) => self
.ecx
.tcx
.offset_of_subfield(self.typing_env(), arg_layout, fields.iter())
.bytes(),
Expand All @@ -685,34 +682,34 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
ImmTy::from_uint(val, ty).into()
}
UnaryOp(un_op, operand) => {
let operand = self.evaluated[operand].as_ref()?;
let operand = self.eval_to_const(operand)?;
let operand = self.ecx.read_immediate(operand).discard_err()?;
let val = self.ecx.unary_op(un_op, &operand).discard_err()?;
val.into()
}
BinaryOp(bin_op, lhs, rhs) => {
let lhs = self.evaluated[lhs].as_ref()?;
let lhs = self.eval_to_const(lhs)?;
let rhs = self.eval_to_const(rhs)?;
let lhs = self.ecx.read_immediate(lhs).discard_err()?;
let rhs = self.evaluated[rhs].as_ref()?;
let rhs = self.ecx.read_immediate(rhs).discard_err()?;
let val = self.ecx.binary_op(bin_op, &lhs, &rhs).discard_err()?;
val.into()
}
Cast { kind, value } => match kind {
CastKind::IntToInt | CastKind::IntToFloat => {
let value = self.evaluated[value].as_ref()?;
let value = self.eval_to_const(value)?;
let value = self.ecx.read_immediate(value).discard_err()?;
let res = self.ecx.int_to_int_or_float(&value, ty).discard_err()?;
res.into()
}
CastKind::FloatToFloat | CastKind::FloatToInt => {
let value = self.evaluated[value].as_ref()?;
let value = self.eval_to_const(value)?;
let value = self.ecx.read_immediate(value).discard_err()?;
let res = self.ecx.float_to_float_or_int(&value, ty).discard_err()?;
res.into()
}
CastKind::Transmute | CastKind::Subtype => {
let value = self.evaluated[value].as_ref()?;
let value = self.eval_to_const(value)?;
// `offset` for immediates generally only supports projections that match the
// type of the immediate. However, as a HACK, we exploit that it can also do
// limited transmutes: it only works between types with the same layout, and
Expand All @@ -724,12 +721,12 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
&& !matches!(s1.primitive(), Primitive::Pointer(..))
}
(BackendRepr::ScalarPair(a1, b1), BackendRepr::ScalarPair(a2, b2)) => {
a1.size(&self.ecx) == a2.size(&self.ecx) &&
b1.size(&self.ecx) == b2.size(&self.ecx) &&
// The alignment of the second component determines its offset, so that also needs to match.
b1.align(&self.ecx) == b2.align(&self.ecx) &&
// None of the inputs may be a pointer.
!matches!(a1.primitive(), Primitive::Pointer(..))
a1.size(&self.ecx) == a2.size(&self.ecx)
&& b1.size(&self.ecx) == b2.size(&self.ecx)
// The alignment of the second component determines its offset, so that also needs to match.
&& b1.align(&self.ecx) == b2.align(&self.ecx)
// None of the inputs may be a pointer.
&& !matches!(a1.primitive(), Primitive::Pointer(..))
&& !matches!(b1.primitive(), Primitive::Pointer(..))
}
_ => false,
Expand All @@ -741,7 +738,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
value.offset(Size::ZERO, ty, &self.ecx).discard_err()?
}
CastKind::PointerCoercion(ty::adjustment::PointerCoercion::Unsize, _) => {
let src = self.evaluated[value].as_ref()?;
let src = self.eval_to_const(value)?;
let dest = self.ecx.allocate(ty, MemoryKind::Stack).discard_err()?;
self.ecx.unsize_into(src, ty, &dest).discard_err()?;
self.ecx
Expand All @@ -750,13 +747,13 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
dest.into()
}
CastKind::FnPtrToPtr | CastKind::PtrToPtr => {
let src = self.evaluated[value].as_ref()?;
let src = self.eval_to_const(value)?;
let src = self.ecx.read_immediate(src).discard_err()?;
let ret = self.ecx.ptr_to_ptr(&src, ty).discard_err()?;
ret.into()
}
CastKind::PointerCoercion(ty::adjustment::PointerCoercion::UnsafeFnPointer, _) => {
let src = self.evaluated[value].as_ref()?;
let src = self.eval_to_const(value)?;
let src = self.ecx.read_immediate(src).discard_err()?;
ImmTy::from_immediate(*src, ty).into()
}
Expand All @@ -766,6 +763,15 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
Some(op)
}

fn eval_to_const(&mut self, index: VnIndex) -> Option<&'a OpTy<'tcx>> {
if let Some(op) = self.evaluated[index] {
return op;
}
let op = self.eval_to_const_inner(index);
self.evaluated[index] = Some(self.arena.alloc(op).as_ref());
self.evaluated[index].unwrap()
}

/// Represent the *value* we obtain by dereferencing an `Address` value.
#[instrument(level = "trace", skip(self), ret)]
fn dereference_address(
Expand Down Expand Up @@ -901,7 +907,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
if let ProjectionElem::Index(idx_local) = elem
&& let Some(idx) = self.locals[idx_local]
{
if let Some(offset) = self.evaluated[idx].as_ref()
if let Some(offset) = self.eval_to_const(idx)
&& let Some(offset) = self.ecx.read_target_usize(offset).discard_err()
&& let Some(min_length) = offset.checked_add(1)
{
Expand Down Expand Up @@ -1389,8 +1395,8 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {

let layout = self.ecx.layout_of(lhs_ty).ok()?;

let as_bits = |value: VnIndex| {
let constant = self.evaluated[value].as_ref()?;
let mut as_bits = |value: VnIndex| {
let constant = self.eval_to_const(value)?;
if layout.backend_repr.is_scalar() {
let scalar = self.ecx.read_scalar(constant).discard_err()?;
scalar.to_bits(constant.layout.size).discard_err()
Expand Down Expand Up @@ -1790,7 +1796,7 @@ impl<'tcx> VnState<'_, '_, 'tcx> {
return Some(ConstOperand { span: DUMMY_SP, user_ty: None, const_: value });
}

let op = self.evaluated[index].as_ref()?;
let op = self.eval_to_const(index)?;
if op.layout.is_unsized() {
// Do not attempt to propagate unsized locals.
return None;
Expand Down
Loading