diff --git a/Cargo.lock b/Cargo.lock index 2de9c84e3b031..fdc32351e2d30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4253,7 +4253,6 @@ dependencies = [ "hashbrown", "itertools", "rustc_abi", - "rustc_arena", "rustc_ast", "rustc_const_eval", "rustc_data_structures", diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs index bf5ec8f459e61..b1ff7ffc60edd 100644 --- a/compiler/rustc_mir_dataflow/src/value_analysis.rs +++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs @@ -24,9 +24,7 @@ rustc_index::newtype_index!( rustc_index::newtype_index!( /// This index uniquely identifies a tracked place and therefore a slot in [`State`]. - /// - /// It is an implementation detail of this module. - struct ValueIndex {} + pub struct ValueIndex {} ); /// See [`State`]. @@ -211,22 +209,9 @@ impl State { /// The target place must have been flooded before calling this method. pub fn insert_place_idx(&mut self, target: PlaceIndex, source: PlaceIndex, map: &Map<'_>) { let State::Reachable(values) = self else { return }; - - // If both places are tracked, we copy the value to the target. - // If the target is tracked, but the source is not, we do nothing, as invalidation has - // already been performed. - if let Some(target_value) = map.places[target].value_index - && let Some(source_value) = map.places[source].value_index - { - values.insert(target_value, values.get(source_value).clone()); - } - for target_child in map.children(target) { - // Try to find corresponding child and recurse. Reasoning is similar as above. - let projection = map.places[target_child].proj_elem.unwrap(); - if let Some(source_child) = map.projections.get(&(source, projection)) { - self.insert_place_idx(target_child, *source_child, map); - } - } + map.for_each_value_pair(target, source, &mut |target, source| { + values.insert(target, values.get(source).clone()); + }); } /// Helper method to interpret `target = result`. @@ -677,6 +662,26 @@ impl<'tcx> Map<'tcx> { self.find_extra(place, [TrackElem::DerefLen]) } + /// Locates the value corresponding to the given place. + pub fn value(&self, place: PlaceIndex) -> Option { + self.places[place].value_index + } + + /// Locates the value corresponding to the given place. + pub fn find_value(&self, place: PlaceRef<'_>) -> Option { + self.value(self.find(place)?) + } + + /// Locates the value corresponding to the given discriminant. + pub fn find_discr_value(&self, place: PlaceRef<'_>) -> Option { + self.value(self.find_discr(place)?) + } + + /// Locates the value corresponding to the given length. + pub fn find_len_value(&self, place: PlaceRef<'_>) -> Option { + self.value(self.find_len(place)?) + } + /// Iterate over all direct children. fn children(&self, parent: PlaceIndex) -> impl Iterator { Children::new(self, parent) @@ -689,7 +694,7 @@ impl<'tcx> Map<'tcx> { /// /// `tail_elem` allows to support discriminants that are not a place in MIR, but that we track /// as such. - fn for_each_aliasing_place( + pub fn for_each_aliasing_place( &self, place: PlaceRef<'_>, tail_elem: Option, @@ -745,11 +750,15 @@ impl<'tcx> Map<'tcx> { } } + /// Return the range of value indices inside this place. + pub fn values_inside(&self, root: PlaceIndex) -> &[ValueIndex] { + let range = self.inner_values[root].clone(); + &self.inner_values_buffer[range] + } + /// Invoke a function on each value in the given place and all descendants. fn for_each_value_inside(&self, root: PlaceIndex, f: &mut impl FnMut(ValueIndex)) { - let range = self.inner_values[root].clone(); - let values = &self.inner_values_buffer[range]; - for &v in values { + for &v in self.values_inside(root) { f(v) } } @@ -778,6 +787,31 @@ impl<'tcx> Map<'tcx> { } } } + + /// Recursively iterates on each value contained in `target`, paired with matching projection + /// inside `source`. + pub fn for_each_value_pair( + &self, + target: PlaceIndex, + source: PlaceIndex, + f: &mut impl FnMut(ValueIndex, ValueIndex), + ) { + // If both places are tracked, we copy the value to the target. + // If the target is tracked, but the source is not, we do nothing, as invalidation has + // already been performed. + if let Some(target_value) = self.places[target].value_index + && let Some(source_value) = self.places[source].value_index + { + f(target_value, source_value) + } + for target_child in self.children(target) { + // Try to find corresponding child and recurse. Reasoning is similar as above. + let projection = self.places[target_child].proj_elem.unwrap(); + if let Some(source_child) = self.projections.get(&(source, projection)) { + self.for_each_value_pair(target_child, *source_child, f); + } + } + } } /// This is the information tracked for every [`PlaceIndex`] and is stored by [`Map`]. diff --git a/compiler/rustc_mir_transform/Cargo.toml b/compiler/rustc_mir_transform/Cargo.toml index 511c1960e40b4..f8e664146aed7 100644 --- a/compiler/rustc_mir_transform/Cargo.toml +++ b/compiler/rustc_mir_transform/Cargo.toml @@ -9,7 +9,6 @@ either = "1" hashbrown = "0.15" itertools = "0.12" rustc_abi = { path = "../rustc_abi" } -rustc_arena = { path = "../rustc_arena" } rustc_ast = { path = "../rustc_ast" } rustc_const_eval = { path = "../rustc_const_eval" } rustc_data_structures = { path = "../rustc_data_structures" } diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 68298767e7fd8..279e4dcdde0cb 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -7,47 +7,62 @@ //! ------------/ \-------- ------------ //! //! -//! We proceed by walking the cfg backwards starting from each `SwitchInt` terminator, -//! looking for assignments that will turn the `SwitchInt` into a simple `Goto`. +//! This implementation is heavily inspired by the work outlined in [libfirm]. //! -//! The algorithm maintains a set of replacement conditions: -//! - `conditions[place]` contains `Condition { value, polarity: Eq, target }` -//! if assigning `value` to `place` turns the `SwitchInt` into `Goto { target }`. -//! - `conditions[place]` contains `Condition { value, polarity: Ne, target }` -//! if assigning anything different from `value` to `place` turns the `SwitchInt` -//! into `Goto { target }`. +//! The general algorithm proceeds in two phases: (1) walk the CFG backwards to construct a +//! graph of threading conditions, and (2) propagate fulfilled conditions forward by duplicating +//! blocks. +//! +//! # 1. Condition graph construction //! //! In this file, we denote as `place ?= value` the existence of a replacement condition //! on `place` with given `value`, irrespective of the polarity and target of that //! replacement condition. //! -//! We then walk the CFG backwards transforming the set of conditions. -//! When we find a fulfilling assignment, we record a `ThreadingOpportunity`. -//! All `ThreadingOpportunity`s are applied to the body, by duplicating blocks if required. +//! Inside a block, we associate with each condition `c` a set of targets: +//! - `Goto(target)` if fulfilling `c` changes the terminator into a `Goto { target }`; +//! - `Chain(target, c2)` if fulfilling `c` means that `c2` is fulfilled inside `target`. +//! +//! Before walking a block `bb`, we construct the exit set of condition from its successors. +//! For each condition `c` in a successor `s`, we record that fulfilling `c` in `bb` will fulfill +//! `c` in `s`, as a `Chain(s, c)` condition. //! -//! The optimization search can be very heavy, as it performs a DFS on MIR starting from -//! each `SwitchInt` terminator. To manage the complexity, we: -//! - bound the maximum depth by a constant `MAX_BACKTRACK`; -//! - we only traverse `Goto` terminators. +//! When encountering a `switchInt(place) -> [value: bb...]` terminator, we also record a +//! `place == value` condition for each `value`, and associate a `Goto(target)` condition. +//! +//! Then, we walk the statements backwards, transforming the set of conditions along the way, +//! resulting in a set of conditions at the block entry. //! //! We try to avoid creating irreducible control-flow by not threading through a loop header. //! -//! Likewise, applying the optimisation can create a lot of new MIR, so we bound the instruction +//! Applying the optimisation can create a lot of new MIR, so we bound the instruction //! cost by `MAX_COST`. +//! +//! # 2. Block duplication +//! +//! We now have the set of fulfilled conditions inside each block and their targets. +//! +//! For each block `bb` in reverse postorder, we apply in turn the target associated with each +//! fulfilled condition: +//! - for `Goto(target)`, change the terminator of `bb` into a `Goto { target }`; +//! - for `Chain(target, cond)`, duplicate `target` into a new block which fulfills the same +//! conditions and also fulfills `cond`. This is made efficient by maintaining a map of duplicates, +//! `duplicate[(target, cond)]` to avoid cloning blocks multiple times. +//! +//! [libfirm]: -use rustc_arena::DroplessArena; +use itertools::Itertools as _; use rustc_const_eval::const_eval::DummyMachine; use rustc_const_eval::interpret::{ImmTy, Immediate, InterpCx, OpTy, Projectable}; -use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet}; use rustc_index::IndexVec; -use rustc_index::bit_set::DenseBitSet; +use rustc_index::bit_set::{DenseBitSet, GrowableBitSet}; use rustc_middle::bug; use rustc_middle::mir::interpret::Scalar; use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::*; use rustc_middle::ty::{self, ScalarInt, TyCtxt}; -use rustc_mir_dataflow::lattice::HasBottom; -use rustc_mir_dataflow::value_analysis::{Map, PlaceIndex, State, TrackElem}; +use rustc_mir_dataflow::value_analysis::{Map, PlaceIndex, TrackElem, ValueIndex}; use rustc_span::DUMMY_SP; use tracing::{debug, instrument, trace}; @@ -55,8 +70,7 @@ use crate::cost_checker::CostChecker; pub(super) struct JumpThreading; -const MAX_BACKTRACK: usize = 5; -const MAX_COST: usize = 100; +const MAX_COST: u8 = 100; const MAX_PLACES: usize = 100; impl<'tcx> crate::MirPass<'tcx> for JumpThreading { @@ -76,33 +90,54 @@ impl<'tcx> crate::MirPass<'tcx> for JumpThreading { } let typing_env = body.typing_env(tcx); - let arena = &DroplessArena::default(); let mut finder = TOFinder { tcx, typing_env, ecx: InterpCx::new(tcx, DUMMY_SP, typing_env, DummyMachine), body, - arena, map: Map::new(tcx, body, Some(MAX_PLACES)), maybe_loop_headers: maybe_loop_headers(body), - opportunities: Vec::new(), + entry_states: IndexVec::from_elem(ConditionSet::default(), &body.basic_blocks), }; - for (bb, _) in traversal::preorder(body) { - finder.start_from_switch(bb); - } + for (bb, bbdata) in traversal::postorder(body) { + if bbdata.is_cleanup { + continue; + } - let opportunities = finder.opportunities; - debug!(?opportunities); - if opportunities.is_empty() { - return; + let mut state = finder.populate_from_outgoing_edges(bb); + trace!("output_states[{bb:?}] = {state:?}"); + + finder.process_terminator(bb, &mut state); + trace!("pre_terminator_states[{bb:?}] = {state:?}"); + + for stmt in bbdata.statements.iter().rev() { + if state.is_empty() { + break; + } + + finder.process_statement(stmt, &mut state); + + // When a statement mutates a place, assignments to that place that happen + // above the mutation cannot fulfill a condition. + // _1 = 5 // Whatever happens here, it won't change the result of a `SwitchInt`. + // _1 = 6 + if let Some((lhs, tail)) = finder.mutated_statement(stmt) { + finder.flood_state(lhs, tail, &mut state); + } + } + + trace!("entry_states[{bb:?}] = {state:?}"); + finder.entry_states[bb] = state; } - // Verify that we do not thread through a loop header. - for to in opportunities.iter() { - assert!(to.chain.iter().all(|&block| !finder.maybe_loop_headers.contains(block))); + let mut entry_states = finder.entry_states; + simplify_conditions(body, &mut entry_states); + remove_costly_conditions(tcx, typing_env, body, &mut entry_states); + + if let Some(opportunities) = OpportunitySet::new(body, entry_states) { + opportunities.apply(); } - OpportunitySet::new(body, opportunities).apply(body); } fn is_required(&self) -> bool { @@ -110,14 +145,6 @@ impl<'tcx> crate::MirPass<'tcx> for JumpThreading { } } -#[derive(Debug)] -struct ThreadingOpportunity { - /// The list of `BasicBlock`s from the one that found the opportunity to the `SwitchInt`. - chain: Vec, - /// The `SwitchInt` will be replaced by `Goto { target }`. - target: BasicBlock, -} - struct TOFinder<'a, 'tcx> { tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>, @@ -125,192 +152,207 @@ struct TOFinder<'a, 'tcx> { body: &'a Body<'tcx>, map: Map<'tcx>, maybe_loop_headers: DenseBitSet, - /// We use an arena to avoid cloning the slices when cloning `state`. - arena: &'a DroplessArena, - opportunities: Vec, + /// This stores the state of each visited block on entry, + /// and the current state of the block being visited. + // Invariant: for each `bb`, each condition in `entry_states[bb]` has a `chain` that + // starts with `bb`. + entry_states: IndexVec, +} + +rustc_index::newtype_index! { + #[derive(Ord, PartialOrd)] + #[debug_format = "_c{}"] + struct ConditionIndex {} } /// Represent the following statement. If we can prove that the current local is equal/not-equal /// to `value`, jump to `target`. -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)] struct Condition { + place: ValueIndex, value: ScalarInt, polarity: Polarity, - target: BasicBlock, } -#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)] enum Polarity { Ne, Eq, } impl Condition { - fn matches(&self, value: ScalarInt) -> bool { - (self.value == value) == (self.polarity == Polarity::Eq) + fn matches(&self, place: ValueIndex, value: ScalarInt) -> bool { + self.place == place && (self.value == value) == (self.polarity == Polarity::Eq) } } -#[derive(Copy, Clone, Debug)] -struct ConditionSet<'a>(&'a [Condition]); - -impl HasBottom for ConditionSet<'_> { - const BOTTOM: Self = ConditionSet(&[]); - - fn is_bottom(&self) -> bool { - self.0.is_empty() - } +/// Represent the effect of fulfilling a condition. +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +enum EdgeEffect { + /// If the condition is fulfilled, replace the current block's terminator by a single goto. + Goto { target: BasicBlock }, + /// If the condition is fulfilled, fulfill the condition `succ_condition` in `succ_block`. + Chain { succ_block: BasicBlock, succ_condition: ConditionIndex }, } -impl<'a> ConditionSet<'a> { - fn iter(self) -> impl Iterator { - self.0.iter().copied() +impl EdgeEffect { + fn block(self) -> BasicBlock { + match self { + EdgeEffect::Goto { target: bb } | EdgeEffect::Chain { succ_block: bb, .. } => bb, + } } - fn iter_matches(self, value: ScalarInt) -> impl Iterator { - self.iter().filter(move |c| c.matches(value)) + fn replace_block(&mut self, target: BasicBlock, new_target: BasicBlock) { + match self { + EdgeEffect::Goto { target: bb } | EdgeEffect::Chain { succ_block: bb, .. } => { + if *bb == target { + *bb = new_target + } + } + } } +} - fn map( - self, - arena: &'a DroplessArena, - f: impl Fn(Condition) -> Option, - ) -> Option> { - let set = arena.try_alloc_from_iter(self.iter().map(|c| f(c).ok_or(()))).ok()?; - Some(ConditionSet(set)) - } +#[derive(Clone, Debug, Default)] +struct ConditionSet { + active: Vec<(ConditionIndex, Condition)>, + fulfilled: Vec, + targets: IndexVec>, } -impl<'a, 'tcx> TOFinder<'a, 'tcx> { - fn is_empty(&self, state: &State>) -> bool { - state.all_bottom() +impl ConditionSet { + fn is_empty(&self) -> bool { + self.active.is_empty() } - /// Recursion entry point to find threading opportunities. - #[instrument(level = "trace", skip(self))] - fn start_from_switch(&mut self, bb: BasicBlock) { - let bbdata = &self.body[bb]; - if bbdata.is_cleanup || self.maybe_loop_headers.contains(bb) { - return; - } - let Some((discr, targets)) = bbdata.terminator().kind.as_switch() else { return }; - let Some(discr) = discr.place() else { return }; - debug!(?discr, ?bb); - - let discr_ty = discr.ty(self.body, self.tcx).ty; - let Ok(discr_layout) = self.ecx.layout_of(discr_ty) else { return }; + #[tracing::instrument(level = "trace", skip(self))] + fn push_condition(&mut self, c: Condition, target: BasicBlock) { + let index = self.targets.push(vec![EdgeEffect::Goto { target }]); + self.active.push((index, c)); + } - let Some(discr) = self.map.find(discr.as_ref()) else { return }; - debug!(?discr); + /// Register fulfilled condition and remove it from the set. + fn fulfill_if(&mut self, f: impl Fn(Condition, &Vec) -> bool) { + self.active.retain(|&(index, condition)| { + let targets = &self.targets[index]; + if f(condition, targets) { + trace!(?index, ?condition, "fulfill"); + self.fulfilled.push(index); + false + } else { + true + } + }) + } - let cost = CostChecker::new(self.tcx, self.typing_env, None, self.body); - let mut state = State::new_reachable(); + /// Register fulfilled condition and remove them from the set. + fn fulfill_matches(&mut self, place: ValueIndex, value: ScalarInt) { + self.fulfill_if(|c, _| c.matches(place, value)) + } - let conds = if let Some((value, then, else_)) = targets.as_static_if() { - let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else { return }; - self.arena.alloc_from_iter([ - Condition { value, polarity: Polarity::Eq, target: then }, - Condition { value, polarity: Polarity::Ne, target: else_ }, - ]) - } else { - self.arena.alloc_from_iter(targets.iter().filter_map(|(value, target)| { - let value = ScalarInt::try_from_uint(value, discr_layout.size)?; - Some(Condition { value, polarity: Polarity::Eq, target }) - })) - }; - let conds = ConditionSet(conds); - state.insert_value_idx(discr, conds, &self.map); + fn retain(&mut self, mut f: impl FnMut(Condition) -> bool) { + self.active.retain(|&(_, c)| f(c)) + } - self.find_opportunity(bb, state, cost, 0) + fn retain_mut(&mut self, mut f: impl FnMut(Condition) -> Option) { + self.active.retain_mut(|(_, c)| { + if let Some(new) = f(*c) { + *c = new; + true + } else { + false + } + }) } - /// Recursively walk statements backwards from this bb's terminator to find threading - /// opportunities. - #[instrument(level = "trace", skip(self, cost), ret)] - fn find_opportunity( - &mut self, - bb: BasicBlock, - mut state: State>, - mut cost: CostChecker<'_, 'tcx>, - depth: usize, - ) { - // Do not thread through loop headers. - if self.maybe_loop_headers.contains(bb) { - return; + fn for_each_mut(&mut self, f: impl Fn(&mut Condition)) { + for (_, c) in &mut self.active { + f(c) } + } +} - debug!(cost = ?cost.cost()); - for (statement_index, stmt) in - self.body.basic_blocks[bb].statements.iter().enumerate().rev() - { - if self.is_empty(&state) { - return; - } +impl<'a, 'tcx> TOFinder<'a, 'tcx> { + /// Construct the condition set for `bb` from the terminator, without executing its effect. + #[instrument(level = "trace", skip(self))] + fn populate_from_outgoing_edges(&mut self, bb: BasicBlock) -> ConditionSet { + let bbdata = &self.body[bb]; - cost.visit_statement(stmt, Location { block: bb, statement_index }); - if cost.cost() > MAX_COST { - return; - } + // This should be the first time we populate `entry_states[bb]`. + debug_assert!(self.entry_states[bb].is_empty()); - // Attempt to turn the `current_condition` on `lhs` into a condition on another place. - self.process_statement(bb, stmt, &mut state); + let state_len = + bbdata.terminator().successors().map(|succ| self.entry_states[succ].active.len()).sum(); + let mut state = ConditionSet { + active: Vec::with_capacity(state_len), + targets: IndexVec::with_capacity(state_len), + fulfilled: Vec::new(), + }; - // When a statement mutates a place, assignments to that place that happen - // above the mutation cannot fulfill a condition. - // _1 = 5 // Whatever happens here, it won't change the result of a `SwitchInt`. - // _1 = 6 - if let Some((lhs, tail)) = self.mutated_statement(stmt) { - state.flood_with_tail_elem(lhs.as_ref(), tail, &self.map, ConditionSet::BOTTOM); + // Use an index-set to deduplicate conditions coming from different successor blocks. + let mut known_conditions = + FxIndexSet::with_capacity_and_hasher(state_len, Default::default()); + let mut insert = |condition, succ_block, succ_condition| { + let (index, new) = known_conditions.insert_full(condition); + let index = ConditionIndex::from_usize(index); + if new { + state.active.push((index, condition)); + let _index = state.targets.push(Vec::new()); + debug_assert_eq!(_index, index); } - } - - if self.is_empty(&state) || depth >= MAX_BACKTRACK { - return; - } + let target = EdgeEffect::Chain { succ_block, succ_condition }; + debug_assert!( + !state.targets[index].contains(&target), + "duplicate targets for index={index:?} as {target:?} targets={:#?}", + &state.targets[index], + ); + state.targets[index].push(target); + }; - let last_non_rec = self.opportunities.len(); + // A given block may have several times the same successor. + let mut seen = FxHashSet::default(); + for succ in bbdata.terminator().successors() { + if !seen.insert(succ) { + continue; + } - let predecessors = &self.body.basic_blocks.predecessors()[bb]; - if let &[pred] = &predecessors[..] - && bb != START_BLOCK - { - let term = self.body.basic_blocks[pred].terminator(); - match term.kind { - TerminatorKind::SwitchInt { ref discr, ref targets } => { - self.process_switch_int(discr, targets, bb, &mut state); - self.find_opportunity(pred, state, cost, depth + 1); - } - _ => self.recurse_through_terminator(pred, || state, &cost, depth), + // Do not thread through loop headers. + if self.maybe_loop_headers.contains(succ) { + continue; } - } else if let &[ref predecessors @ .., last_pred] = &predecessors[..] { - for &pred in predecessors { - self.recurse_through_terminator(pred, || state.clone(), &cost, depth); + + for &(succ_index, cond) in self.entry_states[succ].active.iter() { + insert(cond, succ, succ_index); } - self.recurse_through_terminator(last_pred, || state, &cost, depth); } - let new_tos = &mut self.opportunities[last_non_rec..]; - debug!(?new_tos); + let num_conditions = known_conditions.len(); + debug_assert_eq!(num_conditions, state.active.len()); + debug_assert_eq!(num_conditions, state.targets.len()); + state.fulfilled.reserve(num_conditions); - // Try to deduplicate threading opportunities. - if new_tos.len() > 1 - && new_tos.len() == predecessors.len() - && predecessors - .iter() - .zip(new_tos.iter()) - .all(|(&pred, to)| to.chain == &[pred] && to.target == new_tos[0].target) - { - // All predecessors have a threading opportunity, and they all point to the same block. - debug!(?new_tos, "dedup"); - let first = &mut new_tos[0]; - *first = ThreadingOpportunity { chain: vec![bb], target: first.target }; - self.opportunities.truncate(last_non_rec + 1); + state + } + + /// Remove all conditions in the state that alias given place. + fn flood_state( + &self, + place: Place<'tcx>, + extra_elem: Option, + state: &mut ConditionSet, + ) { + if state.is_empty() { return; } - - for op in self.opportunities[last_non_rec..].iter_mut() { - op.chain.push(bb); + let mut places_to_exclude = FxHashSet::default(); + self.map.for_each_aliasing_place(place.as_ref(), extra_elem, &mut |vi| { + places_to_exclude.insert(vi); + }); + trace!(?places_to_exclude, "flood_state"); + if places_to_exclude.is_empty() { + return; } + state.retain(|c| !places_to_exclude.contains(&c.place)); } /// Extract the mutated place from a statement. @@ -354,35 +396,27 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { } } - #[instrument(level = "trace", skip(self))] - fn process_immediate( - &mut self, - bb: BasicBlock, - lhs: PlaceIndex, - rhs: ImmTy<'tcx>, - state: &mut State>, - ) { - let register_opportunity = |c: Condition| { - debug!(?bb, ?c.target, "register"); - self.opportunities.push(ThreadingOpportunity { chain: vec![bb], target: c.target }) - }; - - if let Some(conditions) = state.try_get_idx(lhs, &self.map) + #[instrument(level = "trace", skip(self, state))] + fn process_immediate(&mut self, lhs: PlaceIndex, rhs: ImmTy<'tcx>, state: &mut ConditionSet) { + if let Some(lhs) = self.map.value(lhs) && let Immediate::Scalar(Scalar::Int(int)) = *rhs { - conditions.iter_matches(int).for_each(register_opportunity); + state.fulfill_matches(lhs, int) } } /// If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`. - #[instrument(level = "trace", skip(self))] + #[instrument(level = "trace", skip(self, state))] fn process_constant( &mut self, - bb: BasicBlock, lhs: PlaceIndex, constant: OpTy<'tcx>, - state: &mut State>, + state: &mut ConditionSet, ) { + let values_inside = self.map.values_inside(lhs); + if !state.active.iter().any(|&(_, cond)| values_inside.contains(&cond.place)) { + return; + } self.map.for_each_projection_value( lhs, constant, @@ -403,28 +437,32 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { } }, &mut |place, op| { - if let Some(conditions) = state.try_get_idx(place, &self.map) + if let Some(place) = self.map.value(place) && let Some(imm) = self.ecx.read_immediate_raw(op).discard_err() && let Some(imm) = imm.right() && let Immediate::Scalar(Scalar::Int(int)) = *imm { - conditions.iter_matches(int).for_each(|c: Condition| { - self.opportunities - .push(ThreadingOpportunity { chain: vec![bb], target: c.target }) - }) + state.fulfill_matches(place, int) } }, ); } - #[instrument(level = "trace", skip(self))] - fn process_operand( - &mut self, - bb: BasicBlock, - lhs: PlaceIndex, - rhs: &Operand<'tcx>, - state: &mut State>, - ) { + #[instrument(level = "trace", skip(self, state))] + fn process_copy(&mut self, lhs: PlaceIndex, rhs: PlaceIndex, state: &mut ConditionSet) { + let mut renames = FxHashMap::default(); + self.map.for_each_value_pair(rhs, lhs, &mut |rhs, lhs| { + renames.insert(lhs, rhs); + }); + state.for_each_mut(|c| { + if let Some(rhs) = renames.get(&c.place) { + c.place = *rhs + } + }); + } + + #[instrument(level = "trace", skip(self, state))] + fn process_operand(&mut self, lhs: PlaceIndex, rhs: &Operand<'tcx>, state: &mut ConditionSet) { match rhs { // If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`. Operand::Constant(constant) => { @@ -433,32 +471,34 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { else { return; }; - self.process_constant(bb, lhs, constant, state); + self.process_constant(lhs, constant, state); } // Transfer the conditions on the copied rhs. Operand::Move(rhs) | Operand::Copy(rhs) => { let Some(rhs) = self.map.find(rhs.as_ref()) else { return }; - state.insert_place_idx(rhs, lhs, &self.map); + self.process_copy(lhs, rhs, state) } } } - #[instrument(level = "trace", skip(self))] + #[instrument(level = "trace", skip(self, state))] fn process_assign( &mut self, - bb: BasicBlock, lhs_place: &Place<'tcx>, - rhs: &Rvalue<'tcx>, - state: &mut State>, + rvalue: &Rvalue<'tcx>, + state: &mut ConditionSet, ) { let Some(lhs) = self.map.find(lhs_place.as_ref()) else { return }; - match rhs { - Rvalue::Use(operand) => self.process_operand(bb, lhs, operand, state), + match rvalue { + Rvalue::Use(operand) => self.process_operand(lhs, operand, state), // Transfer the conditions on the copy rhs. - Rvalue::CopyForDeref(rhs) => self.process_operand(bb, lhs, &Operand::Copy(*rhs), state), + Rvalue::CopyForDeref(rhs) => { + let Some(rhs) = self.map.find(rhs.as_ref()) else { return }; + self.process_copy(lhs, rhs, state) + } Rvalue::Discriminant(rhs) => { let Some(rhs) = self.map.find_discr(rhs.as_ref()) else { return }; - state.insert_place_idx(rhs, lhs, &self.map); + self.process_copy(lhs, rhs, state) } // If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`. Rvalue::Aggregate(box kind, operands) => { @@ -473,7 +513,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { .discriminant_for_variant(agg_ty, *variant_index) .discard_err() { - self.process_immediate(bb, discr_target, discr_value, state); + self.process_immediate(discr_target, discr_value, state); } if let Some(idx) = self.map.apply(lhs, TrackElem::Variant(*variant_index)) { idx @@ -485,37 +525,36 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { }; for (field_index, operand) in operands.iter_enumerated() { if let Some(field) = self.map.apply(lhs, TrackElem::Field(field_index)) { - self.process_operand(bb, field, operand, state); + self.process_operand(field, operand, state); } } } // Transfer the conditions on the copy rhs, after inverting the value of the condition. - Rvalue::UnaryOp(UnOp::Not, Operand::Move(place) | Operand::Copy(place)) => { - let layout = self.ecx.layout_of(place.ty(self.body, self.tcx).ty).unwrap(); - let Some(conditions) = state.try_get_idx(lhs, &self.map) else { return }; - let Some(place) = self.map.find(place.as_ref()) else { return }; - let Some(conds) = conditions.map(self.arena, |mut cond| { - cond.value = self - .ecx - .unary_op(UnOp::Not, &ImmTy::from_scalar_int(cond.value, layout)) - .discard_err()? - .to_scalar_int() - .discard_err()?; - Some(cond) - }) else { - return; - }; - state.insert_value_idx(place, conds, &self.map); + Rvalue::UnaryOp(UnOp::Not, Operand::Move(operand) | Operand::Copy(operand)) => { + let layout = self.ecx.layout_of(operand.ty(self.body, self.tcx).ty).unwrap(); + let Some(lhs) = self.map.value(lhs) else { return }; + let Some(operand) = self.map.find_value(operand.as_ref()) else { return }; + state.retain_mut(|mut c| { + if c.place == lhs { + let value = self + .ecx + .unary_op(UnOp::Not, &ImmTy::from_scalar_int(c.value, layout)) + .discard_err()? + .to_scalar_int() + .discard_err()?; + c.place = operand; + c.value = value; + } + Some(c) + }); } // We expect `lhs ?= A`. We found `lhs = Eq(rhs, B)`. // Create a condition on `rhs ?= B`. Rvalue::BinaryOp( op, - box (Operand::Move(place) | Operand::Copy(place), Operand::Constant(value)) - | box (Operand::Constant(value), Operand::Move(place) | Operand::Copy(place)), + box (Operand::Move(operand) | Operand::Copy(operand), Operand::Constant(value)) + | box (Operand::Constant(value), Operand::Move(operand) | Operand::Copy(operand)), ) => { - let Some(conditions) = state.try_get_idx(lhs, &self.map) else { return }; - let Some(place) = self.map.find(place.as_ref()) else { return }; let equals = match op { BinOp::Eq => ScalarInt::TRUE, BinOp::Ne => ScalarInt::FALSE, @@ -528,38 +567,29 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { // Avoid handling them, though this could be extended in the future. return; } + let Some(lhs) = self.map.value(lhs) else { return }; + let Some(operand) = self.map.find_value(operand.as_ref()) else { return }; let Some(value) = value.const_.try_eval_scalar_int(self.tcx, self.typing_env) else { return; }; - let Some(conds) = conditions.map(self.arena, |c| { - Some(Condition { - value, - polarity: if c.matches(equals) { Polarity::Eq } else { Polarity::Ne }, - ..c - }) - }) else { - return; - }; - state.insert_value_idx(place, conds, &self.map); + state.for_each_mut(|c| { + if c.place == lhs { + let polarity = + if c.matches(lhs, equals) { Polarity::Eq } else { Polarity::Ne }; + c.place = operand; + c.value = value; + c.polarity = polarity; + } + }); } _ => {} } } - #[instrument(level = "trace", skip(self))] - fn process_statement( - &mut self, - bb: BasicBlock, - stmt: &Statement<'tcx>, - state: &mut State>, - ) { - let register_opportunity = |c: Condition| { - debug!(?bb, ?c.target, "register"); - self.opportunities.push(ThreadingOpportunity { chain: vec![bb], target: c.target }) - }; - + #[instrument(level = "trace", skip(self, state))] + fn process_statement(&mut self, stmt: &Statement<'tcx>, state: &mut ConditionSet) { // Below, `lhs` is the return value of `mutated_statement`, // the place to which `conditions` apply. @@ -577,63 +607,59 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { else { return; }; - self.process_immediate(bb, discr_target, discr, state) + self.process_immediate(discr_target, discr, state) } // If we expect `lhs ?= true`, we have an opportunity if we assume `lhs == true`. StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume( Operand::Copy(place) | Operand::Move(place), )) => { - let Some(conditions) = state.try_get(place.as_ref(), &self.map) else { return }; - conditions.iter_matches(ScalarInt::TRUE).for_each(register_opportunity) + let Some(place) = self.map.find_value(place.as_ref()) else { return }; + state.fulfill_matches(place, ScalarInt::TRUE); } StatementKind::Assign(box (lhs_place, rhs)) => { - self.process_assign(bb, lhs_place, rhs, state) + self.process_assign(lhs_place, rhs, state) } _ => {} } } - #[instrument(level = "trace", skip(self, state, cost))] - fn recurse_through_terminator( - &mut self, - bb: BasicBlock, - // Pass a closure that may clone the state, as we don't want to do it each time. - state: impl FnOnce() -> State>, - cost: &CostChecker<'_, 'tcx>, - depth: usize, - ) { + /// Execute the terminator for block `bb` into state `entry_states[bb]`. + #[instrument(level = "trace", skip(self, state))] + fn process_terminator(&mut self, bb: BasicBlock, state: &mut ConditionSet) { let term = self.body.basic_blocks[bb].terminator(); let place_to_flood = match term.kind { - // We come from a target, so those are not possible. - TerminatorKind::UnwindResume - | TerminatorKind::UnwindTerminate(_) - | TerminatorKind::Return - | TerminatorKind::TailCall { .. } - | TerminatorKind::Unreachable - | TerminatorKind::CoroutineDrop => bug!("{term:?} has no terminators"), // Disallowed during optimizations. TerminatorKind::FalseEdge { .. } | TerminatorKind::FalseUnwind { .. } | TerminatorKind::Yield { .. } => bug!("{term:?} invalid"), // Cannot reason about inline asm. - TerminatorKind::InlineAsm { .. } => return, + TerminatorKind::InlineAsm { .. } => { + state.active.clear(); + return; + } // `SwitchInt` is handled specially. - TerminatorKind::SwitchInt { .. } => return, - // We can recurse, no thing particular to do. - TerminatorKind::Goto { .. } => None, + TerminatorKind::SwitchInt { ref discr, ref targets } => { + return self.process_switch_int(discr, targets, state); + } + // These do not modify memory. + TerminatorKind::UnwindResume + | TerminatorKind::UnwindTerminate(_) + | TerminatorKind::Return + | TerminatorKind::Unreachable + | TerminatorKind::CoroutineDrop + // Assertions can be no-op at codegen time, so treat them as such. + | TerminatorKind::Assert { .. } + | TerminatorKind::Goto { .. } => None, // Flood the overwritten place, and progress through. TerminatorKind::Drop { place: destination, .. } | TerminatorKind::Call { destination, .. } => Some(destination), - // Ignore, as this can be a no-op at codegen time. - TerminatorKind::Assert { .. } => None, + TerminatorKind::TailCall { .. } => Some(RETURN_PLACE.into()), }; - // We can recurse through this terminator. - let mut state = state(); + // This terminator modifies `place_to_flood`, cleanup the associated conditions. if let Some(place_to_flood) = place_to_flood { - state.flood_with(place_to_flood.as_ref(), &self.map, ConditionSet::BOTTOM); + self.flood_state(place_to_flood, None, state); } - self.find_opportunity(bb, state, cost.clone(), depth + 1) } #[instrument(level = "trace", skip(self))] @@ -641,196 +667,422 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { &mut self, discr: &Operand<'tcx>, targets: &SwitchTargets, - target_bb: BasicBlock, - state: &mut State>, + state: &mut ConditionSet, ) { - debug_assert_ne!(target_bb, START_BLOCK); - debug_assert_eq!(self.body.basic_blocks.predecessors()[target_bb].len(), 1); - let Some(discr) = discr.place() else { return }; + let Some(discr_idx) = self.map.find_value(discr.as_ref()) else { return }; + let discr_ty = discr.ty(self.body, self.tcx).ty; - let Ok(discr_layout) = self.ecx.layout_of(discr_ty) else { - return; - }; - let Some(conditions) = state.try_get(discr.as_ref(), &self.map) else { return }; + let Ok(discr_layout) = self.ecx.layout_of(discr_ty) else { return }; - if let Some((value, _)) = targets.iter().find(|&(_, target)| target == target_bb) { - let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else { return }; - debug_assert_eq!(targets.iter().filter(|&(_, target)| target == target_bb).count(), 1); - - // We are inside `target_bb`. Since we have a single predecessor, we know we passed - // through the `SwitchInt` before arriving here. Therefore, we know that - // `discr == value`. If one condition can be fulfilled by `discr == value`, - // that's an opportunity. - for c in conditions.iter_matches(value) { - debug!(?target_bb, ?c.target, "register"); - self.opportunities.push(ThreadingOpportunity { chain: vec![], target: c.target }); - } - } else if let Some((value, _, else_bb)) = targets.as_static_if() - && target_bb == else_bb - { - let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else { return }; + // Attempt to fulfill a condition using an outgoing branch's condition. + // Only support the case where there are no duplicated outgoing edges. + if targets.is_distinct() { + for &(index, c) in state.active.iter() { + if c.place != discr_idx { + continue; + } + + // Set of blocks `t` such that the edge `bb -> t` fulfills `c`. + let mut edges_fulfilling_condition = FxHashSet::default(); + + // On edge `bb -> tgt`, we know that `discr_idx == branch`. + for (branch, tgt) in targets.iter() { + if let Some(branch) = ScalarInt::try_from_uint(branch, discr_layout.size) + && c.matches(discr_idx, branch) + { + edges_fulfilling_condition.insert(tgt); + } + } + + // On edge `bb -> otherwise`, we only know that `discr` is different from all the + // constants in the switch. That's much weaker information than the equality we + // had in the previous arm. All we can conclude is that the replacement condition + // `discr != value` can be threaded, and nothing else. + if c.polarity == Polarity::Ne + && let Ok(value) = c.value.try_to_bits(discr_layout.size) + && targets.all_values().contains(&value.into()) + { + edges_fulfilling_condition.insert(targets.otherwise()); + } + + // Register that jumping to a `t` fulfills condition `c`. + // This does *not* mean that `c` is fulfilled in this block: inserting `index` in + // `fulfilled` is wrong if we have targets that jump to other blocks. + let condition_targets = &state.targets[index]; + + let new_edges: Vec<_> = condition_targets + .iter() + .copied() + .filter(|&target| match target { + EdgeEffect::Goto { .. } => false, + EdgeEffect::Chain { succ_block, .. } => { + edges_fulfilling_condition.contains(&succ_block) + } + }) + .collect(); + + if new_edges.len() == condition_targets.len() { + // If `new_edges == condition_targets`, do not bother creating a new + // `ConditionIndex`, we can use the existing one. + state.fulfilled.push(index); + } else { + // Fulfilling `index` may thread conditions that we do not want, + // so create a brand new index to immediately mark fulfilled. + let index = state.targets.push(new_edges); + state.fulfilled.push(index); + } + } + } - // We only know that `discr != value`. That's much weaker information than - // the equality we had in the previous arm. All we can conclude is that - // the replacement condition `discr != value` can be threaded, and nothing else. - for c in conditions.iter() { - if c.value == value && c.polarity == Polarity::Ne { - debug!(?target_bb, ?c.target, "register"); - self.opportunities - .push(ThreadingOpportunity { chain: vec![], target: c.target }); + // Introduce additional conditions of the form `discr ?= value` for each value in targets. + let mut mk_condition = |value, polarity, target| { + let c = Condition { place: discr_idx, value, polarity }; + state.push_condition(c, target); + }; + if let Some((value, then_, else_)) = targets.as_static_if() { + // We have an `if`, generate both `discr == value` and `discr != value`. + let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else { return }; + mk_condition(value, Polarity::Eq, then_); + mk_condition(value, Polarity::Ne, else_); + } else { + // We have a general switch and we cannot express `discr != value0 && discr != value1`, + // so we only generate equality predicates. + for (value, target) in targets.iter() { + if let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) { + mk_condition(value, Polarity::Eq, target); } } } } } -struct OpportunitySet { - opportunities: Vec, - /// For each bb, give the TOs in which it appears. The pair corresponds to the index - /// in `opportunities` and the index in `ThreadingOpportunity::chain`. - involving_tos: IndexVec>, - /// Cache the number of predecessors for each block, as we clear the basic block cache.. - predecessors: IndexVec, +/// Propagate fulfilled conditions forward in the CFG to reduce the amount of duplication. +#[instrument(level = "debug", skip(body, entry_states))] +fn simplify_conditions(body: &Body<'_>, entry_states: &mut IndexVec) { + let basic_blocks = &body.basic_blocks; + let reverse_postorder = basic_blocks.reverse_postorder(); + + // Start by computing the number of *incoming edges* for each block. + // We do not use the cached `basic_blocks.predecessors` as we only want reachable predecessors. + let mut predecessors = IndexVec::from_elem(0, &entry_states); + predecessors[START_BLOCK] = 1; // Account for the implicit entry edge. + for &bb in reverse_postorder { + let term = basic_blocks[bb].terminator(); + for s in term.successors() { + predecessors[s] += 1; + } + } + + // Compute the number of edges into each block that carry each condition. + let mut fulfill_in_pred_count = IndexVec::from_fn_n( + |bb: BasicBlock| IndexVec::from_elem_n(0, entry_states[bb].targets.len()), + entry_states.len(), + ); + + // By traversing in RPO, we increase the likelihood to visit predecessors before successors. + for &bb in reverse_postorder { + let preds = predecessors[bb]; + trace!(?bb, ?preds); + + // We have removed all the input edges towards this block. Just skip visiting it. + if preds == 0 { + continue; + } + + let state = &mut entry_states[bb]; + trace!(?state); + + // Conditions that are fulfilled in all the predecessors, are fulfilled in `bb`. + trace!(fulfilled_count = ?fulfill_in_pred_count[bb]); + for (condition, &cond_preds) in fulfill_in_pred_count[bb].iter_enumerated() { + if cond_preds == preds { + trace!(?condition); + state.fulfilled.push(condition); + } + } + + // We want to count how many times each condition is fulfilled, + // so ensure we are not counting the same edge twice. + let mut targets: Vec<_> = state + .fulfilled + .iter() + .flat_map(|&index| state.targets[index].iter().copied()) + .collect(); + targets.sort(); + targets.dedup(); + trace!(?targets); + + // We may modify the set of successors by applying edges, so track them here. + let mut successors = basic_blocks[bb].terminator().successors().collect::>(); + + targets.reverse(); + while let Some(target) = targets.pop() { + match target { + EdgeEffect::Goto { target } => { + // We update the count of predecessors. If target or any successor has not been + // processed yet, this increases the likelihood we find something relevant. + predecessors[target] += 1; + for &s in successors.iter() { + predecessors[s] -= 1; + } + // Only process edges that still exist. + targets.retain(|t| t.block() == target); + successors.clear(); + successors.push(target); + } + EdgeEffect::Chain { succ_block, succ_condition } => { + // `predecessors` is the number of incoming *edges* in each block. + // Count the number of edges that apply `succ_condition` into `succ_block`. + let count = successors.iter().filter(|&&s| s == succ_block).count(); + fulfill_in_pred_count[succ_block][succ_condition] += count; + } + } + } + } } -impl OpportunitySet { - fn new(body: &Body<'_>, opportunities: Vec) -> OpportunitySet { - let mut involving_tos = IndexVec::from_elem(Vec::new(), &body.basic_blocks); - for (index, to) in opportunities.iter().enumerate() { - for (ibb, &bb) in to.chain.iter().enumerate() { - involving_tos[bb].push((index, ibb)); +#[instrument(level = "debug", skip(tcx, typing_env, body, entry_states))] +fn remove_costly_conditions<'tcx>( + tcx: TyCtxt<'tcx>, + typing_env: ty::TypingEnv<'tcx>, + body: &Body<'tcx>, + entry_states: &mut IndexVec, +) { + let basic_blocks = &body.basic_blocks; + + let mut costs = IndexVec::from_elem(None, basic_blocks); + let mut cost = |bb: BasicBlock| -> u8 { + let c = *costs[bb].get_or_insert_with(|| { + let bbdata = &basic_blocks[bb]; + let mut cost = CostChecker::new(tcx, typing_env, None, body); + cost.visit_basic_block_data(bb, bbdata); + cost.cost().try_into().unwrap_or(MAX_COST) + }); + trace!("cost[{bb:?}] = {c}"); + c + }; + + // Initialize costs with `MAX_COST`: if we have a cycle, the cyclic `bb` has infinite costs. + let mut condition_cost = IndexVec::from_fn_n( + |bb: BasicBlock| IndexVec::from_elem_n(MAX_COST, entry_states[bb].targets.len()), + entry_states.len(), + ); + + let reverse_postorder = basic_blocks.reverse_postorder(); + + for &bb in reverse_postorder.iter().rev() { + let state = &entry_states[bb]; + trace!(?bb, ?state); + + let mut current_costs = IndexVec::from_elem(0u8, &state.targets); + + for (condition, targets) in state.targets.iter_enumerated() { + for &target in targets { + match target { + // A `Goto` has cost 0. + EdgeEffect::Goto { .. } => {} + // Chaining into an already-fulfilled condition is nop. + EdgeEffect::Chain { succ_block, succ_condition } + if entry_states[succ_block].fulfilled.contains(&succ_condition) => {} + // When chaining, use `cost[succ_block][succ_condition] + cost(succ_block)`. + EdgeEffect::Chain { succ_block, succ_condition } => { + // Cost associated with duplicating `succ_block`. + let duplication_cost = cost(succ_block); + // Cost associated with the rest of the chain. + let target_cost = + *condition_cost[succ_block].get(succ_condition).unwrap_or(&MAX_COST); + let cost = current_costs[condition] + .saturating_add(duplication_cost) + .saturating_add(target_cost); + trace!(?condition, ?succ_block, ?duplication_cost, ?target_cost); + current_costs[condition] = cost; + } + } } - involving_tos[to.target].push((index, to.chain.len())); } - let predecessors = predecessor_count(body); - OpportunitySet { opportunities, involving_tos, predecessors } + + trace!("condition_cost[{bb:?}] = {:?}", current_costs); + condition_cost[bb] = current_costs; } - /// Apply the opportunities on the graph. - fn apply(&mut self, body: &mut Body<'_>) { - for i in 0..self.opportunities.len() { - self.apply_once(i, body); + trace!(?condition_cost); + + for &bb in reverse_postorder { + for (index, targets) in entry_states[bb].targets.iter_enumerated_mut() { + if condition_cost[bb][index] >= MAX_COST { + trace!(?bb, ?index, ?targets, c = ?condition_cost[bb][index], "remove"); + targets.clear() + } } } +} - #[instrument(level = "trace", skip(self, body))] - fn apply_once(&mut self, index: usize, body: &mut Body<'_>) { - debug!(?self.predecessors); - debug!(?self.involving_tos); +struct OpportunitySet<'a, 'tcx> { + basic_blocks: &'a mut IndexVec>, + entry_states: IndexVec, + /// Cache duplicated block. When cloning a basic block `bb` to fulfill a condition `c`, + /// record the target of this `bb with c` edge. + duplicates: FxHashMap<(BasicBlock, ConditionIndex), BasicBlock>, +} - // Check that `predecessors` satisfies its invariant. - debug_assert_eq!(self.predecessors, predecessor_count(body)); +impl<'a, 'tcx> OpportunitySet<'a, 'tcx> { + fn new( + body: &'a mut Body<'tcx>, + mut entry_states: IndexVec, + ) -> Option> { + trace!(def_id = ?body.source.def_id(), "apply"); - // Remove the TO from the vector to allow modifying the other ones later. - let op = &mut self.opportunities[index]; - debug!(?op); - let op_chain = std::mem::take(&mut op.chain); - let op_target = op.target; - debug_assert_eq!(op_chain.len(), op_chain.iter().collect::>().len()); + if entry_states.iter().all(|state| state.fulfilled.is_empty()) { + return None; + } - let Some((current, chain)) = op_chain.split_first() else { return }; + // Free some memory, because we will need to clone condition sets. + for state in entry_states.iter_mut() { + state.active = Default::default(); + } + let duplicates = Default::default(); let basic_blocks = body.basic_blocks.as_mut(); + Some(OpportunitySet { basic_blocks, entry_states, duplicates }) + } - // Invariant: the control-flow is well-formed at the end of each iteration. - let mut current = *current; - for &succ in chain { - debug!(?current, ?succ); + /// Apply the opportunities on the graph. + #[instrument(level = "debug", skip(self))] + fn apply(mut self) { + let mut worklist = Vec::with_capacity(self.basic_blocks.len()); + worklist.push(START_BLOCK); - // `succ` must be a successor of `current`. If it is not, this means this TO is not - // satisfiable and a previous TO erased this edge, so we bail out. - if !basic_blocks[current].terminator().successors().any(|s| s == succ) { - debug!("impossible"); - return; - } + // Use a `GrowableBitSet` and not a `DenseBitSet` as we are adding blocks. + let mut visited = GrowableBitSet::with_capacity(self.basic_blocks.len()); - // Fast path: `succ` is only used once, so we can reuse it directly. - if self.predecessors[succ] == 1 { - debug!("single"); - current = succ; + while let Some(bb) = worklist.pop() { + if !visited.insert(bb) { continue; } - let new_succ = basic_blocks.push(basic_blocks[succ].clone()); - debug!(?new_succ); + self.apply_once(bb); - // Replace `succ` by `new_succ` where it appears. - let mut num_edges = 0; - basic_blocks[current].terminator_mut().successors_mut(|s| { - if *s == succ { - *s = new_succ; - num_edges += 1; - } - }); - - // Update predecessors with the new block. - let _new_succ = self.predecessors.push(num_edges); - debug_assert_eq!(new_succ, _new_succ); - self.predecessors[succ] -= num_edges; - self.update_predecessor_count(basic_blocks[new_succ].terminator(), Update::Incr); - - // Replace the `current -> succ` edge by `current -> new_succ` in all the following - // TOs. This is necessary to avoid trying to thread through a non-existing edge. We - // use `involving_tos` here to avoid traversing the full set of TOs on each iteration. - let mut new_involved = Vec::new(); - for &(to_index, in_to_index) in &self.involving_tos[current] { - // That TO has already been applied, do nothing. - if to_index <= index { - continue; - } + // `apply_once` may have modified the terminator of `bb`. + // Only visit actual successors. + worklist.extend(self.basic_blocks[bb].terminator().successors()); + } + } - let other_to = &mut self.opportunities[to_index]; - if other_to.chain.get(in_to_index) != Some(¤t) { - continue; + /// Apply the opportunities on `bb`. + #[instrument(level = "debug", skip(self))] + fn apply_once(&mut self, bb: BasicBlock) { + let state = &mut self.entry_states[bb]; + trace!(?state); + + // We are modifying the `bb` in-place. Once a `EdgeEffect` has been applied, + // it does not need to be applied again. + let mut targets: Vec<_> = state + .fulfilled + .iter() + .flat_map(|&index| std::mem::take(&mut state.targets[index])) + .collect(); + targets.sort(); + targets.dedup(); + trace!(?targets); + + // Use a while-pop to allow modifying `targets` from inside the loop. + targets.reverse(); + while let Some(target) = targets.pop() { + debug!(?target); + trace!(term = ?self.basic_blocks[bb].terminator().kind); + + // By construction, `target.block()` is a successor of `bb`. + // When applying targets, we may change the set of successors. + // The match below updates the set of targets for consistency. + debug_assert!( + self.basic_blocks[bb].terminator().successors().contains(&target.block()), + "missing {target:?} in successors for {bb:?}, term={:?}", + self.basic_blocks[bb].terminator(), + ); + + match target { + EdgeEffect::Goto { target } => { + self.apply_goto(bb, target); + + // We now have `target` as single successor. Drop all other target blocks. + targets.retain(|t| t.block() == target); + // Also do this on targets that may be applied by a duplicate of `bb`. + for ts in self.entry_states[bb].targets.iter_mut() { + ts.retain(|t| t.block() == target); + } } - let s = other_to.chain.get_mut(in_to_index + 1).unwrap_or(&mut other_to.target); - if *s == succ { - // `other_to` references the `current -> succ` edge, so replace `succ`. - *s = new_succ; - new_involved.push((to_index, in_to_index + 1)); + EdgeEffect::Chain { succ_block, succ_condition } => { + let new_succ_block = self.apply_chain(bb, succ_block, succ_condition); + + // We have a new name for `target`, ensure it is correctly applied. + if let Some(new_succ_block) = new_succ_block { + for t in targets.iter_mut() { + t.replace_block(succ_block, new_succ_block) + } + // Also do this on targets that may be applied by a duplicate of `bb`. + for t in + self.entry_states[bb].targets.iter_mut().flat_map(|ts| ts.iter_mut()) + { + t.replace_block(succ_block, new_succ_block) + } + } } } - // The TOs that we just updated now reference `new_succ`. Update `involving_tos` - // in case we need to duplicate an edge starting at `new_succ` later. - let _new_succ = self.involving_tos.push(new_involved); - debug_assert_eq!(new_succ, _new_succ); - - current = new_succ; + trace!(post_term = ?self.basic_blocks[bb].terminator().kind); } + } - let current = &mut basic_blocks[current]; - self.update_predecessor_count(current.terminator(), Update::Decr); - current.terminator_mut().kind = TerminatorKind::Goto { target: op_target }; - self.predecessors[op_target] += 1; + #[instrument(level = "debug", skip(self))] + fn apply_goto(&mut self, bb: BasicBlock, target: BasicBlock) { + self.basic_blocks[bb].terminator_mut().kind = TerminatorKind::Goto { target }; } - fn update_predecessor_count(&mut self, terminator: &Terminator<'_>, incr: Update) { - match incr { - Update::Incr => { - for s in terminator.successors() { - self.predecessors[s] += 1; - } - } - Update::Decr => { - for s in terminator.successors() { - self.predecessors[s] -= 1; - } - } + #[instrument(level = "debug", skip(self), ret)] + fn apply_chain( + &mut self, + bb: BasicBlock, + target: BasicBlock, + condition: ConditionIndex, + ) -> Option { + if self.entry_states[target].fulfilled.contains(&condition) { + // `target` already fulfills `condition`, so we do not need to thread anything. + trace!("fulfilled"); + return None; } - } -} -fn predecessor_count(body: &Body<'_>) -> IndexVec { - let mut predecessors: IndexVec<_, _> = - body.basic_blocks.predecessors().iter().map(|ps| ps.len()).collect(); - predecessors[START_BLOCK] += 1; // Account for the implicit entry edge. - predecessors -} + // We may be tempted to modify `target` in-place to avoid a clone. This is wrong. + // We may still have edges from other blocks to `target` that have not been created yet. + // For instance because we may be threading an edge coming from `bb`, + // or `target` may be a block duplicate for which we may still create predecessors. + + let new_target = *self.duplicates.entry((target, condition)).or_insert_with(|| { + // If we already have a duplicate of `target` which fulfills `condition`, reuse it. + // Otherwise, we clone a new bb to such ends. + let new_target = self.basic_blocks.push(self.basic_blocks[target].clone()); + trace!(?target, ?new_target, ?condition, "clone"); + + // By definition, `new_target` fulfills the same condition as `target`, with + // `condition` added. + let mut condition_set = self.entry_states[target].clone(); + condition_set.fulfilled.push(condition); + let _new_target = self.entry_states.push(condition_set); + debug_assert_eq!(new_target, _new_target); + + new_target + }); + trace!(?target, ?new_target, ?condition, "reuse"); + + // Replace `target` by `new_target` where it appears. + // This changes exactly `direct_count` edges. + self.basic_blocks[bb].terminator_mut().successors_mut(|s| { + if *s == target { + *s = new_target; + } + }); -enum Update { - Incr, - Decr, + Some(new_target) + } } /// Compute the set of loop headers in the given body. A loop header is usually defined as a block diff --git a/tests/coverage/conditions.cov-map b/tests/coverage/conditions.cov-map index 29d9604085ede..0047d52c89fab 100644 --- a/tests/coverage/conditions.cov-map +++ b/tests/coverage/conditions.cov-map @@ -1,95 +1,75 @@ Function name: conditions::main -Raw bytes (656): 0x[01, 01, 57, 05, 09, 01, 05, 09, 5d, 09, 27, 5d, 61, 27, 65, 5d, 61, 09, 23, 27, 65, 5d, 61, 01, 03, 03, 0d, 11, 51, 11, 4f, 51, 55, 4f, 59, 51, 55, 11, 4b, 4f, 59, 51, 55, 03, 9f, 01, 0d, 11, 0d, 11, 0d, 11, 0d, 11, 0d, 11, 0d, 11, 0d, 11, 9f, 01, 15, 0d, 11, 19, 45, 19, 97, 01, 45, 49, 97, 01, 4d, 45, 49, 19, 93, 01, 97, 01, 4d, 45, 49, 9f, 01, 9b, 02, 0d, 11, 15, 19, 15, 19, 15, 19, 15, 19, 15, 19, 1d, 21, 15, 19, 9b, 02, 1d, 15, 19, 21, 39, 21, e3, 01, 39, 3d, e3, 01, 41, 39, 3d, 21, df, 01, e3, 01, 41, 39, 3d, 9b, 02, d7, 02, 15, 19, 1d, 21, 9b, 02, d7, 02, 15, 19, 1d, 21, 9b, 02, d7, 02, 15, 19, 1d, 21, 9b, 02, d7, 02, 15, 19, 1d, 21, 9b, 02, d7, 02, 15, 19, 1d, 21, 25, 29, 1d, 21, d7, 02, 25, 1d, 21, 29, 2d, 29, cf, 02, 2d, 31, cf, 02, 35, 2d, 31, 29, cb, 02, cf, 02, 35, 2d, 31, d7, 02, db, 02, 1d, 21, 25, 29, 53, 01, 03, 01, 00, 0a, 01, 01, 09, 00, 16, 01, 00, 19, 00, 1a, 01, 01, 08, 00, 0c, 01, 00, 0d, 02, 06, 00, 02, 05, 00, 06, 03, 03, 09, 00, 0a, 01, 00, 10, 00, 1d, 05, 01, 09, 00, 17, 05, 01, 09, 00, 0a, 06, 01, 0f, 00, 1c, 09, 01, 0c, 00, 19, 0a, 00, 1d, 00, 2a, 0e, 00, 2e, 00, 3c, 23, 00, 3d, 02, 0a, 1e, 02, 09, 00, 0a, 09, 01, 09, 00, 17, 09, 01, 09, 00, 12, 2a, 02, 09, 00, 0f, 03, 03, 09, 00, 16, 03, 00, 19, 00, 1a, 03, 01, 08, 00, 0c, 03, 00, 0d, 02, 06, 00, 02, 05, 00, 06, 03, 02, 08, 00, 15, 0d, 00, 16, 02, 06, 2e, 02, 0f, 00, 1c, 11, 01, 0c, 00, 19, 32, 00, 1d, 00, 2a, 36, 00, 2e, 00, 3c, 4b, 00, 3d, 02, 0a, 46, 02, 09, 00, 0a, 11, 01, 09, 00, 17, 52, 02, 09, 00, 0f, 9f, 01, 03, 08, 00, 0c, 9f, 01, 01, 0d, 00, 1a, 9f, 01, 00, 1d, 00, 1e, 9f, 01, 01, 0c, 00, 10, 9f, 01, 00, 11, 02, 0a, 00, 02, 09, 00, 0a, 9f, 01, 02, 0c, 00, 19, 15, 00, 1a, 02, 0a, 72, 04, 11, 00, 1e, 19, 01, 10, 00, 1d, 7a, 00, 21, 00, 2e, 7e, 00, 32, 00, 40, 93, 01, 00, 41, 02, 0e, 8e, 01, 02, 0d, 00, 0e, 19, 01, 0d, 00, 1b, 9a, 01, 02, 0d, 00, 13, 00, 02, 05, 00, 06, 9b, 02, 02, 09, 00, 16, 9b, 02, 00, 19, 00, 1a, 9b, 02, 01, 08, 00, 0c, 9b, 02, 00, 0d, 02, 06, 00, 02, 05, 00, 06, d7, 02, 02, 09, 00, 0a, 9b, 02, 00, 10, 00, 1d, 1d, 00, 1e, 02, 06, be, 01, 02, 0f, 00, 1c, 21, 01, 0c, 00, 19, c6, 01, 00, 1d, 00, 2a, ca, 01, 00, 2e, 00, 3c, df, 01, 00, 3d, 02, 0a, da, 01, 02, 09, 00, 0a, 21, 01, 09, 00, 17, 96, 02, 02, 0d, 00, 20, 96, 02, 00, 23, 00, 2c, 96, 02, 01, 09, 00, 11, 96, 02, 00, 12, 00, 1b, 96, 02, 01, 09, 00, 0f, db, 02, 03, 09, 00, 0a, d7, 02, 00, 10, 00, 1d, 25, 00, 1e, 02, 06, aa, 02, 02, 0f, 00, 1c, 29, 01, 0c, 00, 19, b2, 02, 00, 1d, 00, 2a, b6, 02, 00, 2e, 00, 3c, cb, 02, 00, 3d, 02, 0a, c6, 02, 02, 09, 00, 0a, 29, 01, 09, 00, 17, d2, 02, 02, 09, 00, 0f, 01, 02, 01, 00, 02] +Raw bytes (595): 0x[01, 01, 43, 05, 09, 01, 05, 09, 51, 09, 13, 51, 55, 01, 03, 03, 0d, 11, 49, 11, 27, 49, 4d, 03, 63, 0d, 11, 0d, 11, 0d, 11, 0d, 11, 0d, 11, 0d, 11, 0d, 11, 63, 15, 0d, 11, 19, 41, 19, 5b, 41, 45, 63, cb, 01, 0d, 11, 15, 19, 15, 19, 15, 19, 15, 19, 15, 19, 1d, 21, 15, 19, cb, 01, 1d, 15, 19, 21, 39, 21, 93, 01, 39, 3d, cb, 01, 87, 02, 15, 19, 1d, 21, cb, 01, 87, 02, 15, 19, 1d, 21, cb, 01, 87, 02, 15, 19, 1d, 21, cb, 01, 87, 02, 15, 19, 1d, 21, cb, 01, 87, 02, 15, 19, 1d, 21, 25, 29, 1d, 21, 87, 02, 25, 1d, 21, 29, 2d, 29, ff, 01, 2d, 31, ff, 01, 35, 2d, 31, 29, fb, 01, ff, 01, 35, 2d, 31, 87, 02, 8b, 02, 1d, 21, 25, 29, 53, 01, 03, 01, 00, 0a, 01, 01, 09, 00, 16, 01, 00, 19, 00, 1a, 01, 01, 08, 00, 0c, 01, 00, 0d, 02, 06, 00, 02, 05, 00, 06, 03, 03, 09, 00, 0a, 01, 00, 10, 00, 1d, 05, 01, 09, 00, 17, 05, 01, 09, 00, 0a, 06, 01, 0f, 00, 1c, 09, 01, 0c, 00, 19, 0a, 00, 1d, 00, 2a, 0e, 00, 2e, 00, 3c, 09, 00, 3d, 02, 0a, 00, 02, 09, 00, 0a, 09, 01, 09, 00, 17, 09, 01, 09, 00, 12, 16, 02, 09, 00, 0f, 03, 03, 09, 00, 16, 03, 00, 19, 00, 1a, 03, 01, 08, 00, 0c, 03, 00, 0d, 02, 06, 00, 02, 05, 00, 06, 03, 02, 08, 00, 15, 0d, 00, 16, 02, 06, 1a, 02, 0f, 00, 1c, 11, 01, 0c, 00, 19, 1e, 00, 1d, 00, 2a, 22, 00, 2e, 00, 3c, 11, 00, 3d, 02, 0a, 00, 02, 09, 00, 0a, 11, 01, 09, 00, 17, 2a, 02, 09, 00, 0f, 63, 03, 08, 00, 0c, 63, 01, 0d, 00, 1a, 63, 00, 1d, 00, 1e, 63, 01, 0c, 00, 10, 63, 00, 11, 02, 0a, 00, 02, 09, 00, 0a, 63, 02, 0c, 00, 19, 15, 00, 1a, 02, 0a, 4a, 04, 11, 00, 1e, 19, 01, 10, 00, 1d, 52, 00, 21, 00, 2e, 56, 00, 32, 00, 40, 19, 00, 41, 02, 0e, 00, 02, 0d, 00, 0e, 19, 01, 0d, 00, 1b, 5e, 02, 0d, 00, 13, 00, 02, 05, 00, 06, cb, 01, 02, 09, 00, 16, cb, 01, 00, 19, 00, 1a, cb, 01, 01, 08, 00, 0c, cb, 01, 00, 0d, 02, 06, 00, 02, 05, 00, 06, 87, 02, 02, 09, 00, 0a, cb, 01, 00, 10, 00, 1d, 1d, 00, 1e, 02, 06, 82, 01, 02, 0f, 00, 1c, 21, 01, 0c, 00, 19, 8a, 01, 00, 1d, 00, 2a, 8e, 01, 00, 2e, 00, 3c, 21, 00, 3d, 02, 0a, 00, 02, 09, 00, 0a, 21, 01, 09, 00, 17, c6, 01, 02, 0d, 00, 20, c6, 01, 00, 23, 00, 2c, c6, 01, 01, 09, 00, 11, c6, 01, 00, 12, 00, 1b, c6, 01, 01, 09, 00, 0f, 8b, 02, 03, 09, 00, 0a, 87, 02, 00, 10, 00, 1d, 25, 00, 1e, 02, 06, da, 01, 02, 0f, 00, 1c, 29, 01, 0c, 00, 19, e2, 01, 00, 1d, 00, 2a, e6, 01, 00, 2e, 00, 3c, fb, 01, 00, 3d, 02, 0a, f6, 01, 02, 09, 00, 0a, 29, 01, 09, 00, 17, 82, 02, 02, 09, 00, 0f, 01, 02, 01, 00, 02] Number of files: 1 - file 0 => $DIR/conditions.rs -Number of expressions: 87 +Number of expressions: 67 - expression 0 operands: lhs = Counter(1), rhs = Counter(2) - expression 1 operands: lhs = Counter(0), rhs = Counter(1) -- expression 2 operands: lhs = Counter(2), rhs = Counter(23) -- expression 3 operands: lhs = Counter(2), rhs = Expression(9, Add) -- expression 4 operands: lhs = Counter(23), rhs = Counter(24) -- expression 5 operands: lhs = Expression(9, Add), rhs = Counter(25) -- expression 6 operands: lhs = Counter(23), rhs = Counter(24) -- expression 7 operands: lhs = Counter(2), rhs = Expression(8, Add) -- expression 8 operands: lhs = Expression(9, Add), rhs = Counter(25) -- expression 9 operands: lhs = Counter(23), rhs = Counter(24) -- expression 10 operands: lhs = Counter(0), rhs = Expression(0, Add) -- expression 11 operands: lhs = Expression(0, Add), rhs = Counter(3) -- expression 12 operands: lhs = Counter(4), rhs = Counter(20) -- expression 13 operands: lhs = Counter(4), rhs = Expression(19, Add) -- expression 14 operands: lhs = Counter(20), rhs = Counter(21) -- expression 15 operands: lhs = Expression(19, Add), rhs = Counter(22) -- expression 16 operands: lhs = Counter(20), rhs = Counter(21) -- expression 17 operands: lhs = Counter(4), rhs = Expression(18, Add) -- expression 18 operands: lhs = Expression(19, Add), rhs = Counter(22) -- expression 19 operands: lhs = Counter(20), rhs = Counter(21) -- expression 20 operands: lhs = Expression(0, Add), rhs = Expression(39, Add) -- expression 21 operands: lhs = Counter(3), rhs = Counter(4) -- expression 22 operands: lhs = Counter(3), rhs = Counter(4) -- expression 23 operands: lhs = Counter(3), rhs = Counter(4) +- expression 2 operands: lhs = Counter(2), rhs = Counter(20) +- expression 3 operands: lhs = Counter(2), rhs = Expression(4, Add) +- expression 4 operands: lhs = Counter(20), rhs = Counter(21) +- expression 5 operands: lhs = Counter(0), rhs = Expression(0, Add) +- expression 6 operands: lhs = Expression(0, Add), rhs = Counter(3) +- expression 7 operands: lhs = Counter(4), rhs = Counter(18) +- expression 8 operands: lhs = Counter(4), rhs = Expression(9, Add) +- expression 9 operands: lhs = Counter(18), rhs = Counter(19) +- expression 10 operands: lhs = Expression(0, Add), rhs = Expression(24, Add) +- expression 11 operands: lhs = Counter(3), rhs = Counter(4) +- expression 12 operands: lhs = Counter(3), rhs = Counter(4) +- expression 13 operands: lhs = Counter(3), rhs = Counter(4) +- expression 14 operands: lhs = Counter(3), rhs = Counter(4) +- expression 15 operands: lhs = Counter(3), rhs = Counter(4) +- expression 16 operands: lhs = Counter(3), rhs = Counter(4) +- expression 17 operands: lhs = Counter(3), rhs = Counter(4) +- expression 18 operands: lhs = Expression(24, Add), rhs = Counter(5) +- expression 19 operands: lhs = Counter(3), rhs = Counter(4) +- expression 20 operands: lhs = Counter(6), rhs = Counter(16) +- expression 21 operands: lhs = Counter(6), rhs = Expression(22, Add) +- expression 22 operands: lhs = Counter(16), rhs = Counter(17) +- expression 23 operands: lhs = Expression(24, Add), rhs = Expression(50, Add) - expression 24 operands: lhs = Counter(3), rhs = Counter(4) -- expression 25 operands: lhs = Counter(3), rhs = Counter(4) -- expression 26 operands: lhs = Counter(3), rhs = Counter(4) -- expression 27 operands: lhs = Counter(3), rhs = Counter(4) -- expression 28 operands: lhs = Expression(39, Add), rhs = Counter(5) -- expression 29 operands: lhs = Counter(3), rhs = Counter(4) -- expression 30 operands: lhs = Counter(6), rhs = Counter(17) -- expression 31 operands: lhs = Counter(6), rhs = Expression(37, Add) -- expression 32 operands: lhs = Counter(17), rhs = Counter(18) -- expression 33 operands: lhs = Expression(37, Add), rhs = Counter(19) -- expression 34 operands: lhs = Counter(17), rhs = Counter(18) -- expression 35 operands: lhs = Counter(6), rhs = Expression(36, Add) -- expression 36 operands: lhs = Expression(37, Add), rhs = Counter(19) -- expression 37 operands: lhs = Counter(17), rhs = Counter(18) -- expression 38 operands: lhs = Expression(39, Add), rhs = Expression(70, Add) -- expression 39 operands: lhs = Counter(3), rhs = Counter(4) -- expression 40 operands: lhs = Counter(5), rhs = Counter(6) +- expression 25 operands: lhs = Counter(5), rhs = Counter(6) +- expression 26 operands: lhs = Counter(5), rhs = Counter(6) +- expression 27 operands: lhs = Counter(5), rhs = Counter(6) +- expression 28 operands: lhs = Counter(5), rhs = Counter(6) +- expression 29 operands: lhs = Counter(5), rhs = Counter(6) +- expression 30 operands: lhs = Counter(7), rhs = Counter(8) +- expression 31 operands: lhs = Counter(5), rhs = Counter(6) +- expression 32 operands: lhs = Expression(50, Add), rhs = Counter(7) +- expression 33 operands: lhs = Counter(5), rhs = Counter(6) +- expression 34 operands: lhs = Counter(8), rhs = Counter(14) +- expression 35 operands: lhs = Counter(8), rhs = Expression(36, Add) +- expression 36 operands: lhs = Counter(14), rhs = Counter(15) +- expression 37 operands: lhs = Expression(50, Add), rhs = Expression(65, Add) +- expression 38 operands: lhs = Counter(5), rhs = Counter(6) +- expression 39 operands: lhs = Counter(7), rhs = Counter(8) +- expression 40 operands: lhs = Expression(50, Add), rhs = Expression(65, Add) - expression 41 operands: lhs = Counter(5), rhs = Counter(6) -- expression 42 operands: lhs = Counter(5), rhs = Counter(6) -- expression 43 operands: lhs = Counter(5), rhs = Counter(6) +- expression 42 operands: lhs = Counter(7), rhs = Counter(8) +- expression 43 operands: lhs = Expression(50, Add), rhs = Expression(65, Add) - expression 44 operands: lhs = Counter(5), rhs = Counter(6) - expression 45 operands: lhs = Counter(7), rhs = Counter(8) -- expression 46 operands: lhs = Counter(5), rhs = Counter(6) -- expression 47 operands: lhs = Expression(70, Add), rhs = Counter(7) -- expression 48 operands: lhs = Counter(5), rhs = Counter(6) -- expression 49 operands: lhs = Counter(8), rhs = Counter(14) -- expression 50 operands: lhs = Counter(8), rhs = Expression(56, Add) -- expression 51 operands: lhs = Counter(14), rhs = Counter(15) -- expression 52 operands: lhs = Expression(56, Add), rhs = Counter(16) -- expression 53 operands: lhs = Counter(14), rhs = Counter(15) -- expression 54 operands: lhs = Counter(8), rhs = Expression(55, Add) -- expression 55 operands: lhs = Expression(56, Add), rhs = Counter(16) -- expression 56 operands: lhs = Counter(14), rhs = Counter(15) -- expression 57 operands: lhs = Expression(70, Add), rhs = Expression(85, Add) -- expression 58 operands: lhs = Counter(5), rhs = Counter(6) -- expression 59 operands: lhs = Counter(7), rhs = Counter(8) -- expression 60 operands: lhs = Expression(70, Add), rhs = Expression(85, Add) -- expression 61 operands: lhs = Counter(5), rhs = Counter(6) -- expression 62 operands: lhs = Counter(7), rhs = Counter(8) -- expression 63 operands: lhs = Expression(70, Add), rhs = Expression(85, Add) -- expression 64 operands: lhs = Counter(5), rhs = Counter(6) +- expression 46 operands: lhs = Expression(50, Add), rhs = Expression(65, Add) +- expression 47 operands: lhs = Counter(5), rhs = Counter(6) +- expression 48 operands: lhs = Counter(7), rhs = Counter(8) +- expression 49 operands: lhs = Expression(50, Add), rhs = Expression(65, Add) +- expression 50 operands: lhs = Counter(5), rhs = Counter(6) +- expression 51 operands: lhs = Counter(7), rhs = Counter(8) +- expression 52 operands: lhs = Counter(9), rhs = Counter(10) +- expression 53 operands: lhs = Counter(7), rhs = Counter(8) +- expression 54 operands: lhs = Expression(65, Add), rhs = Counter(9) +- expression 55 operands: lhs = Counter(7), rhs = Counter(8) +- expression 56 operands: lhs = Counter(10), rhs = Counter(11) +- expression 57 operands: lhs = Counter(10), rhs = Expression(63, Add) +- expression 58 operands: lhs = Counter(11), rhs = Counter(12) +- expression 59 operands: lhs = Expression(63, Add), rhs = Counter(13) +- expression 60 operands: lhs = Counter(11), rhs = Counter(12) +- expression 61 operands: lhs = Counter(10), rhs = Expression(62, Add) +- expression 62 operands: lhs = Expression(63, Add), rhs = Counter(13) +- expression 63 operands: lhs = Counter(11), rhs = Counter(12) +- expression 64 operands: lhs = Expression(65, Add), rhs = Expression(66, Add) - expression 65 operands: lhs = Counter(7), rhs = Counter(8) -- expression 66 operands: lhs = Expression(70, Add), rhs = Expression(85, Add) -- expression 67 operands: lhs = Counter(5), rhs = Counter(6) -- expression 68 operands: lhs = Counter(7), rhs = Counter(8) -- expression 69 operands: lhs = Expression(70, Add), rhs = Expression(85, Add) -- expression 70 operands: lhs = Counter(5), rhs = Counter(6) -- expression 71 operands: lhs = Counter(7), rhs = Counter(8) -- expression 72 operands: lhs = Counter(9), rhs = Counter(10) -- expression 73 operands: lhs = Counter(7), rhs = Counter(8) -- expression 74 operands: lhs = Expression(85, Add), rhs = Counter(9) -- expression 75 operands: lhs = Counter(7), rhs = Counter(8) -- expression 76 operands: lhs = Counter(10), rhs = Counter(11) -- expression 77 operands: lhs = Counter(10), rhs = Expression(83, Add) -- expression 78 operands: lhs = Counter(11), rhs = Counter(12) -- expression 79 operands: lhs = Expression(83, Add), rhs = Counter(13) -- expression 80 operands: lhs = Counter(11), rhs = Counter(12) -- expression 81 operands: lhs = Counter(10), rhs = Expression(82, Add) -- expression 82 operands: lhs = Expression(83, Add), rhs = Counter(13) -- expression 83 operands: lhs = Counter(11), rhs = Counter(12) -- expression 84 operands: lhs = Expression(85, Add), rhs = Expression(86, Add) -- expression 85 operands: lhs = Counter(7), rhs = Counter(8) -- expression 86 operands: lhs = Counter(9), rhs = Counter(10) +- expression 66 operands: lhs = Counter(9), rhs = Counter(10) Number of file 0 mappings: 83 - Code(Counter(0)) at (prev + 3, 1) to (start + 0, 10) - Code(Counter(0)) at (prev + 1, 9) to (start + 0, 22) @@ -106,16 +86,14 @@ Number of file 0 mappings: 83 = (c0 - c1) - Code(Counter(2)) at (prev + 1, 12) to (start + 0, 25) - Code(Expression(2, Sub)) at (prev + 0, 29) to (start + 0, 42) - = (c2 - c23) + = (c2 - c20) - Code(Expression(3, Sub)) at (prev + 0, 46) to (start + 0, 60) - = (c2 - (c23 + c24)) -- Code(Expression(8, Add)) at (prev + 0, 61) to (start + 2, 10) - = ((c23 + c24) + c25) -- Code(Expression(7, Sub)) at (prev + 2, 9) to (start + 0, 10) - = (c2 - ((c23 + c24) + c25)) + = (c2 - (c20 + c21)) +- Code(Counter(2)) at (prev + 0, 61) to (start + 2, 10) +- Code(Zero) at (prev + 2, 9) to (start + 0, 10) - Code(Counter(2)) at (prev + 1, 9) to (start + 0, 23) - Code(Counter(2)) at (prev + 1, 9) to (start + 0, 18) -- Code(Expression(10, Sub)) at (prev + 2, 9) to (start + 0, 15) +- Code(Expression(5, Sub)) at (prev + 2, 9) to (start + 0, 15) = (c0 - (c1 + c2)) - Code(Expression(0, Add)) at (prev + 3, 9) to (start + 0, 22) = (c1 + c2) @@ -129,103 +107,97 @@ Number of file 0 mappings: 83 - Code(Expression(0, Add)) at (prev + 2, 8) to (start + 0, 21) = (c1 + c2) - Code(Counter(3)) at (prev + 0, 22) to (start + 2, 6) -- Code(Expression(11, Sub)) at (prev + 2, 15) to (start + 0, 28) +- Code(Expression(6, Sub)) at (prev + 2, 15) to (start + 0, 28) = ((c1 + c2) - c3) - Code(Counter(4)) at (prev + 1, 12) to (start + 0, 25) -- Code(Expression(12, Sub)) at (prev + 0, 29) to (start + 0, 42) - = (c4 - c20) -- Code(Expression(13, Sub)) at (prev + 0, 46) to (start + 0, 60) - = (c4 - (c20 + c21)) -- Code(Expression(18, Add)) at (prev + 0, 61) to (start + 2, 10) - = ((c20 + c21) + c22) -- Code(Expression(17, Sub)) at (prev + 2, 9) to (start + 0, 10) - = (c4 - ((c20 + c21) + c22)) +- Code(Expression(7, Sub)) at (prev + 0, 29) to (start + 0, 42) + = (c4 - c18) +- Code(Expression(8, Sub)) at (prev + 0, 46) to (start + 0, 60) + = (c4 - (c18 + c19)) +- Code(Counter(4)) at (prev + 0, 61) to (start + 2, 10) +- Code(Zero) at (prev + 2, 9) to (start + 0, 10) - Code(Counter(4)) at (prev + 1, 9) to (start + 0, 23) -- Code(Expression(20, Sub)) at (prev + 2, 9) to (start + 0, 15) +- Code(Expression(10, Sub)) at (prev + 2, 9) to (start + 0, 15) = ((c1 + c2) - (c3 + c4)) -- Code(Expression(39, Add)) at (prev + 3, 8) to (start + 0, 12) +- Code(Expression(24, Add)) at (prev + 3, 8) to (start + 0, 12) = (c3 + c4) -- Code(Expression(39, Add)) at (prev + 1, 13) to (start + 0, 26) +- Code(Expression(24, Add)) at (prev + 1, 13) to (start + 0, 26) = (c3 + c4) -- Code(Expression(39, Add)) at (prev + 0, 29) to (start + 0, 30) +- Code(Expression(24, Add)) at (prev + 0, 29) to (start + 0, 30) = (c3 + c4) -- Code(Expression(39, Add)) at (prev + 1, 12) to (start + 0, 16) +- Code(Expression(24, Add)) at (prev + 1, 12) to (start + 0, 16) = (c3 + c4) -- Code(Expression(39, Add)) at (prev + 0, 17) to (start + 2, 10) +- Code(Expression(24, Add)) at (prev + 0, 17) to (start + 2, 10) = (c3 + c4) - Code(Zero) at (prev + 2, 9) to (start + 0, 10) -- Code(Expression(39, Add)) at (prev + 2, 12) to (start + 0, 25) +- Code(Expression(24, Add)) at (prev + 2, 12) to (start + 0, 25) = (c3 + c4) - Code(Counter(5)) at (prev + 0, 26) to (start + 2, 10) -- Code(Expression(28, Sub)) at (prev + 4, 17) to (start + 0, 30) +- Code(Expression(18, Sub)) at (prev + 4, 17) to (start + 0, 30) = ((c3 + c4) - c5) - Code(Counter(6)) at (prev + 1, 16) to (start + 0, 29) -- Code(Expression(30, Sub)) at (prev + 0, 33) to (start + 0, 46) - = (c6 - c17) -- Code(Expression(31, Sub)) at (prev + 0, 50) to (start + 0, 64) - = (c6 - (c17 + c18)) -- Code(Expression(36, Add)) at (prev + 0, 65) to (start + 2, 14) - = ((c17 + c18) + c19) -- Code(Expression(35, Sub)) at (prev + 2, 13) to (start + 0, 14) - = (c6 - ((c17 + c18) + c19)) +- Code(Expression(20, Sub)) at (prev + 0, 33) to (start + 0, 46) + = (c6 - c16) +- Code(Expression(21, Sub)) at (prev + 0, 50) to (start + 0, 64) + = (c6 - (c16 + c17)) +- Code(Counter(6)) at (prev + 0, 65) to (start + 2, 14) +- Code(Zero) at (prev + 2, 13) to (start + 0, 14) - Code(Counter(6)) at (prev + 1, 13) to (start + 0, 27) -- Code(Expression(38, Sub)) at (prev + 2, 13) to (start + 0, 19) +- Code(Expression(23, Sub)) at (prev + 2, 13) to (start + 0, 19) = ((c3 + c4) - (c5 + c6)) - Code(Zero) at (prev + 2, 5) to (start + 0, 6) -- Code(Expression(70, Add)) at (prev + 2, 9) to (start + 0, 22) +- Code(Expression(50, Add)) at (prev + 2, 9) to (start + 0, 22) = (c5 + c6) -- Code(Expression(70, Add)) at (prev + 0, 25) to (start + 0, 26) +- Code(Expression(50, Add)) at (prev + 0, 25) to (start + 0, 26) = (c5 + c6) -- Code(Expression(70, Add)) at (prev + 1, 8) to (start + 0, 12) +- Code(Expression(50, Add)) at (prev + 1, 8) to (start + 0, 12) = (c5 + c6) -- Code(Expression(70, Add)) at (prev + 0, 13) to (start + 2, 6) +- Code(Expression(50, Add)) at (prev + 0, 13) to (start + 2, 6) = (c5 + c6) - Code(Zero) at (prev + 2, 5) to (start + 0, 6) -- Code(Expression(85, Add)) at (prev + 2, 9) to (start + 0, 10) +- Code(Expression(65, Add)) at (prev + 2, 9) to (start + 0, 10) = (c7 + c8) -- Code(Expression(70, Add)) at (prev + 0, 16) to (start + 0, 29) +- Code(Expression(50, Add)) at (prev + 0, 16) to (start + 0, 29) = (c5 + c6) - Code(Counter(7)) at (prev + 0, 30) to (start + 2, 6) -- Code(Expression(47, Sub)) at (prev + 2, 15) to (start + 0, 28) +- Code(Expression(32, Sub)) at (prev + 2, 15) to (start + 0, 28) = ((c5 + c6) - c7) - Code(Counter(8)) at (prev + 1, 12) to (start + 0, 25) -- Code(Expression(49, Sub)) at (prev + 0, 29) to (start + 0, 42) +- Code(Expression(34, Sub)) at (prev + 0, 29) to (start + 0, 42) = (c8 - c14) -- Code(Expression(50, Sub)) at (prev + 0, 46) to (start + 0, 60) +- Code(Expression(35, Sub)) at (prev + 0, 46) to (start + 0, 60) = (c8 - (c14 + c15)) -- Code(Expression(55, Add)) at (prev + 0, 61) to (start + 2, 10) - = ((c14 + c15) + c16) -- Code(Expression(54, Sub)) at (prev + 2, 9) to (start + 0, 10) - = (c8 - ((c14 + c15) + c16)) +- Code(Counter(8)) at (prev + 0, 61) to (start + 2, 10) +- Code(Zero) at (prev + 2, 9) to (start + 0, 10) - Code(Counter(8)) at (prev + 1, 9) to (start + 0, 23) -- Code(Expression(69, Sub)) at (prev + 2, 13) to (start + 0, 32) +- Code(Expression(49, Sub)) at (prev + 2, 13) to (start + 0, 32) = ((c5 + c6) - (c7 + c8)) -- Code(Expression(69, Sub)) at (prev + 0, 35) to (start + 0, 44) +- Code(Expression(49, Sub)) at (prev + 0, 35) to (start + 0, 44) = ((c5 + c6) - (c7 + c8)) -- Code(Expression(69, Sub)) at (prev + 1, 9) to (start + 0, 17) +- Code(Expression(49, Sub)) at (prev + 1, 9) to (start + 0, 17) = ((c5 + c6) - (c7 + c8)) -- Code(Expression(69, Sub)) at (prev + 0, 18) to (start + 0, 27) +- Code(Expression(49, Sub)) at (prev + 0, 18) to (start + 0, 27) = ((c5 + c6) - (c7 + c8)) -- Code(Expression(69, Sub)) at (prev + 1, 9) to (start + 0, 15) +- Code(Expression(49, Sub)) at (prev + 1, 9) to (start + 0, 15) = ((c5 + c6) - (c7 + c8)) -- Code(Expression(86, Add)) at (prev + 3, 9) to (start + 0, 10) +- Code(Expression(66, Add)) at (prev + 3, 9) to (start + 0, 10) = (c9 + c10) -- Code(Expression(85, Add)) at (prev + 0, 16) to (start + 0, 29) +- Code(Expression(65, Add)) at (prev + 0, 16) to (start + 0, 29) = (c7 + c8) - Code(Counter(9)) at (prev + 0, 30) to (start + 2, 6) -- Code(Expression(74, Sub)) at (prev + 2, 15) to (start + 0, 28) +- Code(Expression(54, Sub)) at (prev + 2, 15) to (start + 0, 28) = ((c7 + c8) - c9) - Code(Counter(10)) at (prev + 1, 12) to (start + 0, 25) -- Code(Expression(76, Sub)) at (prev + 0, 29) to (start + 0, 42) +- Code(Expression(56, Sub)) at (prev + 0, 29) to (start + 0, 42) = (c10 - c11) -- Code(Expression(77, Sub)) at (prev + 0, 46) to (start + 0, 60) +- Code(Expression(57, Sub)) at (prev + 0, 46) to (start + 0, 60) = (c10 - (c11 + c12)) -- Code(Expression(82, Add)) at (prev + 0, 61) to (start + 2, 10) +- Code(Expression(62, Add)) at (prev + 0, 61) to (start + 2, 10) = ((c11 + c12) + c13) -- Code(Expression(81, Sub)) at (prev + 2, 9) to (start + 0, 10) +- Code(Expression(61, Sub)) at (prev + 2, 9) to (start + 0, 10) = (c10 - ((c11 + c12) + c13)) - Code(Counter(10)) at (prev + 1, 9) to (start + 0, 23) -- Code(Expression(84, Sub)) at (prev + 2, 9) to (start + 0, 15) +- Code(Expression(64, Sub)) at (prev + 2, 9) to (start + 0, 15) = ((c7 + c8) - (c9 + c10)) - Code(Counter(0)) at (prev + 2, 1) to (start + 0, 2) Highest counter ID seen: c10 diff --git a/tests/mir-opt/jump_threading.aggregate.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.aggregate.JumpThreading.panic-abort.diff index a7551c3fb5b79..89d04c557f125 100644 --- a/tests/mir-opt/jump_threading.aggregate.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.aggregate.JumpThreading.panic-abort.diff @@ -1,51 +1,50 @@ - // MIR for `aggregate` before JumpThreading + // MIR for `aggregate` after JumpThreading - fn aggregate(_1: u8) -> u8 { - debug x => _1; + fn aggregate() -> u8 { let mut _0: u8; + let _1: u8; let _2: u8; - let _3: u8; - let mut _4: (u8, u8); - let mut _5: bool; - let mut _6: u8; + let mut _3: (u8, u8); + let mut _4: bool; + let mut _5: u8; scope 1 { - debug a => _2; - debug b => _3; + debug a => _1; + debug b => _2; } bb0: { - StorageLive(_4); - _4 = const aggregate::FOO; - StorageLive(_2); - _2 = copy (_4.0: u8); StorageLive(_3); - _3 = copy (_4.1: u8); - StorageDead(_4); + _3 = const aggregate::FOO; + StorageLive(_1); + _1 = copy (_3.0: u8); + StorageLive(_2); + _2 = copy (_3.1: u8); + StorageDead(_3); + StorageLive(_4); StorageLive(_5); - StorageLive(_6); - _6 = copy _2; - _5 = Eq(move _6, const 7_u8); -- switchInt(move _5) -> [0: bb2, otherwise: bb1]; + _5 = copy _1; + _4 = Eq(move _5, const 7_u8); +- switchInt(move _4) -> [0: bb2, otherwise: bb1]; + goto -> bb2; } bb1: { - StorageDead(_6); - _0 = copy _3; + StorageDead(_5); + _0 = copy _2; goto -> bb3; } bb2: { - StorageDead(_6); - _0 = copy _2; + StorageDead(_5); + _0 = copy _1; goto -> bb3; } bb3: { - StorageDead(_5); - StorageDead(_3); + StorageDead(_4); StorageDead(_2); + StorageDead(_1); return; } } diff --git a/tests/mir-opt/jump_threading.aggregate.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.aggregate.JumpThreading.panic-unwind.diff index a7551c3fb5b79..89d04c557f125 100644 --- a/tests/mir-opt/jump_threading.aggregate.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.aggregate.JumpThreading.panic-unwind.diff @@ -1,51 +1,50 @@ - // MIR for `aggregate` before JumpThreading + // MIR for `aggregate` after JumpThreading - fn aggregate(_1: u8) -> u8 { - debug x => _1; + fn aggregate() -> u8 { let mut _0: u8; + let _1: u8; let _2: u8; - let _3: u8; - let mut _4: (u8, u8); - let mut _5: bool; - let mut _6: u8; + let mut _3: (u8, u8); + let mut _4: bool; + let mut _5: u8; scope 1 { - debug a => _2; - debug b => _3; + debug a => _1; + debug b => _2; } bb0: { - StorageLive(_4); - _4 = const aggregate::FOO; - StorageLive(_2); - _2 = copy (_4.0: u8); StorageLive(_3); - _3 = copy (_4.1: u8); - StorageDead(_4); + _3 = const aggregate::FOO; + StorageLive(_1); + _1 = copy (_3.0: u8); + StorageLive(_2); + _2 = copy (_3.1: u8); + StorageDead(_3); + StorageLive(_4); StorageLive(_5); - StorageLive(_6); - _6 = copy _2; - _5 = Eq(move _6, const 7_u8); -- switchInt(move _5) -> [0: bb2, otherwise: bb1]; + _5 = copy _1; + _4 = Eq(move _5, const 7_u8); +- switchInt(move _4) -> [0: bb2, otherwise: bb1]; + goto -> bb2; } bb1: { - StorageDead(_6); - _0 = copy _3; + StorageDead(_5); + _0 = copy _2; goto -> bb3; } bb2: { - StorageDead(_6); - _0 = copy _2; + StorageDead(_5); + _0 = copy _1; goto -> bb3; } bb3: { - StorageDead(_5); - StorageDead(_3); + StorageDead(_4); StorageDead(_2); + StorageDead(_1); return; } } diff --git a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff new file mode 100644 index 0000000000000..f09a187bfaf8d --- /dev/null +++ b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff @@ -0,0 +1,238 @@ +- // MIR for `chained_conditions` before JumpThreading ++ // MIR for `chained_conditions` after JumpThreading + + fn chained_conditions() -> u8 { + let mut _0: u8; + let _1: chained_conditions::BacktraceStyle; + let mut _2: std::option::Option; + let mut _3: &std::option::Option; + let mut _4: isize; + let _5: std::string::String; + let _6: &std::string::String; + let mut _7: bool; + let mut _8: &&std::string::String; + let _9: &std::string::String; + let mut _10: &&str; + let _11: &str; + let _12: std::string::String; + let _13: &std::string::String; + let mut _14: bool; + let mut _15: &&std::string::String; + let _16: &std::string::String; + let mut _17: &&str; + let _18: &str; + let mut _19: isize; + let mut _20: &&str; + let mut _21: &&str; + let mut _22: bool; + let mut _23: bool; + let mut _24: isize; + let mut _25: isize; + let mut _26: isize; + scope 1 { + debug format => _1; + } + scope 2 { + debug x => _5; + debug x => _6; + } + scope 3 { + debug x => _12; + debug x => _13; + } + scope 4 (inlined std::cmp::impls:: for &String>::eq) { + let mut _27: &std::string::String; + let mut _28: &str; + } + scope 5 (inlined std::cmp::impls:: for &String>::eq) { + let mut _29: &std::string::String; + let mut _30: &str; + } + + bb0: { + _22 = const false; + _23 = const false; + StorageLive(_1); + StorageLive(_2); + _2 = env_var() -> [return: bb1, unwind unreachable]; + } + + bb1: { + _22 = const true; + _23 = const true; + _4 = discriminant(_2); + switchInt(move _4) -> [0: bb3, 1: bb4, otherwise: bb2]; + } + + bb2: { + unreachable; + } + + bb3: { + _1 = chained_conditions::BacktraceStyle::Off; +- goto -> bb18; ++ goto -> bb23; + } + + bb4: { + StorageLive(_6); + _6 = &((_2 as Some).0: std::string::String); + StorageLive(_7); + StorageLive(_8); + StorageLive(_9); + _9 = &(*_6); + _8 = &_9; + StorageLive(_10); + _21 = const chained_conditions::promoted[1]; + _10 = &(*_21); + StorageLive(_27); + StorageLive(_28); + _27 = copy (*_8); + _28 = copy (*_10); + _7 = >::eq(move _27, move _28) -> [return: bb19, unwind unreachable]; + } + + bb5: { + StorageDead(_10); + StorageDead(_9); + StorageDead(_8); + StorageDead(_7); + StorageLive(_5); + _23 = const false; + _5 = move ((_2 as Some).0: std::string::String); + _1 = chained_conditions::BacktraceStyle::Full; + drop(_5) -> [return: bb7, unwind unreachable]; + } + + bb6: { + StorageDead(_10); + StorageDead(_9); + StorageDead(_8); + StorageDead(_7); + StorageDead(_6); + StorageLive(_13); + _13 = &((_2 as Some).0: std::string::String); + StorageLive(_14); + StorageLive(_15); + StorageLive(_16); + _16 = &(*_13); + _15 = &_16; + StorageLive(_17); + _20 = const chained_conditions::promoted[0]; + _17 = &(*_20); + StorageLive(_29); + StorageLive(_30); + _29 = copy (*_15); + _30 = copy (*_17); + _14 = >::eq(move _29, move _30) -> [return: bb20, unwind unreachable]; + } + + bb7: { + StorageDead(_5); + StorageDead(_6); +- goto -> bb18; ++ goto -> bb21; + } + + bb8: { + StorageDead(_17); + StorageDead(_16); + StorageDead(_15); + StorageDead(_14); + StorageLive(_12); + _23 = const false; + _12 = move ((_2 as Some).0: std::string::String); + _1 = chained_conditions::BacktraceStyle::Off; + drop(_12) -> [return: bb10, unwind unreachable]; + } + + bb9: { + StorageDead(_17); + StorageDead(_16); + StorageDead(_15); + StorageDead(_14); + StorageDead(_13); + _1 = chained_conditions::BacktraceStyle::Short; +- goto -> bb18; ++ goto -> bb23; + } + + bb10: { + StorageDead(_12); + StorageDead(_13); +- goto -> bb18; ++ goto -> bb21; + } + + bb11: { + _0 = const 3_u8; + goto -> bb14; + } + + bb12: { + _0 = const 2_u8; + goto -> bb14; + } + + bb13: { + _0 = const 1_u8; + goto -> bb14; + } + + bb14: { + StorageDead(_1); + return; + } + + bb15: { + _22 = const false; + _23 = const false; + StorageDead(_2); + _19 = discriminant(_1); + switchInt(move _19) -> [0: bb13, 1: bb12, 2: bb11, otherwise: bb2]; + } + + bb16: { + switchInt(copy _23) -> [0: bb15, otherwise: bb17]; + } + + bb17: { + drop(((_2 as Some).0: std::string::String)) -> [return: bb15, unwind unreachable]; + } + + bb18: { + _24 = discriminant(_2); + switchInt(move _24) -> [1: bb16, otherwise: bb15]; + } + + bb19: { + StorageDead(_28); + StorageDead(_27); + switchInt(move _7) -> [0: bb6, otherwise: bb5]; + } + + bb20: { + StorageDead(_30); + StorageDead(_29); + switchInt(move _14) -> [0: bb9, otherwise: bb8]; ++ } ++ ++ bb21: { ++ _24 = discriminant(_2); ++ switchInt(move _24) -> [1: bb22, otherwise: bb15]; ++ } ++ ++ bb22: { ++ goto -> bb15; ++ } ++ ++ bb23: { ++ _24 = discriminant(_2); ++ switchInt(move _24) -> [1: bb24, otherwise: bb15]; ++ } ++ ++ bb24: { ++ goto -> bb17; + } + } + diff --git a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff new file mode 100644 index 0000000000000..afd40c1862c3c --- /dev/null +++ b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff @@ -0,0 +1,255 @@ +- // MIR for `chained_conditions` before JumpThreading ++ // MIR for `chained_conditions` after JumpThreading + + fn chained_conditions() -> u8 { + let mut _0: u8; + let _1: chained_conditions::BacktraceStyle; + let mut _2: std::option::Option; + let mut _3: &std::option::Option; + let mut _4: isize; + let _5: std::string::String; + let _6: &std::string::String; + let mut _7: bool; + let mut _8: &&std::string::String; + let _9: &std::string::String; + let mut _10: &&str; + let _11: &str; + let _12: std::string::String; + let _13: &std::string::String; + let mut _14: bool; + let mut _15: &&std::string::String; + let _16: &std::string::String; + let mut _17: &&str; + let _18: &str; + let mut _19: isize; + let mut _20: &&str; + let mut _21: &&str; + let mut _22: bool; + let mut _23: bool; + let mut _24: isize; + let mut _25: isize; + let mut _26: isize; + scope 1 { + debug format => _1; + } + scope 2 { + debug x => _5; + debug x => _6; + } + scope 3 { + debug x => _12; + debug x => _13; + } + scope 4 (inlined std::cmp::impls:: for &String>::eq) { + let mut _27: &std::string::String; + let mut _28: &str; + } + scope 5 (inlined std::cmp::impls:: for &String>::eq) { + let mut _29: &std::string::String; + let mut _30: &str; + } + + bb0: { + _22 = const false; + _23 = const false; + StorageLive(_1); + StorageLive(_2); + _22 = const true; + _23 = const true; + _2 = env_var() -> [return: bb1, unwind continue]; + } + + bb1: { + _4 = discriminant(_2); + switchInt(move _4) -> [0: bb3, 1: bb4, otherwise: bb2]; + } + + bb2: { + unreachable; + } + + bb3: { + _1 = chained_conditions::BacktraceStyle::Off; +- goto -> bb19; ++ goto -> bb27; + } + + bb4: { + StorageLive(_6); + _6 = &((_2 as Some).0: std::string::String); + StorageLive(_7); + StorageLive(_8); + StorageLive(_9); + _9 = &(*_6); + _8 = &_9; + StorageLive(_10); + _21 = const chained_conditions::promoted[1]; + _10 = &(*_21); + StorageLive(_27); + StorageLive(_28); + _27 = copy (*_8); + _28 = copy (*_10); + _7 = >::eq(move _27, move _28) -> [return: bb23, unwind: bb22]; + } + + bb5: { + StorageDead(_10); + StorageDead(_9); + StorageDead(_8); + StorageDead(_7); + StorageLive(_5); + _23 = const false; + _5 = move ((_2 as Some).0: std::string::String); + _1 = chained_conditions::BacktraceStyle::Full; + drop(_5) -> [return: bb7, unwind: bb22]; + } + + bb6: { + StorageDead(_10); + StorageDead(_9); + StorageDead(_8); + StorageDead(_7); + StorageDead(_6); + StorageLive(_13); + _13 = &((_2 as Some).0: std::string::String); + StorageLive(_14); + StorageLive(_15); + StorageLive(_16); + _16 = &(*_13); + _15 = &_16; + StorageLive(_17); + _20 = const chained_conditions::promoted[0]; + _17 = &(*_20); + StorageLive(_29); + StorageLive(_30); + _29 = copy (*_15); + _30 = copy (*_17); + _14 = >::eq(move _29, move _30) -> [return: bb24, unwind: bb22]; + } + + bb7: { + StorageDead(_5); + StorageDead(_6); +- goto -> bb19; ++ goto -> bb25; + } + + bb8: { + StorageDead(_17); + StorageDead(_16); + StorageDead(_15); + StorageDead(_14); + StorageLive(_12); + _23 = const false; + _12 = move ((_2 as Some).0: std::string::String); + _1 = chained_conditions::BacktraceStyle::Off; + drop(_12) -> [return: bb10, unwind: bb22]; + } + + bb9: { + StorageDead(_17); + StorageDead(_16); + StorageDead(_15); + StorageDead(_14); + StorageDead(_13); + _1 = chained_conditions::BacktraceStyle::Short; +- goto -> bb19; ++ goto -> bb27; + } + + bb10: { + StorageDead(_12); + StorageDead(_13); +- goto -> bb19; ++ goto -> bb25; + } + + bb11: { + _0 = const 3_u8; + goto -> bb14; + } + + bb12: { + _0 = const 2_u8; + goto -> bb14; + } + + bb13: { + _0 = const 1_u8; + goto -> bb14; + } + + bb14: { + StorageDead(_1); + return; + } + + bb15 (cleanup): { + resume; + } + + bb16: { + _22 = const false; + _23 = const false; + StorageDead(_2); + _19 = discriminant(_1); + switchInt(move _19) -> [0: bb13, 1: bb12, 2: bb11, otherwise: bb2]; + } + + bb17: { + switchInt(copy _23) -> [0: bb16, otherwise: bb18]; + } + + bb18: { + drop(((_2 as Some).0: std::string::String)) -> [return: bb16, unwind: bb15]; + } + + bb19: { + _24 = discriminant(_2); + switchInt(move _24) -> [1: bb17, otherwise: bb16]; + } + + bb20 (cleanup): { + switchInt(copy _23) -> [0: bb15, otherwise: bb21]; + } + + bb21 (cleanup): { + drop(((_2 as Some).0: std::string::String)) -> [return: bb15, unwind terminate(cleanup)]; + } + + bb22 (cleanup): { + _26 = discriminant(_2); + switchInt(move _26) -> [1: bb20, otherwise: bb15]; + } + + bb23: { + StorageDead(_28); + StorageDead(_27); + switchInt(move _7) -> [0: bb6, otherwise: bb5]; + } + + bb24: { + StorageDead(_30); + StorageDead(_29); + switchInt(move _14) -> [0: bb9, otherwise: bb8]; ++ } ++ ++ bb25: { ++ _24 = discriminant(_2); ++ switchInt(move _24) -> [1: bb26, otherwise: bb16]; ++ } ++ ++ bb26: { ++ goto -> bb16; ++ } ++ ++ bb27: { ++ _24 = discriminant(_2); ++ switchInt(move _24) -> [1: bb28, otherwise: bb16]; ++ } ++ ++ bb28: { ++ goto -> bb18; + } + } + diff --git a/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-abort.diff index a86371794ebe9..0b11a0d188fb8 100644 --- a/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-abort.diff @@ -23,14 +23,14 @@ bb2: { _2 = CustomDiscr::B; - goto -> bb3; +- goto -> bb3; ++ goto -> bb8; } bb3: { StorageDead(_3); _4 = discriminant(_2); -- switchInt(move _4) -> [35: bb5, otherwise: bb4]; -+ goto -> bb4; + switchInt(move _4) -> [35: bb5, otherwise: bb4]; } bb4: { @@ -52,6 +52,12 @@ + StorageDead(_3); + _4 = discriminant(_2); + goto -> bb5; ++ } ++ ++ bb8: { ++ StorageDead(_3); ++ _4 = discriminant(_2); ++ goto -> bb4; } } diff --git a/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-unwind.diff index a86371794ebe9..0b11a0d188fb8 100644 --- a/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-unwind.diff @@ -23,14 +23,14 @@ bb2: { _2 = CustomDiscr::B; - goto -> bb3; +- goto -> bb3; ++ goto -> bb8; } bb3: { StorageDead(_3); _4 = discriminant(_2); -- switchInt(move _4) -> [35: bb5, otherwise: bb4]; -+ goto -> bb4; + switchInt(move _4) -> [35: bb5, otherwise: bb4]; } bb4: { @@ -52,6 +52,12 @@ + StorageDead(_3); + _4 = discriminant(_2); + goto -> bb5; ++ } ++ ++ bb8: { ++ StorageDead(_3); ++ _4 = discriminant(_2); ++ goto -> bb4; } } diff --git a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff index d17f2752f58ec..4955d05214ff4 100644 --- a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff @@ -15,7 +15,7 @@ bb1: { _3 = const false; - goto -> bb4; -+ goto -> bb9; ++ goto -> bb10; } bb2: { @@ -24,7 +24,8 @@ bb3: { _2 = const false; - goto -> bb4; +- goto -> bb4; ++ goto -> bb13; } bb4: { @@ -40,8 +41,7 @@ } bb7: { -- goto -> bb5; -+ goto -> bb10; + goto -> bb5; } bb8: { @@ -49,10 +49,30 @@ + } + + bb9: { -+ goto -> bb5; ++ switchInt(copy _3) -> [0: bb5, otherwise: bb7]; + } + + bb10: { ++ goto -> bb11; ++ } ++ ++ bb11: { ++ goto -> bb8; ++ } ++ ++ bb12: { ++ switchInt(copy _3) -> [0: bb5, otherwise: bb7]; ++ } ++ ++ bb13: { ++ goto -> bb14; ++ } ++ ++ bb14: { ++ goto -> bb15; ++ } ++ ++ bb15: { goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff index d17f2752f58ec..4955d05214ff4 100644 --- a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff @@ -15,7 +15,7 @@ bb1: { _3 = const false; - goto -> bb4; -+ goto -> bb9; ++ goto -> bb10; } bb2: { @@ -24,7 +24,8 @@ bb3: { _2 = const false; - goto -> bb4; +- goto -> bb4; ++ goto -> bb13; } bb4: { @@ -40,8 +41,7 @@ } bb7: { -- goto -> bb5; -+ goto -> bb10; + goto -> bb5; } bb8: { @@ -49,10 +49,30 @@ + } + + bb9: { -+ goto -> bb5; ++ switchInt(copy _3) -> [0: bb5, otherwise: bb7]; + } + + bb10: { ++ goto -> bb11; ++ } ++ ++ bb11: { ++ goto -> bb8; ++ } ++ ++ bb12: { ++ switchInt(copy _3) -> [0: bb5, otherwise: bb7]; ++ } ++ ++ bb13: { ++ goto -> bb14; ++ } ++ ++ bb14: { ++ goto -> bb15; ++ } ++ ++ bb15: { goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff index 79599f856115d..97b8d484194f5 100644 --- a/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff @@ -97,8 +97,7 @@ StorageDead(_10); StorageDead(_4); _5 = discriminant(_3); -- switchInt(move _5) -> [0: bb2, 1: bb3, otherwise: bb1]; -+ goto -> bb2; + switchInt(move _5) -> [0: bb2, 1: bb3, otherwise: bb1]; } bb6: { @@ -114,7 +113,8 @@ bb7: { _11 = move ((_4 as Ok).0: i32); _3 = ControlFlow::, i32>::Continue(copy _11); - goto -> bb5; +- goto -> bb5; ++ goto -> bb9; + } + + bb8: { @@ -124,6 +124,15 @@ + StorageDead(_4); + _5 = discriminant(_3); + goto -> bb3; ++ } ++ ++ bb9: { ++ StorageDead(_12); ++ StorageDead(_11); ++ StorageDead(_10); ++ StorageDead(_4); ++ _5 = discriminant(_3); ++ goto -> bb2; } } diff --git a/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff index 79599f856115d..97b8d484194f5 100644 --- a/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff @@ -97,8 +97,7 @@ StorageDead(_10); StorageDead(_4); _5 = discriminant(_3); -- switchInt(move _5) -> [0: bb2, 1: bb3, otherwise: bb1]; -+ goto -> bb2; + switchInt(move _5) -> [0: bb2, 1: bb3, otherwise: bb1]; } bb6: { @@ -114,7 +113,8 @@ bb7: { _11 = move ((_4 as Ok).0: i32); _3 = ControlFlow::, i32>::Continue(copy _11); - goto -> bb5; +- goto -> bb5; ++ goto -> bb9; + } + + bb8: { @@ -124,6 +124,15 @@ + StorageDead(_4); + _5 = discriminant(_3); + goto -> bb3; ++ } ++ ++ bb9: { ++ StorageDead(_12); ++ StorageDead(_11); ++ StorageDead(_10); ++ StorageDead(_4); ++ _5 = discriminant(_3); ++ goto -> bb2; } } diff --git a/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff index 9a8bdc8f4d952..c8edd704ff9bd 100644 --- a/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff @@ -14,7 +14,7 @@ bb1: { _2 = const false; - goto -> bb3; -+ goto -> bb8; ++ goto -> bb9; } bb2: { @@ -47,10 +47,14 @@ + } + + bb8: { -+ goto -> bb9; ++ switchInt(copy _2) -> [0: bb4, otherwise: bb5]; + } + + bb9: { ++ goto -> bb10; ++ } ++ ++ bb10: { + goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff index 9a8bdc8f4d952..c8edd704ff9bd 100644 --- a/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff @@ -14,7 +14,7 @@ bb1: { _2 = const false; - goto -> bb3; -+ goto -> bb8; ++ goto -> bb9; } bb2: { @@ -47,10 +47,14 @@ + } + + bb8: { -+ goto -> bb9; ++ switchInt(copy _2) -> [0: bb4, otherwise: bb5]; + } + + bb9: { ++ goto -> bb10; ++ } ++ ++ bb10: { + goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.rs b/tests/mir-opt/jump_threading.rs index 009e1060700c1..39a2f16c5ad6f 100644 --- a/tests/mir-opt/jump_threading.rs +++ b/tests/mir-opt/jump_threading.rs @@ -19,9 +19,9 @@ fn too_complex(x: Result) -> Option { // CHECK: goto -> bb8; // CHECK: bb3: { // CHECK: [[controlflow]] = ControlFlow::::Continue( - // CHECK: goto -> bb4; + // CHECK: goto -> bb9; // CHECK: bb4: { - // CHECK: goto -> bb6; + // CHECK: switchInt(move _8) -> [0: bb6, 1: bb5, otherwise: bb1]; // CHECK: bb5: { // CHECK: {{_.*}} = copy (([[controlflow]] as Break).0: usize); // CHECK: _0 = Option::::None; @@ -34,6 +34,8 @@ fn too_complex(x: Result) -> Option { // CHECK: return; // CHECK: bb8: { // CHECK: goto -> bb5; + // CHECK: bb9: { + // CHECK: goto -> bb6; match { match x { Ok(v) => ControlFlow::Continue(v), @@ -63,7 +65,7 @@ fn identity(x: Result) -> Result { // CHECK: bb4: { // CHECK: return; // CHECK: bb5: { - // CHECK: goto -> bb2; + // CHECK: switchInt(move _5) -> [0: bb2, 1: bb3, otherwise: bb1]; // CHECK: bb6: { // CHECK: {{_.*}} = move (([[x]] as Err).0: i32); // CHECK: [[controlflow]] = ControlFlow::, i32>::Break( @@ -71,12 +73,40 @@ fn identity(x: Result) -> Result { // CHECK: bb7: { // CHECK: {{_.*}} = move (([[x]] as Ok).0: i32); // CHECK: [[controlflow]] = ControlFlow::, i32>::Continue( - // CHECK: goto -> bb5; + // CHECK: goto -> bb9; // CHECK: bb8: { // CHECK: goto -> bb3; + // CHECK: bb9: { + // CHECK: goto -> bb2; Ok(x?) } +fn two_reads() -> i32 { + // CHECK-LABEL: fn two_reads( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: debug c => [[c:_.*]]; + // CHECK: bb0: { + // CHECK: [[a]] = const 2_i32; + // CHECK: [[b]] = copy [[a]]; + // CHECK: [[c]] = copy [[a]]; + // CHECK: [[tmp:_.*]] = copy [[c]]; + // CHECK: [[eq:_.*]] = Eq(move [[tmp]], const 2_i32); + // CHECK: goto -> bb1; + // CHECK: bb1: { + // CHECK: _0 = const 0_i32; + // CHECK: goto -> bb3; + // CHECK: bb2: { + // CHECK: _0 = const 1_i32; + // CHECK: goto -> bb3; + // CHECK: bb3: { + // CHECK: return; + let a = 2; + let b = a; + let c = a; + if c == 2 { 0 } else { 1 } +} + enum DFA { A, B, @@ -134,9 +164,9 @@ fn custom_discr(x: bool) -> u8 { // CHECK: goto -> bb7; // CHECK: bb2: { // CHECK: {{_.*}} = CustomDiscr::B; - // CHECK: goto -> bb3; + // CHECK: goto -> bb8; // CHECK: bb3: { - // CHECK: goto -> bb4; + // CHECK: switchInt(move _4) -> [35: bb5, otherwise: bb4]; // CHECK: bb4: { // CHECK: _0 = const 13_u8; // CHECK: goto -> bb6; @@ -147,6 +177,8 @@ fn custom_discr(x: bool) -> u8 { // CHECK: return; // CHECK: bb7: { // CHECK: goto -> bb5; + // CHECK: bb8: { + // CHECK: goto -> bb4; match if x { CustomDiscr::A } else { CustomDiscr::B } { CustomDiscr::A => 5, _ => 13, @@ -258,7 +290,6 @@ fn duplicate_chain(x: bool) -> u8 { bb4 = { // CHECK: bb4: { // CHECK: {{_.*}} = const 15_i32; - // CHECK-NOT: switchInt( // CHECK: goto -> bb5; let c = 15; match a { 5 => bb5, _ => bb6 } @@ -348,7 +379,7 @@ fn renumbered_bb(x: bool) -> u8 { } bb1 = { // CHECK: bb1: { - // CHECK: goto -> bb8; + // CHECK: goto -> bb9; a = false; Goto(bb3) } @@ -389,10 +420,13 @@ fn renumbered_bb(x: bool) -> u8 { } // Duplicate of bb3. // CHECK: bb8: { - // CHECK-NEXT: goto -> bb9; - // Duplicate of bb4. + // CHECK: switchInt(copy _2) -> [0: bb4, otherwise: bb5]; + // Duplicate of bb8. // CHECK: bb9: { - // CHECK-NEXT: goto -> bb6; + // CHECK: goto -> bb10; + // Duplicate of bb4. + // CHECK: bb10: { + // CHECK: goto -> bb6; } } @@ -407,22 +441,26 @@ fn disappearing_bb(x: u8) -> u8 { let a: bool; let b: bool; { + // CHECK: bb0: { a = true; b = true; + // CHECK: switchInt({{.*}}) -> [0: bb3, 1: bb3, 2: bb1, otherwise: bb2]; match x { 0 => bb3, 1 => bb3, 2 => bb1, _ => bb2 } } bb1 = { // CHECK: bb1: { - // CHECK: goto -> bb9; + // CHECK: goto -> bb10; b = false; Goto(bb4) } bb2 = { + // CHECK: bb2: { + // CHECK: unreachable; Unreachable() } bb3 = { // CHECK: bb3: { - // CHECK: goto -> bb10; + // CHECK: goto -> bb13; a = false; Goto(bb4) } @@ -442,16 +480,34 @@ fn disappearing_bb(x: u8) -> u8 { Goto(bb6) } // CHECK: bb9: { - // CHECK: goto -> bb5; + // CHECK: switchInt(copy _3) -> [0: bb5, otherwise: bb7]; // CHECK: bb10: { - // CHECK: goto -> bb6; + // CHECK: goto -> bb11; + // CHECK: bb11: { + // CHECK: goto -> bb8; + // CHECK: bb12: { + // CHECK: switchInt(copy _3) -> [0: bb5, otherwise: bb7]; + // CHECK: bb13: { + // CHECK: goto -> bb14; + // CHECK: bb14: { + // CHECK: goto -> bb15; + // CHECK: bb15: { + // CHECK: goto -> bb6; } } /// Verify that we can thread jumps when we assign from an aggregate constant. -fn aggregate(x: u8) -> u8 { +fn aggregate() -> u8 { // CHECK-LABEL: fn aggregate( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; // CHECK-NOT: switchInt( + // CHECK: [[a2:_.*]] = copy [[a]]; + // CHECK: {{_.*}} = Eq(move [[a2]], const 7_u8); + // CHECK-NEXT: goto -> [[bb:bb.*]]; + // CHECK: [[bb]]: { + // CHECK-NOT: } + // CHECK: _0 = copy [[a]]; const FOO: (u8, u8) = (5, 13); @@ -508,7 +564,16 @@ fn assume(a: u8, b: bool) -> u8 { /// Verify that jump threading succeeds seeing through copies of aggregates. fn aggregate_copy() -> u32 { // CHECK-LABEL: fn aggregate_copy( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: debug c => [[c:_.*]]; // CHECK-NOT: switchInt( + // CHECK: [[c2:_.*]] = copy [[c]]; + // CHECK: {{_.*}} = Eq(move [[c2]], const 2_u32); + // CHECK-NEXT: goto -> [[bb:bb.*]]; + // CHECK: [[bb]]: { + // CHECK-NOT: } + // CHECK: _0 = const 13_u32; const Foo: (u32, u32) = (5, 3); @@ -532,6 +597,14 @@ fn floats() -> u32 { pub fn bitwise_not() -> i32 { // CHECK-LABEL: fn bitwise_not( + // CHECK: debug a => [[a:_.*]]; + // CHECK: [[a2:_.*]] = copy [[a]]; + // CHECK: [[not:_.*]] = Not(move [[a2]]); + // CHECK: {{_.*}} = Eq(move [[not]], const 0_i32); + // CHECK-NEXT: goto -> [[bb:bb.*]]; + // CHECK: [[bb]]: { + // CHECK-NOT: } + // CHECK: _0 = const 0_i32; // Test for #131195, which was optimizing `!a == b` into `a != b`. let a = 1; @@ -540,11 +613,49 @@ pub fn bitwise_not() -> i32 { pub fn logical_not() -> i32 { // CHECK-LABEL: fn logical_not( + // CHECK: debug a => [[a:_.*]]; + // CHECK: [[a2:_.*]] = copy [[a]]; + // CHECK: [[not:_.*]] = Not(move [[a2]]); + // CHECK: {{_.*}} = Eq(move [[not]], const true); + // CHECK-NEXT: goto -> [[bb:bb.*]]; + // CHECK: [[bb]]: { + // CHECK-NOT: } + // CHECK: _0 = const 1_i32; let a = false; if !a == true { 1 } else { 0 } } +/// Verify that we correctly handle threading multiple conditions on the same bb. +/// One version of the implementation was buggy and mutated a bb that would be duplicated later. +fn chained_conditions() -> u8 { + // CHECK-LABEL: fn chained_conditions( + + #[inline(never)] + fn env_var() -> Option { + None + } + + enum BacktraceStyle { + Off, + Short, + Full, + }; + + let format = match env_var() { + Some(x) if &x == "full" => BacktraceStyle::Full, + Some(x) if &x == "0" => BacktraceStyle::Off, + Some(_) => BacktraceStyle::Short, + None => BacktraceStyle::Off, + }; + + match format { + BacktraceStyle::Off => 1, + BacktraceStyle::Short => 2, + BacktraceStyle::Full => 3, + } +} + fn main() { // CHECK-LABEL: fn main( too_complex(Ok(0)); @@ -557,7 +668,7 @@ fn main() { mutable_ref(); renumbered_bb(true); disappearing_bb(7); - aggregate(7); + aggregate(); assume(7, false); floats(); bitwise_not(); @@ -566,6 +677,7 @@ fn main() { // EMIT_MIR jump_threading.too_complex.JumpThreading.diff // EMIT_MIR jump_threading.identity.JumpThreading.diff +// EMIT_MIR jump_threading.two_reads.JumpThreading.diff // EMIT_MIR jump_threading.custom_discr.JumpThreading.diff // EMIT_MIR jump_threading.dfa.JumpThreading.diff // EMIT_MIR jump_threading.multiple_match.JumpThreading.diff @@ -580,3 +692,4 @@ fn main() { // EMIT_MIR jump_threading.floats.JumpThreading.diff // EMIT_MIR jump_threading.bitwise_not.JumpThreading.diff // EMIT_MIR jump_threading.logical_not.JumpThreading.diff +// EMIT_MIR jump_threading.chained_conditions.JumpThreading.diff diff --git a/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-abort.diff index 7de359298922b..6e0cf5ec41abd 100644 --- a/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-abort.diff @@ -57,13 +57,13 @@ _2 = ControlFlow::::Continue(move _5); StorageDead(_5); StorageDead(_4); - goto -> bb4; +- goto -> bb4; ++ goto -> bb9; } bb4: { _8 = discriminant(_2); -- switchInt(move _8) -> [0: bb6, 1: bb5, otherwise: bb1]; -+ goto -> bb6; + switchInt(move _8) -> [0: bb6, 1: bb5, otherwise: bb1]; } bb5: { @@ -93,6 +93,11 @@ + bb8: { + _8 = discriminant(_2); + goto -> bb5; ++ } ++ ++ bb9: { ++ _8 = discriminant(_2); ++ goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-unwind.diff index 7de359298922b..6e0cf5ec41abd 100644 --- a/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-unwind.diff @@ -57,13 +57,13 @@ _2 = ControlFlow::::Continue(move _5); StorageDead(_5); StorageDead(_4); - goto -> bb4; +- goto -> bb4; ++ goto -> bb9; } bb4: { _8 = discriminant(_2); -- switchInt(move _8) -> [0: bb6, 1: bb5, otherwise: bb1]; -+ goto -> bb6; + switchInt(move _8) -> [0: bb6, 1: bb5, otherwise: bb1]; } bb5: { @@ -93,6 +93,11 @@ + bb8: { + _8 = discriminant(_2); + goto -> bb5; ++ } ++ ++ bb9: { ++ _8 = discriminant(_2); ++ goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-abort.diff new file mode 100644 index 0000000000000..090e9d37b4f93 --- /dev/null +++ b/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-abort.diff @@ -0,0 +1,56 @@ +- // MIR for `two_reads` before JumpThreading ++ // MIR for `two_reads` after JumpThreading + + fn two_reads() -> i32 { + let mut _0: i32; + let _1: i32; + let mut _4: bool; + let mut _5: i32; + scope 1 { + debug a => _1; + let _2: i32; + scope 2 { + debug b => _2; + let _3: i32; + scope 3 { + debug c => _3; + } + } + } + + bb0: { + StorageLive(_1); + _1 = const 2_i32; + StorageLive(_2); + _2 = copy _1; + StorageLive(_3); + _3 = copy _1; + StorageLive(_4); + StorageLive(_5); + _5 = copy _3; + _4 = Eq(move _5, const 2_i32); +- switchInt(move _4) -> [0: bb2, otherwise: bb1]; ++ goto -> bb1; + } + + bb1: { + StorageDead(_5); + _0 = const 0_i32; + goto -> bb3; + } + + bb2: { + StorageDead(_5); + _0 = const 1_i32; + goto -> bb3; + } + + bb3: { + StorageDead(_4); + StorageDead(_3); + StorageDead(_2); + StorageDead(_1); + return; + } + } + diff --git a/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-unwind.diff new file mode 100644 index 0000000000000..090e9d37b4f93 --- /dev/null +++ b/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-unwind.diff @@ -0,0 +1,56 @@ +- // MIR for `two_reads` before JumpThreading ++ // MIR for `two_reads` after JumpThreading + + fn two_reads() -> i32 { + let mut _0: i32; + let _1: i32; + let mut _4: bool; + let mut _5: i32; + scope 1 { + debug a => _1; + let _2: i32; + scope 2 { + debug b => _2; + let _3: i32; + scope 3 { + debug c => _3; + } + } + } + + bb0: { + StorageLive(_1); + _1 = const 2_i32; + StorageLive(_2); + _2 = copy _1; + StorageLive(_3); + _3 = copy _1; + StorageLive(_4); + StorageLive(_5); + _5 = copy _3; + _4 = Eq(move _5, const 2_i32); +- switchInt(move _4) -> [0: bb2, otherwise: bb1]; ++ goto -> bb1; + } + + bb1: { + StorageDead(_5); + _0 = const 0_i32; + goto -> bb3; + } + + bb2: { + StorageDead(_5); + _0 = const 1_i32; + goto -> bb3; + } + + bb3: { + StorageDead(_4); + StorageDead(_3); + StorageDead(_2); + StorageDead(_1); + return; + } + } + diff --git a/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir index e235fa35c0238..578aff4f7129d 100644 --- a/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir +++ b/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir @@ -7,12 +7,11 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool { scope 1 (inlined ::le) { let mut _6: std::option::Option; scope 2 (inlined Option::::is_some_and:: bool {std::cmp::Ordering::is_le}>) { - let mut _11: isize; - let _12: std::cmp::Ordering; + let _11: std::cmp::Ordering; scope 3 { scope 4 (inlined bool {std::cmp::Ordering::is_le} as FnOnce<(std::cmp::Ordering,)>>::call_once - shim(fn(std::cmp::Ordering) -> bool {std::cmp::Ordering::is_le})) { scope 5 (inlined std::cmp::Ordering::is_le) { - let mut _13: i8; + let mut _12: i8; scope 6 (inlined std::cmp::Ordering::as_raw) { } } @@ -37,7 +36,7 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool { } bb0: { - StorageLive(_12); + StorageLive(_11); StorageLive(_6); StorageLive(_5); StorageLive(_7); @@ -64,42 +63,19 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool { StorageDead(_8); _6 = Option::::Some(move _10); StorageDead(_10); - StorageDead(_7); - StorageDead(_5); - StorageLive(_11); - goto -> bb4; + goto -> bb2; } bb2: { StorageDead(_7); StorageDead(_5); - StorageLive(_11); - _11 = discriminant(_6); - switchInt(move _11) -> [0: bb3, 1: bb4, otherwise: bb6]; - } - - bb3: { - _0 = const false; - goto -> bb5; - } - - bb4: { - _12 = move ((_6 as Some).0: std::cmp::Ordering); - StorageLive(_13); - _13 = discriminant(_12); - _0 = Le(move _13, const 0_i8); - StorageDead(_13); - goto -> bb5; - } - - bb5: { - StorageDead(_11); - StorageDead(_6); + _11 = move ((_6 as Some).0: std::cmp::Ordering); + StorageLive(_12); + _12 = discriminant(_11); + _0 = Le(move _12, const 0_i8); StorageDead(_12); + StorageDead(_6); + StorageDead(_11); return; } - - bb6: { - unreachable; - } } diff --git a/tests/mir-opt/pre-codegen/derived_ord.rs b/tests/mir-opt/pre-codegen/derived_ord.rs index 823e0f6d09c2b..a67756c22d466 100644 --- a/tests/mir-opt/pre-codegen/derived_ord.rs +++ b/tests/mir-opt/pre-codegen/derived_ord.rs @@ -13,22 +13,23 @@ pub fn demo_le(a: &MultiField, b: &MultiField) -> bool { // CHECK: inlined{{.+}}is_some_and // CHECK: inlined ::partial_cmp - // CHECK: [[A0:_[0-9]+]] = copy ((*_1).0: char); - // CHECK: [[B0:_[0-9]+]] = copy ((*_2).0: char); - // CHECK: Cmp(move [[A0]], move [[B0]]); - - // CHECK: [[D0:_[0-9]+]] = discriminant({{.+}}); - // CHECK: switchInt(move [[D0]]) -> [0: bb{{[0-9]+}}, otherwise: bb{{[0-9]+}}]; - - // CHECK: [[A1:_[0-9]+]] = copy ((*_1).1: i16); - // CHECK: [[B1:_[0-9]+]] = copy ((*_2).1: i16); - // CHECK: Cmp(move [[A1]], move [[B1]]); - - // CHECK: [[D1:_[0-9]+]] = discriminant({{.+}}); - // CHECK: switchInt(move [[D1]]) -> [0: bb{{[0-9]+}}, 1: bb{{[0-9]+}}, otherwise: bb{{[0-9]+}}]; - - // CHECK: [[D2:_[0-9]+]] = discriminant({{.+}}); - // CHECK: _0 = Le(move [[D2]], const 0_i8); + // CHECK: bb0: { + // CHECK: [[A0:_[0-9]+]] = copy ((*_1).0: char); + // CHECK: [[B0:_[0-9]+]] = copy ((*_2).0: char); + // CHECK: Cmp(move [[A0]], move [[B0]]); + // CHECK: [[D0:_[0-9]+]] = discriminant({{.+}}); + // CHECK: switchInt(move [[D0]]) -> [0: bb1, otherwise: bb2]; + + // CHECK: bb1: { + // CHECK: [[A1:_[0-9]+]] = copy ((*_1).1: i16); + // CHECK: [[B1:_[0-9]+]] = copy ((*_2).1: i16); + // CHECK: Cmp(move [[A1]], move [[B1]]); + // CHECK: goto -> bb2; + + // CHECK: bb2: { + // CHECK: [[D2:_[0-9]+]] = discriminant({{.+}}); + // CHECK: _0 = Le(move [[D2]], const 0_i8); + // CHECK: return; *a <= *b } diff --git a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir index 104987b0fdda9..f72611b7cb8e3 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir @@ -211,12 +211,6 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { bb7: { StorageDead(_16); - StorageDead(_22); - StorageDead(_13); - StorageDead(_20); - StorageDead(_19); - StorageDead(_12); - StorageDead(_11); goto -> bb10; } @@ -226,16 +220,16 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } bb9: { + goto -> bb10; + } + + bb10: { StorageDead(_22); StorageDead(_13); StorageDead(_20); StorageDead(_19); StorageDead(_12); StorageDead(_11); - goto -> bb10; - } - - bb10: { StorageDead(_23); StorageDead(_26); StorageDead(_25); diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir index 4d0e3548e7d6b..b210efb1f46c2 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir @@ -173,12 +173,6 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { bb7: { StorageDead(_15); - StorageDead(_21); - StorageDead(_12); - StorageDead(_19); - StorageDead(_18); - StorageDead(_11); - StorageDead(_10); goto -> bb10; } @@ -188,16 +182,16 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb9: { + goto -> bb10; + } + + bb10: { StorageDead(_21); StorageDead(_12); StorageDead(_19); StorageDead(_18); StorageDead(_11); StorageDead(_10); - goto -> bb10; - } - - bb10: { StorageDead(_22); drop(_2) -> [return: bb11, unwind unreachable]; } diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir index 2b5d8c27d7109..ab6e2bf0b36b3 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir @@ -173,12 +173,6 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { bb7: { StorageDead(_15); - StorageDead(_21); - StorageDead(_12); - StorageDead(_19); - StorageDead(_18); - StorageDead(_11); - StorageDead(_10); goto -> bb10; } @@ -188,16 +182,16 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb9: { + goto -> bb10; + } + + bb10: { StorageDead(_21); StorageDead(_12); StorageDead(_19); StorageDead(_18); StorageDead(_11); StorageDead(_10); - goto -> bb10; - } - - bb10: { StorageDead(_22); drop(_2) -> [return: bb11, unwind continue]; } diff --git a/tests/mir-opt/separate_const_switch.identity.JumpThreading.diff b/tests/mir-opt/separate_const_switch.identity.JumpThreading.diff index ce9d812701a8f..34f451fc698c7 100644 --- a/tests/mir-opt/separate_const_switch.identity.JumpThreading.diff +++ b/tests/mir-opt/separate_const_switch.identity.JumpThreading.diff @@ -69,8 +69,7 @@ StorageDead(_7); StorageDead(_6); _3 = discriminant(_2); -- switchInt(move _3) -> [0: bb2, 1: bb3, otherwise: bb1]; -+ goto -> bb2; + switchInt(move _3) -> [0: bb2, 1: bb3, otherwise: bb1]; } bb5: { @@ -86,7 +85,8 @@ bb6: { _7 = copy ((_1 as Ok).0: i32); _2 = ControlFlow::, i32>::Continue(copy _7); - goto -> bb4; +- goto -> bb4; ++ goto -> bb8; + } + + bb7: { @@ -95,6 +95,14 @@ + StorageDead(_6); + _3 = discriminant(_2); + goto -> bb3; ++ } ++ ++ bb8: { ++ StorageDead(_8); ++ StorageDead(_7); ++ StorageDead(_6); ++ _3 = discriminant(_2); ++ goto -> bb2; } } diff --git a/tests/mir-opt/separate_const_switch.too_complex.JumpThreading.diff b/tests/mir-opt/separate_const_switch.too_complex.JumpThreading.diff index c88c63e0c1334..794c28ab46da0 100644 --- a/tests/mir-opt/separate_const_switch.too_complex.JumpThreading.diff +++ b/tests/mir-opt/separate_const_switch.too_complex.JumpThreading.diff @@ -44,13 +44,13 @@ bb3: { _4 = copy ((_1 as Ok).0: i32); _2 = ControlFlow::::Continue(copy _4); - goto -> bb4; +- goto -> bb4; ++ goto -> bb9; } bb4: { _6 = discriminant(_2); -- switchInt(move _6) -> [0: bb6, 1: bb5, otherwise: bb1]; -+ goto -> bb6; + switchInt(move _6) -> [0: bb6, 1: bb5, otherwise: bb1]; } bb5: { @@ -75,6 +75,11 @@ + bb8: { + _6 = discriminant(_2); + goto -> bb5; ++ } ++ ++ bb9: { ++ _6 = discriminant(_2); ++ goto -> bb6; } }