MIR: Stop needing an explicit BB for otherwise:unreachable

So many places to update :D

For this PR I tried to keep things doing essentially the same thing as before.  No new passes to try to use it more, no change to MIR building for exhaustive matches, etc.

That said, `UnreachableProp` still picks it up, and thus there's still a bunch of `otherwise` arms removed and `unreachable` blocks that no longer show.

Author: scottmcm

compiler/rustc_codegen_cranelift/src/base.rs

@@ -95,6 +95,7 @@ pub(crate) fn codegen_fn<'tcx>(
         bcx,
         block_map,
         local_map: IndexVec::with_capacity(mir.local_decls.len()),
+        shared_unreachable: None,
         caller_location: None, // set by `codegen_fn_prelude`
 
         clif_comments,
@@ -404,7 +405,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                     assert_eq!(targets.iter().count(), 1);
                     let (then_value, then_block) = targets.iter().next().unwrap();
                     let then_block = fx.get_block(then_block);
-                    let else_block = fx.get_block(targets.otherwise());
+                    let else_block = fx.get_switch_block(targets.otherwise());
                     let test_zero = match then_value {
                         0 => true,
                         1 => false,
@@ -435,7 +436,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                         let block = fx.get_block(block);
                         switch.set_entry(value, block);
                     }
-                    let otherwise_block = fx.get_block(targets.otherwise());
+                    let otherwise_block = fx.get_switch_block(targets.otherwise());
                     switch.emit(&mut fx.bcx, discr, otherwise_block);
                 }
             }
@@ -520,6 +521,11 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
             }
         };
     }
+
+    if let Some(unreachable) = fx.shared_unreachable {
+        fx.bcx.switch_to_block(unreachable);
+        fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+    }
 }
 
 fn codegen_stmt<'tcx>(

compiler/rustc_codegen_cranelift/src/common.rs

@@ -296,6 +296,7 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
     pub(crate) bcx: FunctionBuilder<'clif>,
     pub(crate) block_map: IndexVec<BasicBlock, Block>,
     pub(crate) local_map: IndexVec<Local, CPlace<'tcx>>,
+    pub(crate) shared_unreachable: Option<Block>,
 
     /// When `#[track_caller]` is used, the implicit caller location is stored in this variable.
     pub(crate) caller_location: Option<CValue<'tcx>>,
@@ -377,6 +378,17 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         *self.block_map.get(bb).unwrap()
     }
 
+    pub(crate) fn get_switch_block(&mut self, sa: SwitchAction) -> Block {
+        match sa {
+            SwitchAction::Goto(bb) => self.get_block(bb),
+            SwitchAction::Unreachable => self.unreachable_block(),
+        }
+    }
+
+    pub(crate) fn unreachable_block(&mut self) -> Block {
+        *self.shared_unreachable.get_or_insert_with(|| self.bcx.create_block())
+    }
+
     pub(crate) fn get_local_place(&mut self, local: Local) -> CPlace<'tcx> {
         *self.local_map.get(local).unwrap_or_else(|| {
             panic!("Local {:?} doesn't exist", local);

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -95,6 +95,17 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
         }
     }
 
+    fn llbb_with_cleanup_from_switch_action<Bx: BuilderMethods<'a, 'tcx>>(
+        &self,
+        fx: &mut FunctionCx<'a, 'tcx, Bx>,
+        target: mir::SwitchAction,
+    ) -> Bx::BasicBlock {
+        match target {
+            mir::SwitchAction::Unreachable => fx.unreachable_block(),
+            mir::SwitchAction::Goto(bb) => self.llbb_with_cleanup(fx, bb),
+        }
+    }
+
     fn llbb_characteristics<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
         fx: &mut FunctionCx<'a, 'tcx, Bx>,
@@ -367,7 +378,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         // If our discriminant is a constant we can branch directly
         if let Some(const_discr) = bx.const_to_opt_u128(discr_value, false) {
             let target = targets.target_for_value(const_discr);
-            bx.br(helper.llbb_with_cleanup(self, target));
+            bx.br(helper.llbb_with_cleanup_from_switch_action(self, target));
             return;
         };
 
@@ -377,7 +388,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             // `switch`.
             let (test_value, target) = target_iter.next().unwrap();
             let lltrue = helper.llbb_with_cleanup(self, target);
-            let llfalse = helper.llbb_with_cleanup(self, targets.otherwise());
+            let llfalse = helper.llbb_with_cleanup_from_switch_action(self, targets.otherwise());
             if switch_ty == bx.tcx().types.bool {
                 // Don't generate trivial icmps when switching on bool.
                 match test_value {
@@ -393,7 +404,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             }
         } else if self.cx.sess().opts.optimize == OptLevel::No
             && target_iter.len() == 2
-            && self.mir[targets.otherwise()].is_empty_unreachable()
+            && self.mir.basic_blocks.is_empty_unreachable(targets.otherwise())
         {
             // In unoptimized builds, if there are two normal targets and the `otherwise` target is
             // an unreachable BB, emit `br` instead of `switch`. This leaves behind the unreachable
@@ -418,7 +429,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         } else {
             bx.switch(
                 discr_value,
-                helper.llbb_with_cleanup(self, targets.otherwise()),
+                helper.llbb_with_cleanup_from_switch_action(self, targets.otherwise()),
                 target_iter.map(|(value, target)| (value, helper.llbb_with_cleanup(self, target))),
             );
         }
@@ -1644,11 +1655,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     }
 
     fn unreachable_block(&mut self) -> Bx::BasicBlock {
-        self.unreachable_block.unwrap_or_else(|| {
+        *self.unreachable_block.get_or_insert_with(|| {
             let llbb = Bx::append_block(self.cx, self.llfn, "unreachable");
             let mut bx = Bx::build(self.cx, llbb);
             bx.unreachable();
-            self.unreachable_block = Some(llbb);
             llbb
         })
     }

compiler/rustc_const_eval/src/interpret/terminator.rs

@@ -101,12 +101,16 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                         &ImmTy::from_uint(const_int, discr.layout),
                     )?;
                     if res.to_scalar().to_bool()? {
-                        target_block = target;
+                        target_block = mir::SwitchAction::Goto(target);
                         break;
                     }
                 }
 
-                self.go_to_block(target_block);
+                if let mir::SwitchAction::Goto(target_block) = target_block {
+                    self.go_to_block(target_block);
+                } else {
+                    throw_ub!(Unreachable)
+                }
             }
 
             Call {

compiler/rustc_const_eval/src/transform/validate.rs

@@ -373,10 +373,9 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> {
                 self.check_edge(location, *target, EdgeKind::Normal);
             }
             TerminatorKind::SwitchInt { targets, discr: _ } => {
-                for (_, target) in targets.iter() {
+                for &target in targets.all_targets() {
                     self.check_edge(location, target, EdgeKind::Normal);
                 }
-                self.check_edge(location, targets.otherwise(), EdgeKind::Normal);
 
                 self.value_cache.clear();
                 self.value_cache.extend(targets.iter().map(|(value, _)| value));

compiler/rustc_middle/src/mir/basic_blocks.rs

@@ -1,5 +1,7 @@
 use crate::mir::traversal::Postorder;
-use crate::mir::{BasicBlock, BasicBlockData, Terminator, TerminatorKind, START_BLOCK};
+use crate::mir::{
+    BasicBlock, BasicBlockData, SwitchAction, Terminator, TerminatorKind, START_BLOCK,
+};
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::graph;
@@ -90,7 +92,9 @@ impl<'tcx> BasicBlocks<'tcx> {
                     for (value, target) in targets.iter() {
                         switch_sources.entry((target, bb)).or_default().push(Some(value));
                     }
-                    switch_sources.entry((targets.otherwise(), bb)).or_default().push(None);
+                    if let SwitchAction::Goto(otherwise) = targets.otherwise() {
+                        switch_sources.entry((otherwise, bb)).or_default().push(None);
+                    }
                 }
             }
             switch_sources
@@ -128,6 +132,13 @@ impl<'tcx> BasicBlocks<'tcx> {
     pub fn invalidate_cfg_cache(&mut self) {
         self.cache = Cache::default();
     }
+
+    pub fn is_empty_unreachable(&self, action: SwitchAction) -> bool {
+        match action {
+            SwitchAction::Goto(bb) => self[bb].is_empty_unreachable(),
+            SwitchAction::Unreachable => true,
+        }
+    }
 }
 
 impl<'tcx> std::ops::Deref for BasicBlocks<'tcx> {

compiler/rustc_middle/src/mir/pretty.rs

@@ -875,12 +875,14 @@ impl<'tcx> TerminatorKind<'tcx> {
         match *self {
             Return | UnwindResume | UnwindTerminate(_) | Unreachable | CoroutineDrop => vec![],
             Goto { .. } => vec!["".into()],
-            SwitchInt { ref targets, .. } => targets
-                .values
-                .iter()
-                .map(|&u| Cow::Owned(u.to_string()))
-                .chain(iter::once("otherwise".into()))
-                .collect(),
+            SwitchInt { ref targets, .. } => {
+                let mut labels: Vec<_> =
+                    targets.values.iter().map(|&u| Cow::Owned(u.to_string())).collect();
+                if let SwitchAction::Goto(_) = targets.otherwise() {
+                    labels.push("otherwise".into());
+                }
+                labels
+            }
             Call { target: Some(_), unwind: UnwindAction::Cleanup(_), .. } => {
                 vec!["return".into(), "unwind".into()]
             }

compiler/rustc_middle/src/mir/syntax.rs

@@ -828,21 +828,51 @@ pub struct SwitchTargets {
     /// are found in the corresponding indices from the `targets` vector.
     pub(super) values: SmallVec<[Pu128; 1]>,
 
-    /// Possible branch sites. The last element of this vector is used
-    /// for the otherwise branch, so targets.len() == values.len() + 1
-    /// should hold.
+    /// Possible branch sites.
+    ///
+    /// If `targets.len() == values.len() + 1`, the last element of this vector
+    /// is used for the otherwise branch in [`SwitchAction::Goto`].
+    ///
+    /// If `targets.len() == values.len()`, the otherwise branch is
+    /// [`SwitchAction::Unreachable`].
+    ///
+    /// You must ensure that we're always in at one of those cases.
     //
     // This invariant is quite non-obvious and also could be improved.
     // One way to make this invariant is to have something like this instead:
     //
     // branches: Vec<(ConstInt, BasicBlock)>,
-    // otherwise: Option<BasicBlock> // exhaustive if None
+    // otherwise: SwitchAction,
     //
     // However we’ve decided to keep this as-is until we figure a case
     // where some other approach seems to be strictly better than other.
     pub(super) targets: SmallVec<[BasicBlock; 2]>,
 }
 
+/// The action to be taken for a particular input to the [`TerminatorKind::SwitchInt`]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, TyEncodable, TyDecodable, Hash, HashStable)]
+#[derive(TypeFoldable, TypeVisitable)]
+pub enum SwitchAction {
+    /// Jump to the specified block
+    Goto(BasicBlock),
+    /// Triggers undefined behavior
+    ///
+    /// This is equivalent to jumping to a block with [`TerminatorKind::Unreachable`],
+    /// but lets us not have to store such a block in the body.
+    /// It also makes it easier in analysis to know this action is unreachable
+    /// without needing to have all the basic blocks around to look at.
+    Unreachable,
+}
+
+impl SwitchAction {
+    pub fn into_terminator<'tcx>(self) -> TerminatorKind<'tcx> {
+        match self {
+            SwitchAction::Goto(bb) => TerminatorKind::Goto { target: bb },
+            SwitchAction::Unreachable => TerminatorKind::Unreachable,
+        }
+    }
+}
+
 /// Action to be taken when a stack unwind happens.
 #[derive(Copy, Clone, Debug, PartialEq, Eq, TyEncodable, TyDecodable, Hash, HashStable)]
 #[derive(TypeFoldable, TypeVisitable)]

compiler/rustc_middle/src/mir/terminator.rs

@@ -14,10 +14,12 @@ impl SwitchTargets {
     ///
     /// The iterator may be empty, in which case the `SwitchInt` instruction is equivalent to
     /// `goto otherwise;`.
-    pub fn new(targets: impl Iterator<Item = (u128, BasicBlock)>, otherwise: BasicBlock) -> Self {
+    pub fn new(targets: impl Iterator<Item = (u128, BasicBlock)>, otherwise: SwitchAction) -> Self {
         let (values, mut targets): (SmallVec<_>, SmallVec<_>) =
             targets.map(|(v, t)| (Pu128(v), t)).unzip();
-        targets.push(otherwise);
+        if let SwitchAction::Goto(otherwise) = otherwise {
+            targets.push(otherwise);
+        }
         Self { values, targets }
     }
 
@@ -39,10 +41,17 @@ impl SwitchTargets {
         }
     }
 
-    /// Returns the fallback target that is jumped to when none of the values match the operand.
+    /// Returns the fallback action when none of the values match the operand.
     #[inline]
-    pub fn otherwise(&self) -> BasicBlock {
-        *self.targets.last().unwrap()
+    pub fn otherwise(&self) -> SwitchAction {
+        debug_assert!(
+            self.targets.len() == self.values.len() || self.targets.len() == self.values.len() + 1
+        );
+        if self.targets.len() > self.values.len() {
+            SwitchAction::Goto(*self.targets.last().unwrap())
+        } else {
+            SwitchAction::Unreachable
+        }
     }
 
     /// Returns an iterator over the switch targets.
@@ -71,8 +80,9 @@ impl SwitchTargets {
     /// specific value. This cannot fail, as it'll return the `otherwise`
     /// branch if there's not a specific match for the value.
     #[inline]
-    pub fn target_for_value(&self, value: u128) -> BasicBlock {
-        self.iter().find_map(|(v, t)| (v == value).then_some(t)).unwrap_or_else(|| self.otherwise())
+    pub fn target_for_value(&self, value: u128) -> SwitchAction {
+        let specific = self.iter().find_map(|(v, t)| (v == value).then_some(t));
+        if let Some(specific) = specific { SwitchAction::Goto(specific) } else { self.otherwise() }
     }
 
     /// Adds a new target to the switch. But You cannot add an already present value.
@@ -82,8 +92,12 @@ impl SwitchTargets {
         if self.values.contains(&value) {
             bug!("target value {:?} already present", value);
         }
+
+        // There may or may not be a fallback in `targets`, so make sure to
+        // insert the new target at the same index as its value.
+        let insert_point = self.values.len();
         self.values.push(value);
-        self.targets.insert(self.targets.len() - 1, bb);
+        self.targets.insert(insert_point, bb);
     }
 
     /// Returns true if all targets (including the fallback target) are distinct.

compiler/rustc_middle/src/mir/traversal.rs

@@ -361,8 +361,9 @@ impl<'a, 'tcx> Iterator for MonoReachable<'a, 'tcx> {
             if let Some((bits, targets)) =
                 Body::try_const_mono_switchint(self.tcx, self.instance, data)
             {
-                let target = targets.target_for_value(bits);
-                self.add_work([target]);
+                if let SwitchAction::Goto(target) = targets.target_for_value(bits) {
+                    self.add_work([target]);
+                }
             } else {
                 self.add_work(data.terminator().successors());
             }