Move critical edge splitting to codegen.

Author: cjgillot

Cargo.lock

@@ -3572,6 +3572,7 @@ dependencies = [
  "rustc_macros",
  "rustc_metadata",
  "rustc_middle",
+ "rustc_mir_transform",
  "rustc_query_system",
  "rustc_serialize",
  "rustc_session",

compiler/rustc_codegen_ssa/Cargo.toml

@@ -30,6 +30,7 @@ rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_metadata = { path = "../rustc_metadata" }
 rustc_middle = { path = "../rustc_middle" }
+rustc_mir_transform = { path = "../rustc_mir_transform" }
 rustc_query_system = { path = "../rustc_query_system" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -7,6 +7,7 @@ use rustc_middle::mir::{Body, Local, UnwindTerminateReason, traversal};
 use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, TyAndLayout};
 use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeVisitableExt};
 use rustc_middle::{bug, mir, span_bug};
+use rustc_mir_transform::{add_call_guards, dump_mir, pass_manager};
 use rustc_target::callconv::{FnAbi, PassMode};
 use tracing::{debug, instrument};
 
@@ -167,6 +168,18 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         ty::TypingEnv::fully_monomorphized(),
         ty::EarlyBinder::bind(mir.clone()),
     );
+    pass_manager::run_passes(
+        cx.tcx(),
+        &mut mir,
+        &[
+            // Some cleanup necessary at least for LLVM and potentially other codegen backends.
+            &add_call_guards::CriticalCallEdges,
+            // Dump the end result for testing and debugging purposes.
+            &dump_mir::Marker("Monomorphic"),
+        ],
+        Some(mir::MirPhase::Runtime(mir::RuntimePhase::Monomorphic)),
+        pass_manager::Optimizations::Allowed,
+    );
     if tcx.features().ergonomic_clones() {
         mir = optimize_use_clone::<Bx>(cx, mir);
     }

compiler/rustc_middle/src/mir/mod.rs

@@ -103,6 +103,7 @@ impl MirPhase {
             MirPhase::Runtime(RuntimePhase::Initial) => "runtime",
             MirPhase::Runtime(RuntimePhase::PostCleanup) => "runtime-post-cleanup",
             MirPhase::Runtime(RuntimePhase::Optimized) => "runtime-optimized",
+            MirPhase::Runtime(RuntimePhase::Monomorphic) => "runtime-monomorphic",
         }
     }
 
@@ -154,6 +155,7 @@ impl RuntimePhase {
             "initial" => Self::Initial,
             "post_cleanup" | "post-cleanup" | "postcleanup" => Self::PostCleanup,
             "optimized" => Self::Optimized,
+            "monomorphic" => Self::Monomorphic,
             _ => bug!("Unknown runtime phase: '{}'", phase),
         }
     }

compiler/rustc_middle/src/mir/syntax.rs

@@ -143,6 +143,8 @@ pub enum RuntimePhase {
     /// * [`ProjectionElem::Deref`] of `Box`
     PostCleanup = 1,
     Optimized = 2,
+    /// This corresponds to monomorphised MIR that is being processed by codegen.
+    Monomorphic = 3,
 }
 
 ///////////////////////////////////////////////////////////////////////////

compiler/rustc_mir_transform/src/add_call_guards.rs

@@ -4,11 +4,11 @@ use rustc_middle::ty::TyCtxt;
 use tracing::debug;
 
 #[derive(PartialEq)]
-pub(super) enum AddCallGuards {
+pub enum AddCallGuards {
     AllCallEdges,
     CriticalCallEdges,
 }
-pub(super) use self::AddCallGuards::*;
+pub use self::AddCallGuards::*;
 
 /**
  * Breaks outgoing critical edges for call terminators in the MIR.

compiler/rustc_mir_transform/src/dump_mir.rs

@@ -7,7 +7,7 @@ use rustc_middle::mir::{Body, write_mir_pretty};
 use rustc_middle::ty::TyCtxt;
 use rustc_session::config::{OutFileName, OutputType};
 
-pub(super) struct Marker(pub &'static str);
+pub struct Marker(pub &'static str);
 
 impl<'tcx> crate::MirPass<'tcx> for Marker {
     fn name(&self) -> &'static str {

compiler/rustc_mir_transform/src/lib.rs

@@ -35,7 +35,7 @@ use rustc_span::{DUMMY_SP, sym};
 use tracing::debug;
 
 #[macro_use]
-mod pass_manager;
+pub mod pass_manager;
 
 use std::sync::LazyLock;
 
@@ -110,7 +110,7 @@ macro_rules! declare_passes {
 
 declare_passes! {
     mod abort_unwinding_calls : AbortUnwindingCalls;
-    mod add_call_guards : AddCallGuards { AllCallEdges, CriticalCallEdges };
+    pub mod add_call_guards : AddCallGuards { AllCallEdges, CriticalCallEdges };
     mod add_moves_for_packed_drops : AddMovesForPackedDrops;
     mod add_retag : AddRetag;
     mod add_subtyping_projections : Subtyper;
@@ -722,8 +722,6 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'
             &simplify::SimplifyLocals::Final,
             &multiple_return_terminators::MultipleReturnTerminators,
             &large_enums::EnumSizeOpt { discrepancy: 128 },
-            // Some cleanup necessary at least for LLVM and potentially other codegen backends.
-            &add_call_guards::CriticalCallEdges,
             // Cleanup for human readability, off by default.
             &prettify::ReorderBasicBlocks,
             &prettify::ReorderLocals,

compiler/rustc_mir_transform/src/pass_manager.rs

@@ -60,7 +60,7 @@ const fn c_name(name: &'static str) -> &'static str {
 /// A streamlined trait that you can implement to create a pass; the
 /// pass will be named after the type, and it will consist of a main
 /// loop that goes over each available MIR and applies `run_pass`.
-pub(super) trait MirPass<'tcx> {
+pub trait MirPass<'tcx> {
     fn name(&self) -> &'static str {
         // FIXME(const-hack) Simplify the implementation once more `str` methods get const-stable.
         // See copypaste in `MirLint`
@@ -172,7 +172,7 @@ where
 ///
 /// [required]: MirPass::is_required
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
-pub(crate) enum Optimizations {
+pub enum Optimizations {
     Suppressed,
     Allowed,
 }
@@ -189,7 +189,7 @@ pub(super) fn run_passes_no_validate<'tcx>(
 }
 
 /// The optional `phase_change` is applied after executing all the passes, if present
-pub(super) fn run_passes<'tcx>(
+pub fn run_passes<'tcx>(
     tcx: TyCtxt<'tcx>,
     body: &mut Body<'tcx>,
     passes: &[&dyn MirPass<'tcx>],

compiler/rustc_mir_transform/src/validate.rs

@@ -390,10 +390,8 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> {
 
                     // The code generation assumes that there are no critical call edges. The
                     // assumption is used to simplify inserting code that should be executed along
-                    // the return edge from the call. FIXME(tmiasko): Since this is a strictly code
-                    // generation concern, the code generation should be responsible for handling
-                    // it.
-                    if self.body.phase >= MirPhase::Runtime(RuntimePhase::Optimized)
+                    // the return edge from the call.
+                    if self.body.phase >= MirPhase::Runtime(RuntimePhase::Monomorphic)
                         && self.is_critical_call_edge(target, unwind)
                     {
                         self.fail(