Auto merge of rust-lang#146096 - adwinwhite:handle_normalization_overflow_in_mono1, r=saethlin

Fix normalization overflow ICEs in monomorphization

Fixes rust-lang#92004
Fixes rust-lang#92470
Fixes rust-lang#95134
Fixes rust-lang#105275
Fixes rust-lang#105937
Fixes rust-lang#117696-2
Fixes rust-lang#118590
Fixes rust-lang#122823
Fixes rust-lang#131342
Fixes rust-lang#139659

## Analysis:
The causes of these issues are similar. They contain generic recursive functions that can be instantiated with different args infinitely at monomorphization stage.
Ideally this should be caught by the [`check_recursion_limit`](https://github.com/rust-lang/rust/blob/c0bb3b98bb7aac24a37635e5d36d961e0b14f435/compiler/rustc_monomorphize/src/collector.rs#L468) function. The reality is that normalization can reach recursion limit earlier than monomorphization's check because they calculate depths in different ways.
Since normalization is called everywhere, ICEs appear in different locations.

## Fix:
If we abort on overflow with `TypingMode::PostAnalysis` in the trait solver, it would also catch these errors.
The main challenge is providing good diagnostics for them. So it's quite natural to put the check right before these normalization happening.
I first tried to check the whole MIR body's normalization and `references_error`. (As elaborate_drop handles normalization failure by [returning `ty::Error`](https://github.com/rust-lang/rust/blob/c0bb3b98bb7aac24a37635e5d36d961e0b14f435/compiler/rustc_mir_transform/src/elaborate_drop.rs#L514-L519).)
It turns out that checking all `Local`s seems sufficient.
These types are gonna be normalized anyway. So with cache, these checks shouldn't be expensive.

This fixes these ICEs for both the next and old solver, though I'm not sure the change I made to the old solver is proper. Its overflow handling looks convoluted thus I didn't try to fix it more "upstream".

Author: bors

compiler/rustc_middle/src/mir/mono.rs

@@ -47,6 +47,9 @@ pub enum InstantiationMode {
     LocalCopy,
 }
 
+#[derive(PartialEq, Eq, Clone, Copy, Debug, Hash, HashStable, TyEncodable, TyDecodable)]
+pub struct NormalizationErrorInMono;
+
 #[derive(PartialEq, Eq, Clone, Copy, Debug, Hash, HashStable, TyEncodable, TyDecodable)]
 pub enum MonoItem<'tcx> {
     Fn(Instance<'tcx>),

compiler/rustc_middle/src/query/erase.rs

@@ -3,8 +3,10 @@ use std::intrinsics::transmute_unchecked;
 use std::mem::MaybeUninit;
 
 use rustc_span::ErrorGuaranteed;
+use rustc_span::source_map::Spanned;
 
 use crate::mir::interpret::EvalToValTreeResult;
+use crate::mir::mono::{MonoItem, NormalizationErrorInMono};
 use crate::query::CyclePlaceholder;
 use crate::traits::solve;
 use crate::ty::adjustment::CoerceUnsizedInfo;
@@ -171,6 +173,17 @@ impl EraseType for Result<ty::EarlyBinder<'_, Ty<'_>>, CyclePlaceholder> {
     type Result = [u8; size_of::<Result<ty::EarlyBinder<'static, Ty<'_>>, CyclePlaceholder>>()];
 }
 
+impl EraseType
+    for Result<(&'_ [Spanned<MonoItem<'_>>], &'_ [Spanned<MonoItem<'_>>]), NormalizationErrorInMono>
+{
+    type Result = [u8; size_of::<
+        Result<
+            (&'static [Spanned<MonoItem<'static>>], &'static [Spanned<MonoItem<'static>>]),
+            NormalizationErrorInMono,
+        >,
+    >()];
+}
+
 impl<T> EraseType for Option<&'_ T> {
     type Result = [u8; size_of::<Option<&'static ()>>()];
 }

compiler/rustc_middle/src/query/mod.rs

@@ -116,7 +116,9 @@ use crate::mir::interpret::{
     EvalStaticInitializerRawResult, EvalToAllocationRawResult, EvalToConstValueResult,
     EvalToValTreeResult, GlobalId, LitToConstInput,
 };
-use crate::mir::mono::{CodegenUnit, CollectionMode, MonoItem, MonoItemPartitions};
+use crate::mir::mono::{
+    CodegenUnit, CollectionMode, MonoItem, MonoItemPartitions, NormalizationErrorInMono,
+};
 use crate::query::erase::{Erase, erase, restore};
 use crate::query::plumbing::{CyclePlaceholder, DynamicQuery};
 use crate::traits::query::{
@@ -2702,7 +2704,7 @@ rustc_queries! {
         desc { "functions to skip for move-size check" }
     }
 
-    query items_of_instance(key: (ty::Instance<'tcx>, CollectionMode)) -> (&'tcx [Spanned<MonoItem<'tcx>>], &'tcx [Spanned<MonoItem<'tcx>>]) {
+    query items_of_instance(key: (ty::Instance<'tcx>, CollectionMode)) -> Result<(&'tcx [Spanned<MonoItem<'tcx>>], &'tcx [Spanned<MonoItem<'tcx>>]), NormalizationErrorInMono> {
         desc { "collecting items used by `{}`", key.0 }
         cache_on_disk_if { true }
     }

compiler/rustc_monomorphize/src/collector.rs

@@ -208,6 +208,7 @@
 mod autodiff;
 
 use std::cell::OnceCell;
+use std::ops::ControlFlow;
 
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::sync::{MTLock, par_for_each_in};
@@ -220,15 +221,17 @@ use rustc_hir::lang_items::LangItem;
 use rustc_hir::limit::Limit;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::interpret::{AllocId, ErrorHandled, GlobalAlloc, Scalar};
-use rustc_middle::mir::mono::{CollectionMode, InstantiationMode, MonoItem};
+use rustc_middle::mir::mono::{
+    CollectionMode, InstantiationMode, MonoItem, NormalizationErrorInMono,
+};
 use rustc_middle::mir::visit::Visitor as MirVisitor;
-use rustc_middle::mir::{self, Location, MentionedItem, traversal};
+use rustc_middle::mir::{self, Body, Location, MentionedItem, traversal};
 use rustc_middle::query::TyCtxtAt;
 use rustc_middle::ty::adjustment::{CustomCoerceUnsized, PointerCoercion};
 use rustc_middle::ty::layout::ValidityRequirement;
 use rustc_middle::ty::{
     self, GenericArgs, GenericParamDefKind, Instance, InstanceKind, Ty, TyCtxt, TypeFoldable,
-    TypeVisitableExt, VtblEntry,
+    TypeVisitable, TypeVisitableExt, TypeVisitor, VtblEntry,
 };
 use rustc_middle::util::Providers;
 use rustc_middle::{bug, span_bug};
@@ -474,7 +477,22 @@ fn collect_items_rec<'tcx>(
             ));
 
             rustc_data_structures::stack::ensure_sufficient_stack(|| {
-                let (used, mentioned) = tcx.items_of_instance((instance, mode));
+                let Ok((used, mentioned)) = tcx.items_of_instance((instance, mode)) else {
+                    // Normalization errors here are usually due to trait solving overflow.
+                    // FIXME: I assume that there are few type errors at post-analysis stage, but not
+                    // entirely sure.
+                    // We have to emit the error outside of `items_of_instance` to access the
+                    // span of the `starting_item`.
+                    let def_id = instance.def_id();
+                    let def_span = tcx.def_span(def_id);
+                    let def_path_str = tcx.def_path_str(def_id);
+                    tcx.dcx().emit_fatal(RecursionLimit {
+                        span: starting_item.span,
+                        instance,
+                        def_span,
+                        def_path_str,
+                    });
+                };
                 used_items.extend(used.into_iter().copied());
                 mentioned_items.extend(mentioned.into_iter().copied());
             });
@@ -603,6 +621,35 @@ fn collect_items_rec<'tcx>(
     }
 }
 
+// Check whether we can normalize every type in the instantiated MIR body.
+fn check_normalization_error<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    instance: Instance<'tcx>,
+    body: &Body<'tcx>,
+) -> Result<(), NormalizationErrorInMono> {
+    struct NormalizationChecker<'tcx> {
+        tcx: TyCtxt<'tcx>,
+        instance: Instance<'tcx>,
+    }
+    impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for NormalizationChecker<'tcx> {
+        type Result = ControlFlow<()>;
+
+        fn visit_ty(&mut self, t: Ty<'tcx>) -> Self::Result {
+            match self.instance.try_instantiate_mir_and_normalize_erasing_regions(
+                self.tcx,
+                ty::TypingEnv::fully_monomorphized(),
+                ty::EarlyBinder::bind(t),
+            ) {
+                Ok(_) => ControlFlow::Continue(()),
+                Err(_) => ControlFlow::Break(()),
+            }
+        }
+    }
+
+    let mut checker = NormalizationChecker { tcx, instance };
+    if body.visit_with(&mut checker).is_break() { Err(NormalizationErrorInMono) } else { Ok(()) }
+}
+
 fn check_recursion_limit<'tcx>(
     tcx: TyCtxt<'tcx>,
     instance: Instance<'tcx>,
@@ -1259,11 +1306,15 @@ fn collect_items_of_instance<'tcx>(
     tcx: TyCtxt<'tcx>,
     instance: Instance<'tcx>,
     mode: CollectionMode,
-) -> (MonoItems<'tcx>, MonoItems<'tcx>) {
+) -> Result<(MonoItems<'tcx>, MonoItems<'tcx>), NormalizationErrorInMono> {
     // This item is getting monomorphized, do mono-time checks.
+    let body = tcx.instance_mir(instance.def);
+    // Plenty of code paths later assume that everything can be normalized. So we have to check
+    // normalization first.
+    // We choose to emit the error outside to provide helpful diagnostics.
+    check_normalization_error(tcx, instance, body)?;
     tcx.ensure_ok().check_mono_item(instance);
 
-    let body = tcx.instance_mir(instance.def);
     // Naively, in "used" collection mode, all functions get added to *both* `used_items` and
     // `mentioned_items`. Mentioned items processing will then notice that they have already been
     // visited, but at that point each mentioned item has been monomorphized, added to the
@@ -1313,19 +1364,22 @@ fn collect_items_of_instance<'tcx>(
         }
     }
 
-    (used_items, mentioned_items)
+    Ok((used_items, mentioned_items))
 }
 
 fn items_of_instance<'tcx>(
     tcx: TyCtxt<'tcx>,
     (instance, mode): (Instance<'tcx>, CollectionMode),
-) -> (&'tcx [Spanned<MonoItem<'tcx>>], &'tcx [Spanned<MonoItem<'tcx>>]) {
-    let (used_items, mentioned_items) = collect_items_of_instance(tcx, instance, mode);
+) -> Result<
+    (&'tcx [Spanned<MonoItem<'tcx>>], &'tcx [Spanned<MonoItem<'tcx>>]),
+    NormalizationErrorInMono,
+> {
+    let (used_items, mentioned_items) = collect_items_of_instance(tcx, instance, mode)?;
 
     let used_items = tcx.arena.alloc_from_iter(used_items);
     let mentioned_items = tcx.arena.alloc_from_iter(mentioned_items);
 
-    (used_items, mentioned_items)
+    Ok((used_items, mentioned_items))
 }
 
 /// `item` must be already monomorphized.

compiler/rustc_traits/src/normalize_erasing_regions.rs

@@ -42,7 +42,18 @@ fn try_normalize_after_erasing_regions<'tcx, T: TypeFoldable<TyCtxt<'tcx>> + Par
             // us a test case.
             debug_assert_eq!(normalized_value, resolved_value);
             let erased = infcx.tcx.erase_and_anonymize_regions(resolved_value);
-            debug_assert!(!erased.has_infer(), "{erased:?}");
+            if infcx.next_trait_solver() {
+                debug_assert!(!erased.has_infer(), "{erased:?}");
+            } else {
+                // The old solver returns an ty var with the failed obligation in case of
+                // selection error. And when the obligation is re-tried, the error should be
+                // reported. However in case of overflow error, the obligation may be fulfilled
+                // due to the original depth being dropped.
+                // In conclusion, overflow results in an unconstrained ty var.
+                if erased.has_infer() {
+                    return Err(NoSolution);
+                }
+            }
             Ok(erased)
         }
         Err(NoSolution) => Err(NoSolution),

tests/crashes/105275.rs renamed to tests/ui/codegen/normalization-overflow/recursion-issue-105275.rs

@@ -1,9 +1,10 @@
-//@ known-bug: #105275
+//@ build-fail
 //@ compile-flags: -Copt-level=0
 
 pub fn encode_num<Writer: ExampleWriter>(n: u32, mut writer: Writer) -> Result<(), Writer::Error> {
     if n > 15 {
         encode_num(n / 16, &mut writer)?;
+        //~^ ERROR: reached the recursion limit while instantiating
     }
     Ok(())
 }

tests/ui/codegen/normalization-overflow/recursion-issue-105275.stderr

@@ -0,0 +1,14 @@
+error: reached the recursion limit while instantiating `encode_num::<&mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut Error>`
+  --> $DIR/recursion-issue-105275.rs:6:9
+   |
+LL |         encode_num(n / 16, &mut writer)?;
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+note: `encode_num` defined here
+  --> $DIR/recursion-issue-105275.rs:4:1
+   |
+LL | pub fn encode_num<Writer: ExampleWriter>(n: u32, mut writer: Writer) -> Result<(), Writer::Error> {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to 1 previous error
+

tests/crashes/105937.rs renamed to tests/ui/codegen/normalization-overflow/recursion-issue-105937.rs

@@ -1,9 +1,10 @@
-//@ known-bug: #105937
+//@ build-fail
 //@ compile-flags: -Copt-level=0
 
 pub fn encode_num<Writer: ExampleWriter>(n: u32, mut writer: Writer) -> Result<(), Writer::Error> {
     if n > 15 {
         encode_num(n / 16, &mut writer)?;
+        //~^ ERROR: reached the recursion limit while instantiating
     }
     Ok(())
 }

tests/ui/codegen/normalization-overflow/recursion-issue-105937.stderr

@@ -0,0 +1,14 @@
+error: reached the recursion limit while instantiating `encode_num::<&mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut Error>`
+  --> $DIR/recursion-issue-105937.rs:6:9
+   |
+LL |         encode_num(n / 16, &mut writer)?;
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+note: `encode_num` defined here
+  --> $DIR/recursion-issue-105937.rs:4:1
+   |
+LL | pub fn encode_num<Writer: ExampleWriter>(n: u32, mut writer: Writer) -> Result<(), Writer::Error> {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to 1 previous error
+

tests/crashes/117696-1.rs renamed to tests/ui/codegen/normalization-overflow/recursion-issue-117696-1.rs

@@ -1,4 +1,4 @@
-//@ known-bug: #117696
+//@ build-fail
 fn main() {
     let mut it = (Empty);
     rec(&mut it);
@@ -23,6 +23,7 @@ where
 {
     if () == () {
         T::count(it);
+        //~^ ERROR: reached the recursion limit while instantiating
     } else {
         rec(identity(&mut it))
     }