Rollup merge of rust-lang#136311 - compiler-errors:vtable-fixes-2, r=lcnr

Ensure that we never try to monomorphize the upcasting or vtable calls of impossible dyn types

Check for impossible obligations in the `dyn Trait` type we're trying to compute its the vtable upcasting and method call slots.

r? lcnr

Author: matthiaskrgr

compiler/rustc_trait_selection/src/traits/vtable.rs

@@ -318,9 +318,17 @@ pub(crate) fn first_method_vtable_slot<'tcx>(tcx: TyCtxt<'tcx>, key: ty::TraitRe
         bug!();
     };
     let source_principal = tcx.instantiate_bound_regions_with_erased(
-        source.principal().unwrap().with_self_ty(tcx, tcx.types.trait_object_dummy_self),
+        source.principal().unwrap().with_self_ty(tcx, key.self_ty()),
     );
 
+    // We're monomorphizing a call to a dyn trait object that can never be constructed.
+    if tcx.instantiate_and_check_impossible_predicates((
+        source_principal.def_id,
+        source_principal.args,
+    )) {
+        return 0;
+    }
+
     let target_principal = ty::ExistentialTraitRef::erase_self_ty(tcx, key);
 
     let vtable_segment_callback = {
@@ -373,19 +381,27 @@ pub(crate) fn supertrait_vtable_slot<'tcx>(
     let (source, target) = key;
 
     // If the target principal is `None`, we can just return `None`.
-    let ty::Dynamic(target, _, _) = *target.kind() else {
+    let ty::Dynamic(target_data, _, _) = *target.kind() else {
         bug!();
     };
-    let target_principal = tcx.instantiate_bound_regions_with_erased(target.principal()?);
+    let target_principal = tcx.instantiate_bound_regions_with_erased(target_data.principal()?);
 
     // Given that we have a target principal, it is a bug for there not to be a source principal.
-    let ty::Dynamic(source, _, _) = *source.kind() else {
+    let ty::Dynamic(source_data, _, _) = *source.kind() else {
         bug!();
     };
     let source_principal = tcx.instantiate_bound_regions_with_erased(
-        source.principal().unwrap().with_self_ty(tcx, tcx.types.trait_object_dummy_self),
+        source_data.principal().unwrap().with_self_ty(tcx, source),
     );
 
+    // We're monomorphizing a dyn trait object upcast that can never be constructed.
+    if tcx.instantiate_and_check_impossible_predicates((
+        source_principal.def_id,
+        source_principal.args,
+    )) {
+        return None;
+    }
+
     let vtable_segment_callback = {
         let mut vptr_offset = 0;
         move |segment| {

tests/ui/traits/trait-upcasting/mono-impossible.rs

@@ -0,0 +1,26 @@
+//@ build-pass
+
+#![feature(trait_upcasting)]
+
+trait Supertrait<T> {
+    fn method(&self) {}
+}
+impl<T> Supertrait<T> for () {}
+
+trait WithAssoc {
+    type Assoc;
+}
+trait Trait<P: WithAssoc>: Supertrait<P::Assoc> + Supertrait<()> {}
+
+fn upcast<P>(x: &dyn Trait<P>) -> &dyn Supertrait<()> {
+    x
+}
+
+fn call<P>(x: &dyn Trait<P>) {
+    x.method();
+}
+
+fn main() {
+    println!("{:p}", upcast::<()> as *const ());
+    println!("{:p}", call::<()> as *const ());
+}