Rollup merge of #147243 - folkertdev:cmse-bail-impl-trait, r=davidtwco

matthiaskrgr · web-flow · commit 7ab1fd1816cf · 2025-10-07T17:42:11.000+02:00
cmse: disallow `impl Trait` in `cmse-nonsecure-entry` return types tracking issue: #75835 fixes #147242 Refactors some logic to be more robust in the future, and then disallows `impl Trait` as a return type for the cmse ABIs. The `is_valid_cmse_output_layout` function disallows `union` values like before. That is not entirely correct, but preserves the current behavior. Some additional logic is needed for `union` values (and any types where parts may be uninitialized) that I'll tackle in a later PR. can be reviewed commit-by-commit. r? types
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/cmse.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/cmse.rs
@@ -1,9 +1,9 @@
-use rustc_abi::ExternAbi;
+use rustc_abi::{BackendRepr, ExternAbi, Float, Integer, Primitive, Scalar};
 use rustc_errors::{DiagCtxtHandle, E0781, struct_span_code_err};
 use rustc_hir::{self as hir, HirId};
 use rustc_middle::bug;
-use rustc_middle::ty::layout::LayoutError;
-use rustc_middle::ty::{self, TyCtxt};
+use rustc_middle::ty::layout::{LayoutError, TyAndLayout};
+use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt};
 
 use crate::errors;
 
@@ -164,44 +164,48 @@ fn is_valid_cmse_output<'tcx>(
 ) -> Result<bool, &'tcx LayoutError<'tcx>> {
     // this type is only used for layout computation, which does not rely on regions
     let fn_sig = tcx.instantiate_bound_regions_with_erased(fn_sig);
+    let fn_sig = tcx.erase_and_anonymize_regions(fn_sig);
+    let return_type = fn_sig.output();
+
+    // `impl Trait` is already disallowed with `cmse-nonsecure-call`, because that ABI is only
+    // allowed on function pointers, and function pointers cannot contain `impl Trait` in their
+    // signature.
+    //
+    // Here we explicitly disallow `impl Trait` in the `cmse-nonsecure-entry` return type too, to
+    // prevent query cycles when calculating the layout. This ABI is meant to be used with
+    // `#[no_mangle]` or similar, so generics in the type really don't make sense.
+    //
+    // see also https://github.com/rust-lang/rust/issues/147242.
+    if return_type.has_opaque_types() {
+        return Err(tcx.arena.alloc(LayoutError::TooGeneric(return_type)));
+    }
 
     let typing_env = ty::TypingEnv::fully_monomorphized();
+    let layout = tcx.layout_of(typing_env.as_query_input(return_type))?;
+
+    Ok(is_valid_cmse_output_layout(layout))
+}
 
-    let mut ret_ty = fn_sig.output();
-    let layout = tcx.layout_of(typing_env.as_query_input(ret_ty))?;
+/// Returns whether the output will fit into the available registers
+fn is_valid_cmse_output_layout<'tcx>(layout: TyAndLayout<'tcx>) -> bool {
     let size = layout.layout.size().bytes();
 
     if size <= 4 {
-        return Ok(true);
+        return true;
     } else if size > 8 {
-        return Ok(false);
+        return false;
     }
 
-    // next we need to peel any repr(transparent) layers off
-    'outer: loop {
-        let ty::Adt(adt_def, args) = ret_ty.kind() else {
-            break;
-        };
-
-        if !adt_def.repr().transparent() {
-            break;
-        }
-
-        // the first field with non-trivial size and alignment must be the data
-        for variant_def in adt_def.variants() {
-            for field_def in variant_def.fields.iter() {
-                let ty = field_def.ty(tcx, args);
-                let layout = tcx.layout_of(typing_env.as_query_input(ty))?;
+    // Accept scalar 64-bit types.
+    let BackendRepr::Scalar(scalar) = layout.layout.backend_repr else {
+        return false;
+    };
 
-                if !layout.layout.is_1zst() {
-                    ret_ty = ty;
-                    continue 'outer;
-                }
-            }
-        }
-    }
+    let Scalar::Initialized { value, .. } = scalar else {
+        return false;
+    };
 
-    Ok(ret_ty == tcx.types.i64 || ret_ty == tcx.types.u64 || ret_ty == tcx.types.f64)
+    matches!(value, Primitive::Int(Integer::I64, _) | Primitive::Float(Float::F64))
 }
 
 fn should_emit_generic_error<'tcx>(abi: ExternAbi, layout_err: &'tcx LayoutError<'tcx>) -> bool {
diff --git a/tests/ui/cmse-nonsecure/cmse-nonsecure-call/generics.rs b/tests/ui/cmse-nonsecure/cmse-nonsecure-call/generics.rs
@@ -14,8 +14,9 @@ struct Test<T: Copy> {
     f1: extern "cmse-nonsecure-call" fn<U: Copy>(U, u32, u32, u32) -> u64,
     //~^ ERROR cannot find type `U` in this scope
     //~| ERROR function pointer types may not have generic parameters
-    f2: extern "cmse-nonsecure-call" fn(impl Copy, u32, u32, u32) -> u64,
+    f2: extern "cmse-nonsecure-call" fn(impl Copy, u32, u32, u32) -> impl Copy,
     //~^ ERROR `impl Trait` is not allowed in `fn` pointer parameters
+    //~| ERROR `impl Trait` is not allowed in `fn` pointer return types
     f3: extern "cmse-nonsecure-call" fn(T, u32, u32, u32) -> u64, //~ ERROR [E0798]
     f4: extern "cmse-nonsecure-call" fn(Wrapper<T>, u32, u32, u32) -> u64, //~ ERROR [E0798]
 }
diff --git a/tests/ui/cmse-nonsecure/cmse-nonsecure-call/generics.stderr b/tests/ui/cmse-nonsecure/cmse-nonsecure-call/generics.stderr
@@ -25,25 +25,33 @@ LL | struct Test<T: Copy, U> {
 error[E0562]: `impl Trait` is not allowed in `fn` pointer parameters
   --> $DIR/generics.rs:17:41
    |
-LL |     f2: extern "cmse-nonsecure-call" fn(impl Copy, u32, u32, u32) -> u64,
+LL |     f2: extern "cmse-nonsecure-call" fn(impl Copy, u32, u32, u32) -> impl Copy,
    |                                         ^^^^^^^^^
    |
    = note: `impl Trait` is only allowed in arguments and return types of functions and methods
 
+error[E0562]: `impl Trait` is not allowed in `fn` pointer return types
+  --> $DIR/generics.rs:17:70
+   |
+LL |     f2: extern "cmse-nonsecure-call" fn(impl Copy, u32, u32, u32) -> impl Copy,
+   |                                                                      ^^^^^^^^^
+   |
+   = note: `impl Trait` is only allowed in arguments and return types of functions and methods
+
 error[E0798]: function pointers with the `"cmse-nonsecure-call"` ABI cannot contain generics in their type
-  --> $DIR/generics.rs:19:9
+  --> $DIR/generics.rs:20:9
    |
 LL |     f3: extern "cmse-nonsecure-call" fn(T, u32, u32, u32) -> u64,
    |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error[E0798]: function pointers with the `"cmse-nonsecure-call"` ABI cannot contain generics in their type
-  --> $DIR/generics.rs:20:9
+  --> $DIR/generics.rs:21:9
    |
 LL |     f4: extern "cmse-nonsecure-call" fn(Wrapper<T>, u32, u32, u32) -> u64,
    |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error[E0798]: return value of `"cmse-nonsecure-call"` function too large to pass via registers
-  --> $DIR/generics.rs:26:71
+  --> $DIR/generics.rs:27:71
    |
 LL | type WithTraitObject = extern "cmse-nonsecure-call" fn(&dyn Trait) -> &dyn Trait;
    |                                                                       ^^^^^^^^^^ this type doesn't fit in the available registers
@@ -52,7 +60,7 @@ LL | type WithTraitObject = extern "cmse-nonsecure-call" fn(&dyn Trait) -> &dyn
    = note: the result must either be a (transparently wrapped) i64, u64 or f64, or be at most 4 bytes in size
 
 error[E0798]: return value of `"cmse-nonsecure-call"` function too large to pass via registers
-  --> $DIR/generics.rs:30:60
+  --> $DIR/generics.rs:31:60
    |
 LL |     extern "cmse-nonsecure-call" fn(&'static dyn Trait) -> &'static dyn Trait;
    |                                                            ^^^^^^^^^^^^^^^^^^ this type doesn't fit in the available registers
@@ -61,7 +69,7 @@ LL |     extern "cmse-nonsecure-call" fn(&'static dyn Trait) -> &'static dyn Tra
    = note: the result must either be a (transparently wrapped) i64, u64 or f64, or be at most 4 bytes in size
 
 error[E0798]: return value of `"cmse-nonsecure-call"` function too large to pass via registers
-  --> $DIR/generics.rs:37:60
+  --> $DIR/generics.rs:38:60
    |
 LL |     extern "cmse-nonsecure-call" fn(WrapperTransparent) -> WrapperTransparent;
    |                                                            ^^^^^^^^^^^^^^^^^^ this type doesn't fit in the available registers
@@ -70,12 +78,12 @@ LL |     extern "cmse-nonsecure-call" fn(WrapperTransparent) -> WrapperTranspare
    = note: the result must either be a (transparently wrapped) i64, u64 or f64, or be at most 4 bytes in size
 
 error[E0045]: C-variadic functions with the "cmse-nonsecure-call" calling convention are not supported
-  --> $DIR/generics.rs:40:20
+  --> $DIR/generics.rs:41:20
    |
 LL | type WithVarArgs = extern "cmse-nonsecure-call" fn(u32, ...);
    |                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ C-variadic function must have a compatible calling convention
 
-error: aborting due to 9 previous errors
+error: aborting due to 10 previous errors
 
 Some errors have detailed explanations: E0045, E0412, E0562, E0798.
 For more information about an error, try `rustc --explain E0045`.
diff --git a/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/generics.rs b/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/generics.rs
@@ -65,3 +65,15 @@ extern "cmse-nonsecure-entry" fn wrapped_trait_object(x: WrapperTransparent) ->
     //~^ ERROR return value of `"cmse-nonsecure-entry"` function too large to pass via registers [E0798]
     x
 }
+
+extern "cmse-nonsecure-entry" fn return_impl_trait(_: impl Copy) -> impl Copy {
+    //~^ ERROR functions with the `"cmse-nonsecure-entry"` ABI cannot contain generics in their type
+    //~| ERROR functions with the `"cmse-nonsecure-entry"` ABI cannot contain generics in their type
+    0u128
+}
+
+extern "cmse-nonsecure-entry" fn return_impl_trait_nested(v: (impl Copy, i32)) -> (impl Copy, i32) {
+    //~^ ERROR functions with the `"cmse-nonsecure-entry"` ABI cannot contain generics in their type
+    //~| ERROR functions with the `"cmse-nonsecure-entry"` ABI cannot contain generics in their type
+    v
+}
diff --git a/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/generics.stderr b/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/generics.stderr
@@ -16,6 +16,34 @@ error[E0798]: functions with the `"cmse-nonsecure-entry"` ABI cannot contain gen
 LL | extern "cmse-nonsecure-entry" fn impl_trait(_: impl Copy, _: u32, _: u32, _: u32) -> u64 {
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+error[E0798]: functions with the `"cmse-nonsecure-entry"` ABI cannot contain generics in their type
+  --> $DIR/generics.rs:69:1
+   |
+LL | extern "cmse-nonsecure-entry" fn return_impl_trait(_: impl Copy) -> impl Copy {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error[E0798]: functions with the `"cmse-nonsecure-entry"` ABI cannot contain generics in their type
+  --> $DIR/generics.rs:69:1
+   |
+LL | extern "cmse-nonsecure-entry" fn return_impl_trait(_: impl Copy) -> impl Copy {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+
+error[E0798]: functions with the `"cmse-nonsecure-entry"` ABI cannot contain generics in their type
+  --> $DIR/generics.rs:75:1
+   |
+LL | extern "cmse-nonsecure-entry" fn return_impl_trait_nested(v: (impl Copy, i32)) -> (impl Copy, i32) {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error[E0798]: functions with the `"cmse-nonsecure-entry"` ABI cannot contain generics in their type
+  --> $DIR/generics.rs:75:1
+   |
+LL | extern "cmse-nonsecure-entry" fn return_impl_trait_nested(v: (impl Copy, i32)) -> (impl Copy, i32) {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+
 error[E0798]: functions with the `"cmse-nonsecure-entry"` ABI cannot contain generics in their type
   --> $DIR/generics.rs:14:5
    |
@@ -61,6 +89,6 @@ LL | extern "cmse-nonsecure-entry" fn wrapped_trait_object(x: WrapperTransparent
    = note: functions with the `"cmse-nonsecure-entry"` ABI must pass their result via the available return registers
    = note: the result must either be a (transparently wrapped) i64, u64 or f64, or be at most 4 bytes in size
 
-error: aborting due to 7 previous errors
+error: aborting due to 11 previous errors
 
 For more information about this error, try `rustc --explain E0798`.
