Distinguish lowered-to-MIR intrinsics from handled-by-backend intrinsics

compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

@@ -1278,11 +1278,12 @@ fn codegen_regular_intrinsic_call<'tcx>(
         // by converting the `InstanceKind::Intrinsic` to an `InstanceKind::Item`.
         _ => {
             let intrinsic = fx.tcx.intrinsic(instance.def_id()).unwrap();
-            if intrinsic.must_be_overridden {
+            if !intrinsic.has_fallback() {
                 span_bug!(
                     source_info.span,
-                    "intrinsic {} must be overridden by codegen_cranelift, but isn't",
+                    "intrinsic {} ({:?}) must be overridden by codegen_cranelift, but isn't",
                     intrinsic.name,
+                    intrinsic.kind,
                 );
             }
             return Err(Instance::new(instance.def_id(), instance.args));

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -952,11 +952,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         };
                     }
                     Err(instance) => {
-                        if intrinsic.must_be_overridden {
+                        if !intrinsic.has_fallback() {
                             span_bug!(
                                 span,
-                                "intrinsic {} must be overridden by codegen backend, but isn't",
+                                "intrinsic {} ({:?}) got to the codegen backend, but wasn't overridden",
                                 intrinsic.name,
+                                intrinsic.kind,
                             );
                         }
                         Some(instance)

compiler/rustc_const_eval/src/check_consts/check.rs

@@ -766,7 +766,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                     // We also ask is_safe_to_expose_on_stable_const_fn; this determines whether the intrinsic
                     // fallback body is safe to expose on stable.
                     let is_const_stable = intrinsic.const_stable
-                        || (!intrinsic.must_be_overridden
+                        || (intrinsic.has_fallback()
                             && is_safe_to_expose_on_stable_const_fn(tcx, callee));
                     match tcx.lookup_const_stability(callee) {
                         None => {

compiler/rustc_const_eval/src/const_eval/machine.rs

@@ -466,7 +466,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
             sym::is_val_statically_known => ecx.write_scalar(Scalar::from_bool(false), dest)?,
             _ => {
                 // We haven't handled the intrinsic, let's see if we can use a fallback body.
-                if ecx.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden {
+                if !ecx.tcx.intrinsic(instance.def_id()).unwrap().has_fallback() {
                     throw_unsup_format!(
                         "intrinsic `{intrinsic_name}` is not supported at compile-time"
                     );

compiler/rustc_const_eval/src/interpret/call.rs

@@ -540,7 +540,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                     target,
                     unwind,
                 )? {
-                    assert!(!self.tcx.intrinsic(fallback.def_id()).unwrap().must_be_overridden);
+                    assert!(self.tcx.intrinsic(fallback.def_id()).unwrap().has_fallback());
                     assert_matches!(fallback.def, ty::InstanceKind::Item(_));
                     return self.init_fn_call(
                         FnVal::Instance(fallback),

compiler/rustc_feature/src/builtin_attrs.rs

@@ -1010,6 +1010,10 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         rustc_intrinsic_must_be_overridden, Normal, template!(Word), ErrorFollowing, EncodeCrossCrate::Yes, intrinsics,
         "the `#[rustc_intrinsic_must_be_overridden]` attribute is used to declare intrinsics without real bodies",
     ),
+    gated!(
+        rustc_intrinsic_lowers_to_mir, Normal, template!(Word), ErrorFollowing, EncodeCrossCrate::Yes, intrinsics,
+        "the `#[rustc_intrinsic_lowers_to_mir]` attribute is used to declare intrinsics translated by `LowerIntrinsics`",
+    ),
     rustc_attr!(
         rustc_no_mir_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes,
         "#[rustc_no_mir_inline] prevents the MIR inliner from inlining a function while not affecting codegen"

compiler/rustc_middle/src/ty/intrinsic.rs

@@ -4,15 +4,39 @@ use rustc_span::def_id::DefId;
 
 use super::TyCtxt;
 
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Decodable, Encodable, HashStable)]
+pub enum IntrinsicKind {
+    /// The intrinsic has no meaningful body and all backends need to shim all calls to it.
+    MustBeOverridden,
+    /// The intrinsic lowers to MIR, so does not need to be considered by backends.
+    LowersToMir,
+    /// The intrinsic has a meaningful body usable by backends that don't need something special.
+    HasFallback,
+}
+
 #[derive(Copy, Clone, Debug, Decodable, Encodable, HashStable)]
 pub struct IntrinsicDef {
     pub name: Symbol,
-    /// Whether the intrinsic has no meaningful body and all backends need to shim all calls to it.
-    pub must_be_overridden: bool,
+    /// Describes how the intrinsic is expected to be handled, based on its definition.
+    pub kind: IntrinsicKind,
     /// Whether the intrinsic can be invoked from stable const fn
     pub const_stable: bool,
 }
 
+impl IntrinsicDef {
+    pub fn must_be_overridden(self) -> bool {
+        self.kind == IntrinsicKind::MustBeOverridden
+    }
+
+    pub fn has_fallback(self) -> bool {
+        self.kind == IntrinsicKind::HasFallback
+    }
+
+    pub fn lowers_to_mir(self) -> bool {
+        self.kind == IntrinsicKind::LowersToMir
+    }
+}
+
 impl TyCtxt<'_> {
     pub fn is_intrinsic(self, def_id: DefId, name: Symbol) -> bool {
         let Some(i) = self.intrinsic(def_id) else { return false };

compiler/rustc_middle/src/ty/mod.rs

@@ -23,7 +23,7 @@ pub use adt::*;
 pub use assoc::*;
 pub use generic_args::{GenericArgKind, TermKind, *};
 pub use generics::*;
-pub use intrinsic::IntrinsicDef;
+pub use intrinsic::{IntrinsicDef, IntrinsicKind};
 use rustc_abi::{Align, FieldIdx, Integer, IntegerType, ReprFlags, ReprOptions, VariantIdx};
 use rustc_ast::expand::StrippedCfgItem;
 use rustc_ast::node_id::NodeMap;

compiler/rustc_middle/src/ty/util.rs

@@ -1760,11 +1760,20 @@ pub fn intrinsic_raw(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<ty::Intrinsi
         && (matches!(tcx.fn_sig(def_id).skip_binder().abi(), ExternAbi::RustIntrinsic)
             || tcx.has_attr(def_id, sym::rustc_intrinsic))
     {
-        Some(ty::IntrinsicDef {
-            name: tcx.item_name(def_id.into()),
-            must_be_overridden: tcx.has_attr(def_id, sym::rustc_intrinsic_must_be_overridden),
-            const_stable: tcx.has_attr(def_id, sym::rustc_intrinsic_const_stable_indirect),
-        })
+        let name = tcx.item_name(def_id.into());
+        let must_be_overridden = tcx.has_attr(def_id, sym::rustc_intrinsic_must_be_overridden);
+        let lowers_to_mir = tcx.has_attr(def_id, sym::rustc_intrinsic_lowers_to_mir);
+        let const_stable = tcx.has_attr(def_id, sym::rustc_intrinsic_const_stable_indirect);
+        let kind = match (must_be_overridden, lowers_to_mir) {
+            (false, false) => ty::IntrinsicKind::HasFallback,
+            (true, false) => ty::IntrinsicKind::MustBeOverridden,
+            (false, true) => ty::IntrinsicKind::LowersToMir,
+            (true, true) => bug!(
+                "intrinsic {name} should be marked with at most one of \
+                rustc_intrinsic_must_be_overridden and rustc_intrinsic_lowers_to_mir",
+            ),
+        };
+        Some(ty::IntrinsicDef { name, kind, const_stable })
     } else {
         None
     }

compiler/rustc_mir_build/src/build/mod.rs

@@ -997,7 +997,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         if let Some(source_scope) = scope {
             self.source_scope = source_scope;
         }
-        if self.tcx.intrinsic(self.def_id).is_some_and(|i| i.must_be_overridden) {
+        if self.tcx.intrinsic(self.def_id).is_some_and(|i| !i.has_fallback()) {
             let source_info = self.source_info(rustc_span::DUMMY_SP);
             self.cfg.terminate(block, source_info, TerminatorKind::Unreachable);
             self.cfg.start_new_block().unit()

compiler/rustc_mir_transform/src/lower_intrinsics.rs

@@ -320,7 +320,16 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
                         });
                         terminator.kind = TerminatorKind::Goto { target };
                     }
-                    _ => {}
+                    _ => {
+                        if intrinsic.lowers_to_mir() {
+                            span_bug!(
+                                terminator.source_info.span,
+                                "Intrinsic {} was marked as #[rustc_intrinsic_lowers_to_mir] \
+                                 but wasn't lowered by `LowerIntrinsics` pass",
+                                intrinsic.name,
+                            );
+                        }
+                    }
                 }
             }
         }

compiler/rustc_smir/src/rustc_smir/mod.rs

@@ -73,7 +73,7 @@ impl<'tcx> Tables<'tcx> {
     /// In StableMIR, we handle this case as if the body is not available.
     pub(crate) fn item_has_body(&self, def_id: DefId) -> bool {
         let must_override = if let Some(intrinsic) = self.tcx.intrinsic(def_id) {
-            intrinsic.must_be_overridden
+            intrinsic.must_be_overridden()
         } else {
             false
         };

compiler/rustc_span/src/symbol.rs

@@ -1702,6 +1702,7 @@ symbols! {
         rustc_insignificant_dtor,
         rustc_intrinsic,
         rustc_intrinsic_const_stable_indirect,
+        rustc_intrinsic_lowers_to_mir,
         rustc_intrinsic_must_be_overridden,
         rustc_layout,
         rustc_layout_scalar_valid_range_end,