Use a much simpler hack: just consult the flag inside cg_llvm

Author: saethlin

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -367,7 +367,11 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
         (InlineAttr::None, _) if instance.def.requires_inline(cx.tcx) => InlineAttr::Hint,
         (inline, _) => inline,
     };
-    to_add.extend(inline_attr(cx, inline));
+    if cx.tcx.has_inline_always_override(instance) {
+        to_add.extend(inline_attr(cx, InlineAttr::Always));
+    } else {
+        to_add.extend(inline_attr(cx, inline));
+    }
 
     // The `uwtable` attribute according to LLVM is:
     //

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

@@ -51,7 +51,7 @@ fn linkage_by_name(tcx: TyCtxt<'_>, def_id: LocalDefId, name: &str) -> Linkage {
     }
 }
 
-fn codegen_fn_attrs_provider(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
+fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
     if cfg!(debug_assertions) {
         let def_kind = tcx.def_kind(did);
         assert!(
@@ -923,20 +923,6 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> {
     Some(AutoDiffAttrs { mode, width, ret_activity, input_activity: arg_activities })
 }
 
-fn codegen_fn_attrs_overridden(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
-    let mut attrs = tcx.codegen_fn_attrs_imp(did).clone();
-    let overrides = tcx.sess.opts.unstable_opts.inline_always_overrides.as_ref().unwrap();
-    if overrides.contains(&tcx.def_path_str(did)) {
-        attrs.inline = InlineAttr::Always;
-    }
-    attrs
-}
-
 pub(crate) fn provide(providers: &mut Providers) {
-    *providers = Providers {
-        codegen_fn_attrs_imp: codegen_fn_attrs_provider,
-        codegen_fn_attrs_overridden,
-        should_inherit_track_caller,
-        ..*providers
-    };
+    *providers = Providers { codegen_fn_attrs, should_inherit_track_caller, ..*providers };
 }

compiler/rustc_const_eval/src/interpret/intern.rs

@@ -131,7 +131,7 @@ fn intern_as_new_static<'tcx>(
     // These do not inherit the codegen attrs of the parent static allocation, since
     // it doesn't make sense for them to inherit their `#[no_mangle]` and `#[link_name = ..]`
     // and the like.
-    feed.codegen_fn_attrs_imp(CodegenFnAttrs::new());
+    feed.codegen_fn_attrs(CodegenFnAttrs::new());
 
     feed.eval_static_initializer(Ok(alloc));
     feed.generics_of(tcx.generics_of(static_id).clone());

compiler/rustc_hir_analysis/src/collect.rs

@@ -313,7 +313,7 @@ impl<'tcx> Visitor<'tcx> for CollectItemTypesVisitor<'tcx> {
     fn visit_expr(&mut self, expr: &'tcx hir::Expr<'tcx>) {
         if let hir::ExprKind::Closure(closure) = expr.kind {
             self.tcx.ensure_ok().generics_of(closure.def_id);
-            self.tcx.ensure_ok().codegen_fn_attrs_imp(closure.def_id);
+            self.tcx.ensure_ok().codegen_fn_attrs(closure.def_id);
             // We do not call `type_of` for closures here as that
             // depends on typecheck and would therefore hide
             // any further errors in case one typeck fails.
@@ -691,11 +691,11 @@ fn lower_item(tcx: TyCtxt<'_>, item_id: hir::ItemId) {
                 }
                 match item.kind {
                     hir::ForeignItemKind::Fn(..) => {
-                        tcx.ensure_ok().codegen_fn_attrs_imp(item.owner_id);
+                        tcx.ensure_ok().codegen_fn_attrs(item.owner_id);
                         tcx.ensure_ok().fn_sig(item.owner_id)
                     }
                     hir::ForeignItemKind::Static(..) => {
-                        tcx.ensure_ok().codegen_fn_attrs_imp(item.owner_id);
+                        tcx.ensure_ok().codegen_fn_attrs(item.owner_id);
                         let mut visitor = HirPlaceholderCollector::default();
                         visitor.visit_foreign_item(item);
                         placeholder_type_error(
@@ -782,7 +782,7 @@ fn lower_item(tcx: TyCtxt<'_>, item_id: hir::ItemId) {
             tcx.ensure_ok().type_of(def_id);
             tcx.ensure_ok().predicates_of(def_id);
             tcx.ensure_ok().fn_sig(def_id);
-            tcx.ensure_ok().codegen_fn_attrs_imp(def_id);
+            tcx.ensure_ok().codegen_fn_attrs(def_id);
         }
     }
 }
@@ -795,7 +795,7 @@ fn lower_trait_item(tcx: TyCtxt<'_>, trait_item_id: hir::TraitItemId) {
 
     match trait_item.kind {
         hir::TraitItemKind::Fn(..) => {
-            tcx.ensure_ok().codegen_fn_attrs_imp(def_id);
+            tcx.ensure_ok().codegen_fn_attrs(def_id);
             tcx.ensure_ok().type_of(def_id);
             tcx.ensure_ok().fn_sig(def_id);
         }
@@ -867,7 +867,7 @@ fn lower_impl_item(tcx: TyCtxt<'_>, impl_item_id: hir::ImplItemId) {
     let icx = ItemCtxt::new(tcx, def_id.def_id);
     match impl_item.kind {
         hir::ImplItemKind::Fn(..) => {
-            tcx.ensure_ok().codegen_fn_attrs_imp(def_id);
+            tcx.ensure_ok().codegen_fn_attrs(def_id);
             tcx.ensure_ok().fn_sig(def_id);
         }
         hir::ImplItemKind::Type(_) => {

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -251,7 +251,7 @@ provide! { tcx, def_id, other, cdata,
     type_alias_is_lazy => { table_direct }
     variances_of => { table }
     fn_sig => { table }
-    codegen_fn_attrs_imp => { table }
+    codegen_fn_attrs => { table }
     impl_trait_header => { table }
     const_param_default => { table }
     object_lifetime_default => { table }

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -1431,7 +1431,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 record!(self.tables.def_ident_span[def_id] <- ident_span);
             }
             if def_kind.has_codegen_attrs() {
-                record!(self.tables.codegen_fn_attrs_imp[def_id] <- self.tcx.codegen_fn_attrs(def_id));
+                record!(self.tables.codegen_fn_attrs[def_id] <- self.tcx.codegen_fn_attrs(def_id));
             }
             if should_encode_visibility(def_kind) {
                 let vis =

compiler/rustc_metadata/src/rmeta/mod.rs

@@ -431,7 +431,7 @@ define_tables! {
     type_of: Table<DefIndex, LazyValue<ty::EarlyBinder<'static, Ty<'static>>>>,
     variances_of: Table<DefIndex, LazyArray<ty::Variance>>,
     fn_sig: Table<DefIndex, LazyValue<ty::EarlyBinder<'static, ty::PolyFnSig<'static>>>>,
-    codegen_fn_attrs_imp: Table<DefIndex, LazyValue<CodegenFnAttrs>>,
+    codegen_fn_attrs: Table<DefIndex, LazyValue<CodegenFnAttrs>>,
     impl_trait_header: Table<DefIndex, LazyValue<ty::ImplTraitHeader<'static>>>,
     const_param_default: Table<DefIndex, LazyValue<ty::EarlyBinder<'static, rustc_middle::ty::Const<'static>>>>,
     object_lifetime_default: Table<DefIndex, LazyValue<ObjectLifetimeDefault>>,

compiler/rustc_middle/src/mir/mono.rs

@@ -209,6 +209,8 @@ impl<'tcx> MonoItem<'tcx> {
         if codegen_fn_attrs.inline.always() {
             return InstantiationMode::LocalCopy;
         }
+        // FIXME: Ideally we'd check has_inline_always_override here, but we can't because symbol names
+        // depend on instantiation mode so instantiation mode can't depend on symbol name.
 
         // #[inline(never)] functions in general are poor candidates for inlining and thus since
         // LocalCopy generally increases code size for the benefit of optimizations from inlining,

compiler/rustc_middle/src/query/mod.rs

@@ -1443,19 +1443,14 @@ rustc_queries! {
         separate_provide_extern
     }
 
-    query codegen_fn_attrs_imp(def_id: DefId) -> &'tcx CodegenFnAttrs {
+    query codegen_fn_attrs(def_id: DefId) -> &'tcx CodegenFnAttrs {
         desc { |tcx| "computing codegen attributes of `{}`", tcx.def_path_str(def_id) }
         arena_cache
         cache_on_disk_if { def_id.is_local() }
         separate_provide_extern
         feedable
     }
 
-    query codegen_fn_attrs_overridden(def_id: DefId) -> &'tcx CodegenFnAttrs {
-        desc { |tcx| "computing codegen attributes of `{}` (with overrides)", tcx.def_path_str(def_id) }
-        arena_cache
-    }
-
     query asm_target_features(def_id: DefId) -> &'tcx FxIndexSet<Symbol> {
         desc { |tcx| "computing target features for inline asm of `{}`", tcx.def_path_str(def_id) }
     }

compiler/rustc_middle/src/ty/context.rs

@@ -78,8 +78,8 @@ use crate::traits::solve::{
 use crate::ty::predicate::ExistentialPredicateStableCmpExt as _;
 use crate::ty::{
     self, AdtDef, AdtDefData, AdtKind, Binder, Clause, Clauses, Const, GenericArg, GenericArgs,
-    GenericArgsRef, GenericParamDefKind, List, ListWithCachedTypeInfo, ParamConst, ParamTy,
-    Pattern, PatternKind, PolyExistentialPredicate, PolyFnSig, Predicate, PredicateKind,
+    GenericArgsRef, GenericParamDefKind, Instance, List, ListWithCachedTypeInfo, ParamConst,
+    ParamTy, Pattern, PatternKind, PolyExistentialPredicate, PolyFnSig, Predicate, PredicateKind,
     PredicatePolarity, Region, RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind, TyVid,
     ValTree, ValTreeKind, Visibility,
 };
@@ -3396,13 +3396,12 @@ impl<'tcx> TyCtxt<'tcx> {
         self.get_diagnostic_attr(def_id, sym::do_not_recommend).is_some()
     }
 
-    pub fn codegen_fn_attrs(self, def_id: impl IntoQueryParam<DefId>) -> &'tcx CodegenFnAttrs {
-        let def_id = def_id.into_query_param();
-        if self.sess.opts.unstable_opts.inline_always_overrides.is_some() {
-            self.codegen_fn_attrs_overridden(def_id)
-        } else {
-            self.codegen_fn_attrs_imp(def_id)
-        }
+    pub fn has_inline_always_override(self, instance: Instance<'tcx>) -> bool {
+        let Some(overrides) = &self.sess.opts.unstable_opts.inline_always_overrides else {
+            return false;
+        };
+        let symbol_name = self.symbol_name(instance).name;
+        overrides.iter().any(|o| o == symbol_name)
     }
 }