Auto merge of #143511 - orlp:tls-cold-init, r=<try>

Improve TLS codegen by marking the panic/init path as cold

This is an extension of the performance improvements seen from <#141685>. I noticed that the non-`const` TLS still didn't have the `#[cold]` attribute for the uninit/panic path, and I also realized that neither implementation should have the initialization or panic path inlined, ever.

These paths are taken either only once per thread (`init`) or never (`panic`, in a well-behaving Rust program), thus they don't deserve to litter the code generated each time you access a thread-local variable. So in addition to `#[cold]` I added the more aggressive `#[inline(never)]` to both cold paths as well.

Author: rust-bors[bot]

library/std/src/sys/thread_local/native/eager.rs

@@ -4,7 +4,7 @@ use crate::sys::thread_local::{abort_on_dtor_unwind, destructors};
 
 #[derive(Clone, Copy)]
 enum State {
-    Initial,
+    Uninitialized,
     Alive,
     Destroyed,
 }
@@ -17,7 +17,7 @@ pub struct Storage<T> {
 
 impl<T> Storage<T> {
     pub const fn new(val: T) -> Storage<T> {
-        Storage { state: Cell::new(State::Initial), val: UnsafeCell::new(val) }
+        Storage { state: Cell::new(State::Uninitialized), val: UnsafeCell::new(val) }
     }
 
     /// Gets a pointer to the TLS value. If the TLS variable has been destroyed,
@@ -28,18 +28,25 @@ impl<T> Storage<T> {
     ///
     /// # Safety
     /// The `self` reference must remain valid until the TLS destructor is run.
-    #[inline]
+    #[inline(always)]
     pub unsafe fn get(&self) -> *const T {
-        match self.state.get() {
-            State::Alive => self.val.get(),
-            State::Destroyed => ptr::null(),
-            State::Initial => unsafe { self.initialize() },
+        if let State::Alive = self.state.get() {
+            self.val.get()
+        } else {
+            unsafe { self.get_or_init_slow() }
         }
     }
 
     #[cold]
-    unsafe fn initialize(&self) -> *const T {
-        // Register the destructor
+    #[inline(never)]
+    unsafe fn get_or_init_slow(&self) -> *const T {
+        match self.state.get() {
+            State::Uninitialized => {}
+            State::Alive => return self.val.get(),
+            State::Destroyed => return ptr::null(),
+        }
+
+        // Register the destructor.
 
         // SAFETY:
         // The caller guarantees that `self` will be valid until thread destruction.

library/std/src/sys/thread_local/native/lazy.rs

@@ -52,7 +52,7 @@ where
     ///
     /// # Safety
     /// The `self` reference must remain valid until the TLS destructor is run.
-    #[inline]
+    #[inline(always)]
     pub unsafe fn get_or_init(&self, i: Option<&mut Option<T>>, f: impl FnOnce() -> T) -> *const T {
         if let State::Alive = self.state.get() {
             self.value.get().cast()
@@ -64,6 +64,7 @@ where
     /// # Safety
     /// The `self` reference must remain valid until the TLS destructor is run.
     #[cold]
+    #[inline(never)]
     unsafe fn get_or_init_slow(
         &self,
         i: Option<&mut Option<T>>,