Fix includes and fuse constructors of SpinLock.

PiperOrigin-RevId: 774835829
Change-Id: I0fa7cab1b98c1b7222de0acd71b7846df693f1e2

Author: Abseil Team

absl/base/internal/spinlock.cc

@@ -16,15 +16,18 @@
 
 #include <algorithm>
 #include <atomic>
+#include <cstdint>
 #include <limits>
 
 #include "absl/base/attributes.h"
+#include "absl/base/call_once.h"
 #include "absl/base/config.h"
 #include "absl/base/internal/atomic_hook.h"
 #include "absl/base/internal/cycleclock.h"
+#include "absl/base/internal/scheduling_mode.h"
 #include "absl/base/internal/spinlock_wait.h"
 #include "absl/base/internal/sysinfo.h" /* For NumCPUs() */
-#include "absl/base/call_once.h"
+#include "absl/base/internal/tsan_mutex_interface.h"
 
 // Description of lock-word:
 //  31..00: [............................3][2][1][0]
@@ -58,7 +61,7 @@ namespace absl {
 ABSL_NAMESPACE_BEGIN
 namespace base_internal {
 
-ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static base_internal::AtomicHook<void (*)(
+ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static AtomicHook<void (*)(
     const void *lock, int64_t wait_cycles)>
     submit_profile_data;
 
@@ -67,12 +70,6 @@ void RegisterSpinLockProfiler(void (*fn)(const void *contendedlock,
   submit_profile_data.Store(fn);
 }
 
-// Uncommon constructors.
-SpinLock::SpinLock(base_internal::SchedulingMode mode)
-    : lockword_(IsCooperative(mode) ? kSpinLockCooperative : 0) {
-  ABSL_TSAN_MUTEX_CREATE(this, __tsan_mutex_not_static);
-}
-
 // Monitor the lock to see if its value changes within some time period
 // (adaptive_spin_count loop iterations). The last value read from the lock
 // is returned from the method.
@@ -81,9 +78,8 @@ uint32_t SpinLock::SpinLoop() {
   // adaptive_spin_count here.
   ABSL_CONST_INIT static absl::once_flag init_adaptive_spin_count;
   ABSL_CONST_INIT static int adaptive_spin_count = 0;
-  base_internal::LowLevelCallOnce(&init_adaptive_spin_count, []() {
-    adaptive_spin_count = base_internal::NumCPUs() > 1 ? 1000 : 1;
-  });
+  LowLevelCallOnce(&init_adaptive_spin_count,
+                   []() { adaptive_spin_count = NumCPUs() > 1 ? 1000 : 1; });
 
   int c = adaptive_spin_count;
   uint32_t lock_value;
@@ -100,11 +96,11 @@ void SpinLock::SlowLock() {
     return;
   }
 
-  base_internal::SchedulingMode scheduling_mode;
+  SchedulingMode scheduling_mode;
   if ((lock_value & kSpinLockCooperative) != 0) {
-    scheduling_mode = base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL;
+    scheduling_mode = SCHEDULE_COOPERATIVE_AND_KERNEL;
   } else {
-    scheduling_mode = base_internal::SCHEDULE_KERNEL_ONLY;
+    scheduling_mode = SCHEDULE_KERNEL_ONLY;
   }
 
   // The lock was not obtained initially, so this thread needs to wait for
@@ -134,7 +130,7 @@ void SpinLock::SlowLock() {
         // new lock state will be the number of cycles this thread waited if
         // this thread obtains the lock.
         lock_value = TryLockInternal(lock_value, wait_cycles);
-        continue;   // Skip the delay at the end of the loop.
+        continue;  // Skip the delay at the end of the loop.
       } else if ((lock_value & kWaitTimeMask) == 0) {
         // The lock is still held, without a waiter being marked, but something
         // else about the lock word changed, causing our CAS to fail. For
@@ -150,8 +146,8 @@ void SpinLock::SlowLock() {
     // synchronization there to avoid false positives.
     ABSL_TSAN_MUTEX_PRE_DIVERT(this, 0);
     // Wait for an OS specific delay.
-    base_internal::SpinLockDelay(&lockword_, lock_value, ++lock_wait_call_count,
-                                 scheduling_mode);
+    SpinLockDelay(&lockword_, lock_value, ++lock_wait_call_count,
+                  scheduling_mode);
     ABSL_TSAN_MUTEX_POST_DIVERT(this, 0);
     // Spin again after returning from the wait routine to give this thread
     // some chance of obtaining the lock.
@@ -162,8 +158,8 @@ void SpinLock::SlowLock() {
 }
 
 void SpinLock::SlowUnlock(uint32_t lock_value) {
-  base_internal::SpinLockWake(&lockword_,
-                              false);  // wake waiter if necessary
+  SpinLockWake(&lockword_,
+               false);  // wake waiter if necessary
 
   // If our acquisition was contended, collect contentionz profile info.  We
   // reserve a unitary wait time to represent that a waiter exists without our

absl/base/internal/spinlock.h

@@ -19,7 +19,7 @@
 //   - for use by Abseil internal code that Mutex itself depends on
 //   - for async signal safety (see below)
 
-// SpinLock with a base_internal::SchedulingMode::SCHEDULE_KERNEL_ONLY is async
+// SpinLock with a SchedulingMode::SCHEDULE_KERNEL_ONLY is async
 // signal safe. If a spinlock is used within a signal handler, all code that
 // acquires the lock must ensure that the signal cannot arrive while they are
 // holding the lock. Typically, this is done by blocking the signal.
@@ -31,14 +31,16 @@
 
 #include <atomic>
 #include <cstdint>
+#include <type_traits>
 
 #include "absl/base/attributes.h"
+#include "absl/base/config.h"
 #include "absl/base/const_init.h"
-#include "absl/base/dynamic_annotations.h"
 #include "absl/base/internal/low_level_scheduling.h"
 #include "absl/base/internal/raw_logging.h"
 #include "absl/base/internal/scheduling_mode.h"
 #include "absl/base/internal/tsan_mutex_interface.h"
+#include "absl/base/macros.h"
 #include "absl/base/thread_annotations.h"
 
 namespace tcmalloc {
@@ -55,17 +57,31 @@ namespace base_internal {
 
 class ABSL_LOCKABLE ABSL_ATTRIBUTE_WARN_UNUSED SpinLock {
  public:
-  SpinLock() : lockword_(kSpinLockCooperative) {
-    ABSL_TSAN_MUTEX_CREATE(this, __tsan_mutex_not_static);
-  }
+  constexpr SpinLock() : lockword_(kSpinLockCooperative) { RegisterWithTsan(); }
 
   // Constructors that allow non-cooperative spinlocks to be created for use
   // inside thread schedulers.  Normal clients should not use these.
-  explicit SpinLock(base_internal::SchedulingMode mode);
+  constexpr explicit SpinLock(SchedulingMode mode)
+      : lockword_(IsCooperative(mode) ? kSpinLockCooperative : 0) {
+    RegisterWithTsan();
+  }
+
+#if ABSL_HAVE_ATTRIBUTE(enable_if) && !defined(_WIN32)
+  // Constructor to inline users of the default scheduling mode.
+  //
+  // This only needs to exists for inliner runs, but doesn't work correctly in
+  // clang+windows builds, likely due to mangling differences.
+  ABSL_DEPRECATE_AND_INLINE()
+  constexpr explicit SpinLock(SchedulingMode mode)
+      __attribute__((enable_if(mode == SCHEDULE_COOPERATIVE_AND_KERNEL,
+                               "Cooperative use default constructor")))
+      : SpinLock() {}
+#endif
 
   // Constructor for global SpinLock instances.  See absl/base/const_init.h.
-  constexpr SpinLock(absl::ConstInitType, base_internal::SchedulingMode mode)
-      : lockword_(IsCooperative(mode) ? kSpinLockCooperative : 0) {}
+  ABSL_DEPRECATE_AND_INLINE()
+  constexpr SpinLock(absl::ConstInitType, SchedulingMode mode)
+      : SpinLock(mode) {}
 
   // For global SpinLock instances prefer trivial destructor when possible.
   // Default but non-trivial destructor in some build configurations causes an
@@ -106,7 +122,7 @@ class ABSL_LOCKABLE ABSL_ATTRIBUTE_WARN_UNUSED SpinLock {
                                     std::memory_order_release);
 
     if ((lock_value & kSpinLockDisabledScheduling) != 0) {
-      base_internal::SchedulingGuard::EnableRescheduling(true);
+      SchedulingGuard::EnableRescheduling(true);
     }
     if ((lock_value & kWaitTimeMask) != 0) {
       // Collect contentionz profile info, and speed the wakeup of any waiter.
@@ -175,9 +191,16 @@ class ABSL_LOCKABLE ABSL_ATTRIBUTE_WARN_UNUSED SpinLock {
       ~(kSpinLockHeld | kSpinLockCooperative | kSpinLockDisabledScheduling);
 
   // Returns true if the provided scheduling mode is cooperative.
-  static constexpr bool IsCooperative(
-      base_internal::SchedulingMode scheduling_mode) {
-    return scheduling_mode == base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL;
+  static constexpr bool IsCooperative(SchedulingMode scheduling_mode) {
+    return scheduling_mode == SCHEDULE_COOPERATIVE_AND_KERNEL;
+  }
+
+  constexpr void RegisterWithTsan() {
+#if ABSL_HAVE_BUILTIN(__builtin_is_constant_evaluated)
+    if (!__builtin_is_constant_evaluated()) {
+      ABSL_TSAN_MUTEX_CREATE(this, __tsan_mutex_not_static);
+    }
+#endif
   }
 
   bool IsCooperative() const {
@@ -243,7 +266,7 @@ inline uint32_t SpinLock::TryLockInternal(uint32_t lock_value,
   if ((lock_value & kSpinLockCooperative) == 0) {
     // For non-cooperative locks we must make sure we mark ourselves as
     // non-reschedulable before we attempt to CompareAndSwap.
-    if (base_internal::SchedulingGuard::DisableRescheduling()) {
+    if (SchedulingGuard::DisableRescheduling()) {
       sched_disabled_bit = kSpinLockDisabledScheduling;
     }
   }
@@ -252,7 +275,7 @@ inline uint32_t SpinLock::TryLockInternal(uint32_t lock_value,
           lock_value,
           kSpinLockHeld | lock_value | wait_cycles | sched_disabled_bit,
           std::memory_order_acquire, std::memory_order_relaxed)) {
-    base_internal::SchedulingGuard::EnableRescheduling(sched_disabled_bit != 0);
+    SchedulingGuard::EnableRescheduling(sched_disabled_bit != 0);
   }
 
   return lock_value;

absl/base/internal/thread_identity_test.cc

@@ -31,7 +31,7 @@ namespace base_internal {
 namespace {
 
 ABSL_CONST_INIT static absl::base_internal::SpinLock map_lock(
-    absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY);
+    base_internal::SCHEDULE_KERNEL_ONLY);
 ABSL_CONST_INIT static int num_identities_reused ABSL_GUARDED_BY(map_lock);
 
 static const void* const kCheckNoIdentity = reinterpret_cast<void*>(1);

absl/base/spinlock_test_common.cc

@@ -60,24 +60,41 @@ namespace {
 static constexpr size_t kArrayLength = 10;
 static uint32_t values[kArrayLength];
 
-ABSL_CONST_INIT static SpinLock static_cooperative_spinlock(
-    absl::kConstInit, base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL);
+ABSL_CONST_INIT static SpinLock static_cooperative_spinlock;
 ABSL_CONST_INIT static SpinLock static_noncooperative_spinlock(
-    absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY);
+    base_internal::SCHEDULE_KERNEL_ONLY);
 
 // Simple integer hash function based on the public domain lookup2 hash.
 // http://burtleburtle.net/bob/c/lookup2.c
 static uint32_t Hash32(uint32_t a, uint32_t c) {
   uint32_t b = 0x9e3779b9UL;  // The golden ratio; an arbitrary value.
-  a -= b; a -= c; a ^= (c >> 13);
-  b -= c; b -= a; b ^= (a << 8);
-  c -= a; c -= b; c ^= (b >> 13);
-  a -= b; a -= c; a ^= (c >> 12);
-  b -= c; b -= a; b ^= (a << 16);
-  c -= a; c -= b; c ^= (b >> 5);
-  a -= b; a -= c; a ^= (c >> 3);
-  b -= c; b -= a; b ^= (a << 10);
-  c -= a; c -= b; c ^= (b >> 15);
+  a -= b;
+  a -= c;
+  a ^= (c >> 13);
+  b -= c;
+  b -= a;
+  b ^= (a << 8);
+  c -= a;
+  c -= b;
+  c ^= (b >> 13);
+  a -= b;
+  a -= c;
+  a ^= (c >> 12);
+  b -= c;
+  b -= a;
+  b ^= (a << 16);
+  c -= a;
+  c -= b;
+  c ^= (b >> 5);
+  a -= b;
+  a -= c;
+  a ^= (c >> 3);
+  b -= c;
+  b -= a;
+  b ^= (a << 10);
+  c -= a;
+  c -= b;
+  c ^= (b >> 15);
   return c;
 }
 
@@ -134,7 +151,7 @@ TEST(SpinLock, WaitCyclesEncoding) {
   // We should be able to encode up to (1^kMaxCycleBits - 1) without clamping
   // but the lower kProfileTimestampShift will be dropped.
   const int kMaxCyclesShift =
-    32 - kLockwordReservedShift + kProfileTimestampShift;
+      32 - kLockwordReservedShift + kProfileTimestampShift;
   const int64_t kMaxCycles = (int64_t{1} << kMaxCyclesShift) - 1;
 
   // These bits should be zero after encoding.
@@ -171,22 +188,22 @@ TEST(SpinLock, WaitCyclesEncoding) {
             SpinLockTest::DecodeWaitCycles(~kLockwordReservedMask));
 
   // Check that we cannot produce kSpinLockSleeper during encoding.
-  int64_t sleeper_cycles =
-      kSpinLockSleeper << (kProfileTimestampShift - kLockwordReservedShift);
+  int64_t sleeper_cycles = kSpinLockSleeper
+                           << (kProfileTimestampShift - kLockwordReservedShift);
   uint32_t sleeper_value =
       SpinLockTest::EncodeWaitCycles(start_time, start_time + sleeper_cycles);
   EXPECT_NE(sleeper_value, kSpinLockSleeper);
 
   // Test clamping
   uint32_t max_value =
-    SpinLockTest::EncodeWaitCycles(start_time, start_time + kMaxCycles);
+      SpinLockTest::EncodeWaitCycles(start_time, start_time + kMaxCycles);
   int64_t max_value_decoded = SpinLockTest::DecodeWaitCycles(max_value);
   int64_t expected_max_value_decoded = kMaxCycles & ~kProfileTimestampMask;
   EXPECT_EQ(expected_max_value_decoded, max_value_decoded);
 
   const int64_t step = (1 << kProfileTimestampShift);
-  uint32_t after_max_value =
-    SpinLockTest::EncodeWaitCycles(start_time, start_time + kMaxCycles + step);
+  uint32_t after_max_value = SpinLockTest::EncodeWaitCycles(
+      start_time, start_time + kMaxCycles + step);
   int64_t after_max_value_decoded =
       SpinLockTest::DecodeWaitCycles(after_max_value);
   EXPECT_EQ(expected_max_value_decoded, after_max_value_decoded);

absl/debugging/symbolize_elf.inc

@@ -171,18 +171,18 @@ struct FileMappingHint {
 // is being modified (is busy), we skip all decorators, and possibly
 // loose some info. Sorry, that's the best we could do.
 ABSL_CONST_INIT absl::base_internal::SpinLock g_decorators_mu(
-    absl::kConstInit, absl::base_internal::SCHEDULE_KERNEL_ONLY);
+    absl::base_internal::SCHEDULE_KERNEL_ONLY);
 
 const int kMaxFileMappingHints = 8;
 int g_num_file_mapping_hints;
 FileMappingHint g_file_mapping_hints[kMaxFileMappingHints];
 // Protects g_file_mapping_hints.
 ABSL_CONST_INIT absl::base_internal::SpinLock g_file_mapping_mu(
-    absl::kConstInit, absl::base_internal::SCHEDULE_KERNEL_ONLY);
+    absl::base_internal::SCHEDULE_KERNEL_ONLY);
 
 // Async-signal-safe function to zero a buffer.
 // memset() is not guaranteed to be async-signal-safe.
-static void SafeMemZero(void* p, size_t size) {
+static void SafeMemZero(void *p, size_t size) {
   unsigned char *c = static_cast<unsigned char *>(p);
   while (size--) {
     *c++ = 0;
@@ -1469,14 +1469,15 @@ static bool MaybeInitializeObjFile(ObjFile *obj) {
       constexpr int interesting = PF_X | PF_R;
 #endif
 
-      if (phdr.p_type != PT_LOAD
-          || (phdr.p_flags & interesting) != interesting) {
+      if (phdr.p_type != PT_LOAD ||
+          (phdr.p_flags & interesting) != interesting) {
         // Not a LOAD segment, not executable code, and not a function
         // descriptor.
         continue;
       }
       if (num_interesting_load_segments < obj->phdr.size()) {
-        memcpy(&obj->phdr[num_interesting_load_segments++], &phdr, sizeof(phdr));
+        memcpy(&obj->phdr[num_interesting_load_segments++], &phdr,
+               sizeof(phdr));
       } else {
         ABSL_RAW_LOG(
             WARNING, "%s: too many interesting LOAD segments: %zu >= %zu",
@@ -1525,7 +1526,8 @@ const char *Symbolizer::GetUncachedSymbol(const void *pc) {
             ABSL_RAW_CHECK(p.p_type == PT_NULL, "unexpected p_type");
             break;
           }
-          if (pc < reinterpret_cast<void *>(start_addr + p.p_vaddr + p.p_memsz)) {
+          if (pc <
+              reinterpret_cast<void *>(start_addr + p.p_vaddr + p.p_memsz)) {
             phdr = &p;
             break;
           }
@@ -1671,8 +1673,8 @@ int InstallSymbolDecorator(SymbolDecorator decorator, void *arg) {
   return ret;
 }
 
-bool RegisterFileMappingHint(const void *start, const void *end, uint64_t offset,
-                             const char *filename) {
+bool RegisterFileMappingHint(const void *start, const void *end,
+                             uint64_t offset, const char *filename) {
   SAFE_ASSERT(start <= end);
   SAFE_ASSERT(filename != nullptr);
 
@@ -1765,7 +1767,8 @@ ABSL_NAMESPACE_END
 }  // namespace absl
 
 extern "C" bool AbslInternalGetFileMappingHint(const void **start,
-                                               const void **end, uint64_t *offset,
+                                               const void **end,
+                                               uint64_t *offset,
                                                const char **filename) {
   return absl::debugging_internal::GetFileMappingHint(start, end, offset,
                                                       filename);

absl/log/internal/vlog_config.cc

@@ -90,7 +90,7 @@ struct VModuleInfo final {
 // To avoid problems with the heap checker which calls into `VLOG`, `mutex` must
 // be a `SpinLock` that prevents fiber scheduling instead of a `Mutex`.
 ABSL_CONST_INIT absl::base_internal::SpinLock mutex(
-    absl::kConstInit, absl::base_internal::SCHEDULE_KERNEL_ONLY);
+    absl::base_internal::SCHEDULE_KERNEL_ONLY);
 
 // `GetUpdateSitesMutex()` serializes updates to all of the sites (i.e. those in
 // `site_list_head`) themselves.