mempool: switch from thread local storage to cpu local storage

ceph#53130 (comment)

Signed-off-by: Radosław Zarzyński <[email protected]>

Author: rzarzynski

src/common/mempool.cc

@@ -15,9 +15,11 @@
 #include "include/mempool.h"
 #include "include/demangle.h"
 
+#ifndef _GNU_SOURCE
 // Thread local variables should save index, not &shard[index],
 // because shard[] is defined in the class
 static thread_local size_t thread_shard_index = mempool::num_shards;
+#endif
 
 // default to debug_mode off
 bool mempool::debug_mode = false;
@@ -95,9 +97,21 @@ size_t mempool::pool_t::allocated_items() const
 
 void mempool::pool_t::adjust_count(ssize_t items, ssize_t bytes)
 {
-  thread_shard_index = (thread_shard_index == num_shards) ? pick_a_shard_int() : thread_shard_index;
-  shard[thread_shard_index].items += items;
-  shard[thread_shard_index].bytes += bytes;
+#ifndef _GNU_SOURCE
+  // fallback for lack of sched_getcpu()
+  const size_t shard_index = []() {
+    if (thread_shard_index == num_shards) {
+      thread_shard_index = pick_a_shard_int();
+    }
+    return thread_shard_index;
+  }();
+#else
+  // the expected path: we alway pick the shard for a cpu core
+  // a thread is executing on.
+  const size_t shard_index = pick_a_shard_int();
+#endif
+  shard[shard_index].items += items;
+  shard[shard_index].bytes += bytes;
 }
 
 void mempool::pool_t::get_stats(

src/include/mempool.h

@@ -26,6 +26,10 @@
 #include <boost/container/flat_set.hpp>
 #include <boost/container/flat_map.hpp>
 
+#ifdef _GNU_SOURCE
+#  include <sched.h>
+#endif
+
 #include "common/Formatter.h"
 #include "common/ceph_atomic.h"
 #include "include/ceph_assert.h"
@@ -265,11 +269,21 @@ class pool_t {
   void adjust_count(ssize_t items, ssize_t bytes);
 
   static size_t pick_a_shard_int() {
+#ifndef _GNU_SOURCE
     // Dirt cheap, see:
     //   https://fossies.org/dox/glibc-2.32/pthread__self_8c_source.html
     size_t me = (size_t)pthread_self();
     size_t i = (me >> CEPH_PAGE_SHIFT) & ((1 << num_shard_bits) - 1);
     return i;
+#else
+    // a thread local storage is actually just an approximation;
+    // what we truly want is a _cpu local storage_.
+    //
+    // on the architectures we care about sched_getcpu() is
+    // a syscall-handled-in-userspace (vdso!). it grabs the cpu
+    // id kernel exposes to a task on context switch.
+    return sched_getcpu() & ((1 << num_shard_bits) - 1);
+#endif
   }
 
   shard_t* pick_a_shard() {