Merge pull request #46606 from Dr15Jones/fixModuleAllocMonitor

Remove use of thread_local from ModuleAllocMonitor

Author: cmsbuild

PerfTools/AllocMonitor/README.md

@@ -29,6 +29,9 @@ The monitor is owned by the registry and should not be deleted by any other code
 of the monitor, one can call `cms::perftools::AllocMonitorRegistry::deregisterMonitor` to have the monitor removed from
 the callback list and be deleted (again, without the deallocation causing any callbacks).
 
+NOTE: Experience has shown that using thread_local within a call to `allocCalled` or `deallocCalled` can lead to unexpected behavior. Therefore if per thread information must be gathered it is recommended to make a system that uses thread ids.
+An example of such code can be found in the implementation of ModuleAllocMonitor.
+
 ## General usage
 
 To use the facility, one needs to use LD_PRELOAD to load in the memory proxies before the application runs, e.g.
@@ -99,3 +102,16 @@ The output file contains the following information on each line
 - Number of calls made to deallocation functions
 
 This service is multi-thread safe. Note that when run multi-threaded the maximum reported value will vary from job to job.
+
+### ModuleAllocMonitor
+This service registers a monitor when the service is created (after python parsing is finished but before any modules
+have been loaded into cmsRun) and writes module related information to the specified file. The file name, an optional
+list of module names, and  an optional number of initial events to skip are specified by setting parameters of the
+service in the configuration. The parameters are
+- filename: name of file to which to write reports
+- moduleNames: list of modules which should have their information added to the file. An empty list specifies all modules should be included.
+- nEventsToSkip: the number of initial events that must be processed before reporting happens.
+
+The beginning of the file contains a description of the structure and contents of the file.
+
+This service is multi-thread safe.

PerfTools/AllocMonitor/plugins/ModuleAllocMonitor.cc

@@ -23,10 +23,115 @@
 #include "FWCore/ServiceRegistry/interface/ModuleCallingContext.h"
 #include "DataFormats/Provenance/interface/ModuleDescription.h"
 
+#if defined(ALLOC_USE_PTHREADS)
+#include <pthread.h>
+#else
+#include <unistd.h>
+#include <sys/syscall.h>
+#endif
+
 #include "moduleAlloc_setupFile.h"
 #include "ThreadAllocInfo.h"
 
 namespace {
+  inline auto thread_id() {
+#if defined(ALLOC_USE_PTHREADS)
+    /*NOTE: if use pthread_self, the values returned by linux had
+     lots of hash collisions when using a simple % hash. Worked
+     better if first divided value by 0x700 and then did %. 
+     [test done on el8]
+ */
+    return pthread_self();
+#else
+    return syscall(SYS_gettid);
+#endif
+  }
+
+  struct ThreadTracker {
+    static constexpr unsigned int kHashedEntries = 128;
+    static constexpr unsigned int kExtraEntries = 128;
+    static constexpr unsigned int kTotalEntries = kHashedEntries + kExtraEntries;
+    using entry_type = decltype(thread_id());
+    static constexpr entry_type kUnusedEntry = ~entry_type(0);
+    std::array<std::atomic<entry_type>, kHashedEntries> hashed_threads_;
+    std::array<std::atomic<entry_type>, kExtraEntries> extra_threads_;
+
+    ThreadTracker() {
+      //put a value which will not match the % used when looking up the entry
+      entry_type entry = 0;
+      for (auto& v : hashed_threads_) {
+        v = ++entry;
+      }
+      //assume kUsedEntry is not a valid thread-id
+      for (auto& v : extra_threads_) {
+        v = kUnusedEntry;
+      }
+    }
+
+    std::size_t thread_index() {
+      auto id = thread_id();
+      auto index = thread_index_guess(id);
+      auto used_id = hashed_threads_[index].load();
+
+      if (id == used_id) {
+        return index;
+      }
+      //try to be first thread to grab the index
+      auto expected = entry_type(index + 1);
+      if (used_id == expected) {
+        if (hashed_threads_[index].compare_exchange_strong(expected, id)) {
+          return index;
+        } else {
+          //another thread just beat us so have to go to non-hash storage
+          return find_new_index(id);
+        }
+      }
+      //search in non-hash storage
+      return find_index(id);
+    }
+
+  private:
+    std::size_t thread_index_guess(entry_type id) const {
+#if defined(ALLOC_USE_PTHREADS)
+      return (id / 0x700) % kHashedEntries;
+#else
+      return id % kHashedEntries;
+#endif
+    }
+
+    std::size_t find_new_index(entry_type id) {
+      std::size_t index = 0;
+      for (auto& v : extra_threads_) {
+        entry_type expected = kUnusedEntry;
+        if (v == expected) {
+          if (v.compare_exchange_strong(expected, id)) {
+            return index + kHashedEntries;
+          }
+        }
+        ++index;
+      }
+      //failed to find an open entry
+      abort();
+      return 0;
+    }
+
+    std::size_t find_index(entry_type id) {
+      std::size_t index = 0;
+      for (auto const& v : extra_threads_) {
+        if (v == id) {
+          return index + kHashedEntries;
+        }
+        ++index;
+      }
+      return find_new_index(id);
+    }
+  };
+
+  static ThreadTracker& getTracker() {
+    static ThreadTracker s_tracker;
+    return s_tracker;
+  }
+
   using namespace edm::service::moduleAlloc;
   class MonitorAdaptor : public cms::perftools::AllocMonitorBase {
   public:
@@ -43,8 +148,8 @@ namespace {
 
   private:
     static ThreadAllocInfo& threadAllocInfo() {
-      thread_local ThreadAllocInfo s_info;
-      return s_info;
+      static ThreadAllocInfo s_info[ThreadTracker::kTotalEntries];
+      return s_info[getTracker().thread_index()];
     }
     void allocCalled(size_t iRequested, size_t iActual, void const*) final {
       auto& allocInfo = threadAllocInfo();