Handle multiple threads reading RefCores

Deal with the case where the call to TTree::GetEntry uses multiple threads to de-serialize data from an Event.

Author: Dr15Jones

DataFormats/Common/interface/EDProductGetter.h

@@ -97,6 +97,8 @@ namespace edm {
 
     ///These can only be used internally by the framework
     static EDProductGetter const* switchProductGetter(EDProductGetter const*);
+    static void setMultiThreadProductGetter(EDProductGetter const*);
+    static void unsetMultiThreadProductGetter();
     static void assignEDProductGetter(EDProductGetter const*&);
 
   private:

DataFormats/Common/interface/RefCoreStreamer.h

@@ -43,5 +43,15 @@ namespace edm {
     static void unsetRefCoreStreamer();
     static EDProductGetter const* setRefCoreStreamer(EDProductGetter const* ep);
   };
+  class MultiThreadRefCoreStreamerGuard {
+    public:
+      MultiThreadRefCoreStreamerGuard(EDProductGetter const* ep) { setRefCoreStreamer(ep); }
+      ~MultiThreadRefCoreStreamerGuard() { unsetRefCoreStreamer(); }
+  
+    private:
+      static void setRefCoreStreamer(EDProductGetter const* ep);
+      static void unsetRefCoreStreamer();
+    };
+  
 }  // namespace edm
 #endif

DataFormats/Common/src/EDProductGetter.cc

@@ -11,11 +11,13 @@
 //
 
 // system include files
+#include <atomic>
 
 // user include files
 #include "DataFormats/Common/interface/EDProductGetter.h"
 #include "DataFormats/Provenance/interface/ProductID.h"
 #include "FWCore/Utilities/interface/EDMException.h"
+#include "FWCore/Utilities/interface/Likely.h"
 
 namespace edm {
   //
@@ -75,16 +77,32 @@ namespace edm {
   }
 
   thread_local EDProductGetter const* s_productGetter = nullptr;
+  static std::atomic<EDProductGetter const*> s_multiThreadProductGetter{nullptr};
   EDProductGetter const* EDProductGetter::switchProductGetter(EDProductGetter const* iNew) {
     //std::cout <<"switch from "<<s_productGetter<<" to "<<iNew<<std::endl;
     EDProductGetter const* old = s_productGetter;
     s_productGetter = iNew;
     return old;
   }
+
+  void EDProductGetter::setMultiThreadProductGetter(EDProductGetter const* prodGetter) {
+    EDProductGetter const* expected = nullptr;
+    while( not s_multiThreadProductGetter.compare_exchange_strong(expected, prodGetter, std::memory_order_acq_rel)) {
+      expected = nullptr;
+    };
+  }
+
+  void EDProductGetter::unsetMultiThreadProductGetter() {
+    s_multiThreadProductGetter.store(nullptr, std::memory_order_release);
+  }
+
   void EDProductGetter::assignEDProductGetter(EDProductGetter const*& iGetter) {
     //std::cout <<"assign "<<s_productGetter<<std::endl;
-
-    iGetter = s_productGetter;
+    if LIKELY (s_productGetter != nullptr) {
+      iGetter = s_productGetter;
+      return;
+    }
+    iGetter = s_multiThreadProductGetter.load(std::memory_order_acquire);
   }
 
 }  // namespace edm

DataFormats/Common/src/RefCoreStreamer.cc

@@ -89,4 +89,13 @@ namespace edm {
     }
     return returnValue;
   }
+
+  void MultiThreadRefCoreStreamerGuard::setRefCoreStreamer(EDProductGetter const* ep) {
+    edm::EDProductGetter::setMultiThreadProductGetter(ep);
+  }
+
+  void MultiThreadRefCoreStreamerGuard::unsetRefCoreStreamer() {
+    edm::EDProductGetter::unsetMultiThreadProductGetter();
+  }
+
 }  // namespace edm

IOPool/Input/src/RootPromptReadDelayedReader.cc

@@ -99,13 +99,10 @@ namespace edm {
       branchInfo->productBranch_->SetAddress(&cache.wrapperBasePtr_);
     }
 
-    setRefCoreStreamer(ep);
-    //make code exception safe
-    std::shared_ptr<void> refCoreStreamerGuard(nullptr, [](void*) {
-      setRefCoreStreamer(false);
-      ;
-    });
-
-    tree_.getEntryForAllBranches();
+    {
+      // ROOT might use multiple threads while reading the entries
+      MultiThreadRefCoreStreamerGuard epGuard(ep);
+      tree_.getEntryForAllBranches();
+    }
   }
 }  // namespace edm