rocr: Ensure globals are initialized at first use

When ROCr is built as a static library, global variables
were often not initialized to valid values at their first
use. This change addresses that problem.

Change-Id: I550fa41feb3bc04b9cc686bcfb4acf2a7b651a88

Author: cfreeamd

runtime/hsa-runtime/core/common/shared.cpp

@@ -44,7 +44,6 @@
 
 namespace rocr {
 namespace core {
-std::function<void*(size_t, size_t, uint32_t, int)> BaseShared::allocate_ = nullptr;
-std::function<void(void*)> BaseShared::free_ = nullptr;
+
 }   // namespace core
 }   // namespace rocr

runtime/hsa-runtime/core/common/shared.h

@@ -58,25 +58,36 @@ namespace core {
 class BaseShared {
  public:
   static void SetAllocateAndFree(
-      const std::function<void*(size_t, size_t, uint32_t, int)>& allocate,
-      const std::function<void(void*)>& free) {
-    allocate_ = allocate;
-    free_ = free;
+      const std::function<void*(size_t, size_t, uint32_t, int)>& alloc,
+      const std::function<void(void*)>& fr) {
+    allocate_() = alloc;
+    free_() = fr;
   }
 
  protected:
-  static std::function<void*(size_t, size_t, uint32_t, int)> allocate_;
-  static std::function<void(void*)> free_;
+  static __forceinline std::function<void*(size_t, size_t, uint32_t, int)>&
+   allocate_() {
+    static std::function<void*(size_t, size_t, uint32_t, int)> alloc =
+                                                                      nullptr;
+    return alloc;
+  }
+  static __forceinline std::function<void(void*)>&
+   free_() {
+    static std::function<void(void*)> fr = nullptr;
+    return fr;
+  }
+
 };
 
 /// @brief Default Allocator for Shared.  Ensures allocations are whole pages.
 template <typename T> class PageAllocator : private BaseShared {
  public:
   __forceinline static T* alloc(int flags = 0) {
-    T* ret = reinterpret_cast<T*>(allocate_(AlignUp(sizeof(T), 4096), 4096, flags, 0));
+    T* ret = reinterpret_cast<T*>(
+                        allocate_()(AlignUp(sizeof(T), 4096), 4096, flags, 0));
     if (ret == nullptr) throw std::bad_alloc();
 
-    MAKE_NAMED_SCOPE_GUARD(throwGuard, [&]() { free_(ret); });
+    MAKE_NAMED_SCOPE_GUARD(throwGuard, [&]() { free_()(ret); });
 
     new (ret) T;
 
@@ -85,10 +96,11 @@ template <typename T> class PageAllocator : private BaseShared {
   }
 
   __forceinline static T* alloc(int agent_node_id, int flags) {
-    T* ret = reinterpret_cast<T*>(allocate_(AlignUp(sizeof(T), 4096), 4096, flags, agent_node_id));
+    T* ret = reinterpret_cast<T*>(
+            allocate_()(AlignUp(sizeof(T), 4096), 4096, flags, agent_node_id));
     if (ret == nullptr) throw std::bad_alloc();
 
-    MAKE_NAMED_SCOPE_GUARD(throwGuard, [&]() { free_(ret); });
+    MAKE_NAMED_SCOPE_GUARD(throwGuard, [&]() { free_()(ret); });
 
     new (ret) T;
 
@@ -99,7 +111,7 @@ template <typename T> class PageAllocator : private BaseShared {
   __forceinline static void free(T* ptr) {
     if (ptr != nullptr) {
       ptr->~T();
-      free_(ptr);
+      free_()(ptr);
     }
   }
 };
@@ -110,7 +122,7 @@ template <typename T, typename Allocator = PageAllocator<T>>
 class Shared final : private BaseShared {
  public:
   explicit Shared(Allocator* pool = nullptr, int flags = 0) : pool_(pool) {
-    assert(allocate_ != nullptr && free_ != nullptr &&
+    assert(allocate_() != nullptr && free_() != nullptr &&
            "Shared object allocator is not set");
 
     if (pool_)
@@ -120,7 +132,7 @@ class Shared final : private BaseShared {
   }
 
   explicit Shared(int agent_node_id, Allocator* pool = nullptr, int flags = 0) : pool_(pool) {
-    assert(allocate_ != nullptr && free_ != nullptr &&
+    assert(allocate_() != nullptr && free_() != nullptr &&
            "Shared object allocator is not set");
 
     if (pool_)
@@ -130,7 +142,8 @@ class Shared final : private BaseShared {
   }
 
   ~Shared() {
-    assert(allocate_ != nullptr && free_ != nullptr && "Shared object allocator is not set");
+    assert(allocate_() != nullptr && free_() != nullptr &&
+                                        "Shared object allocator is not set");
 
     if (pool_)
       pool_->free(shared_object_);
@@ -164,19 +177,20 @@ class Shared final : private BaseShared {
 template <typename T> class Shared<T, PageAllocator<T>> final : private BaseShared {
  public:
   Shared(int flags = 0) {
-    assert(allocate_ != nullptr && free_ != nullptr && "Shared object allocator is not set");
+    assert(allocate_() != nullptr && free_() != nullptr &&
+                                        "Shared object allocator is not set");
 
     shared_object_ = PageAllocator<T>::alloc(flags);
   }
 
   Shared(int agent_node_id, int flags) {
-    assert(allocate_ != nullptr && free_ != nullptr && "Shared object allocator is not set");
+    assert(allocate_() != nullptr && free_() != nullptr && "Shared object allocator is not set");
 
     shared_object_ = PageAllocator<T>::alloc(agent_node_id, flags);
   }
 
   ~Shared() {
-    assert(allocate_ != nullptr && free_ != nullptr &&
+    assert(allocate_() != nullptr && free_() != nullptr &&
            "Shared object allocator is not set");
 
     PageAllocator<T>::free(shared_object_);
@@ -207,18 +221,19 @@ template <typename T, size_t Align> class SharedArray final : private BaseShared
   SharedArray() : shared_object_(nullptr) {}
 
   explicit SharedArray(size_t length) : shared_object_(nullptr), len(length) {
-    assert(allocate_ != nullptr && free_ != nullptr && "Shared object allocator is not set");
+    assert(allocate_() != nullptr && free_() != nullptr &&
+                                        "Shared object allocator is not set");
     static_assert((__alignof(T) <= Align) || (Align == 0), "Align is less than alignof(T)");
 
     shared_object_ =
-        reinterpret_cast<T*>(allocate_(sizeof(T) * length, Max(__alignof(T), Align), 0, 0));
+        reinterpret_cast<T*>(allocate_()(sizeof(T) * length, Max(__alignof(T), Align), 0, 0));
     if (shared_object_ == nullptr) throw std::bad_alloc();
 
     size_t i = 0;
 
     MAKE_NAMED_SCOPE_GUARD(loopGuard, [&]() {
       for (size_t t = 0; t < i - 1; t++) shared_object_[t].~T();
-      free_(shared_object_);
+      free_()(shared_object_);
     });
 
     for (; i < length; i++) new (&shared_object_[i]) T;
@@ -227,11 +242,12 @@ template <typename T, size_t Align> class SharedArray final : private BaseShared
   }
 
   ~SharedArray() {
-    assert(allocate_ != nullptr && free_ != nullptr && "Shared object allocator is not set");
+    assert(allocate_() != nullptr && free_() != nullptr &&
+                                        "Shared object allocator is not set");
 
     if (shared_object_ != nullptr) {
       for (size_t i = 0; i < len; i++) shared_object_[i].~T();
-      free_(shared_object_);
+      free_()(shared_object_);
     }
   }
 

runtime/hsa-runtime/core/inc/amd_aie_aql_queue.h

@@ -62,11 +62,11 @@ class AieAqlQueue : public core::Queue,
                     core::DoorbellSignal {
 public:
   static __forceinline bool IsType(core::Signal *signal) {
-    return signal->IsType(&rtti_id_);
+    return signal->IsType(&rtti_id());
   }
 
   static __forceinline bool IsType(core::Queue *queue) {
-    return queue->IsType(&rtti_id_);
+    return queue->IsType(&rtti_id());
   }
 
   AieAqlQueue() = delete;
@@ -123,7 +123,7 @@ class AieAqlQueue : public core::Queue,
   uint32_t queue_size_bytes_ = std::numeric_limits<uint32_t>::max();
 
 protected:
-  bool _IsA(Queue::rtti_t id) const override { return id == &rtti_id_; }
+  bool _IsA(Queue::rtti_t id) const override { return id == &rtti_id(); }
 
 private:
   AieAgent &agent_;
@@ -144,7 +144,11 @@ class AieAqlQueue : public core::Queue,
 
   /// @brief Indicates if queue is active.
   std::atomic<bool> active_;
-  static int rtti_id_;
+  static __forceinline int& rtti_id() {
+    static int rtti_id_ = 0;
+    return rtti_id_;
+  }
+
 };
 
 } // namespace AMD

runtime/hsa-runtime/core/inc/amd_aql_queue.h

@@ -57,10 +57,10 @@ namespace AMD {
 class AqlQueue : public core::Queue, private core::LocalSignal, public core::DoorbellSignal {
  public:
   static __forceinline bool IsType(core::Signal* signal) {
-    return signal->IsType(&rtti_id_);
+    return signal->IsType(&rtti_id());
   }
 
-  static __forceinline bool IsType(core::Queue* queue) { return queue->IsType(&rtti_id_); }
+  static __forceinline bool IsType(core::Queue* queue) { return queue->IsType(&rtti_id()); }
 
   // Acquires/releases queue resources and requests HW schedule/deschedule.
   AqlQueue(GpuAgent* agent, size_t req_size_pkts, HSAuint32 node_id,
@@ -228,7 +228,7 @@ class AqlQueue : public core::Queue, private core::LocalSignal, public core::Doo
   void AsyncReclaimAltScratch();
 
  protected:
-  bool _IsA(Queue::rtti_t id) const override { return id == &rtti_id_; }
+  bool _IsA(Queue::rtti_t id) const override { return id == &rtti_id(); }
 
  private:
   uint32_t ComputeRingBufferMinPkts();
@@ -331,18 +331,28 @@ class AqlQueue : public core::Queue, private core::LocalSignal, public core::Doo
   std::vector<uint32_t> cu_mask_;
 
   // Shared event used for queue errors
-  static HsaEvent* queue_event_;
-
+  static __forceinline HsaEvent*& queue_event() {
+    static HsaEvent* queue_event_ = nullptr;
+    return queue_event_;
+  }
   // Queue count - used to ref count queue_event_
-  static std::atomic<uint32_t> queue_count_;
+  static __forceinline std::atomic<uint32_t>& queue_count() {
+    static std::atomic<uint32_t> queue_count_(0);
+    return queue_count_;
+  }
 
   // Mutex for queue_event_ manipulation
-  static KernelMutex queue_lock_;
-
+  static __forceinline KernelMutex& queue_lock() {
+    static KernelMutex queue_lock_;
+    return queue_lock_;
+}
   // Async scratch single limit - may be modified after init
   size_t async_scratch_single_limit_;
 
-  static int rtti_id_;
+  static __forceinline int& rtti_id() {
+    static int rtti_id_ = 0;
+    return rtti_id_;
+  }
 
   // Forbid copying and moving of this object
   DISALLOW_COPY_AND_ASSIGN(AqlQueue);

runtime/hsa-runtime/core/inc/amd_blit_kernel.h

@@ -117,6 +117,7 @@ class BlitKernel : public core::Blit {
   virtual void GangLeader(bool gang_leader) override {}
   virtual bool GangLeader() const override { return false; }
 
+  const uint16_t kInvalidPacketHeader = HSA_PACKET_TYPE_INVALID;
  private:
   union KernelArgs {
     struct __ALIGNED__(16) {

runtime/hsa-runtime/core/inc/amd_memory_region.h

@@ -167,7 +167,7 @@ class MemoryRegion : public core::MemoryRegion {
     return static_cast<uint32_t>(mem_props_.MemoryClockMax);
   }
 
-  __forceinline size_t GetPageSize() const { return kPageSize_; }
+  __forceinline size_t GetPageSize() const { return kPageSize(); }
 
   __forceinline const HsaMemFlags &mem_flags() const { return mem_flag_; }
   __forceinline const HsaMemMapFlags &map_flags() const { return map_flag_; }
@@ -195,7 +195,10 @@ class MemoryRegion : public core::MemoryRegion {
   // fragments of the block routing to the same MemoryRegion.
   mutable KernelMutex access_lock_;
 
-  static size_t kPageSize_;
+  static __forceinline const size_t& kPageSize() {
+    static size_t kPageSize_ = sysconf(_SC_PAGESIZE);
+    return kPageSize_;
+  }
 
   // Determine access type allowed to requesting device
   hsa_amd_memory_pool_access_t GetAccessInfo(const core::Agent& agent,

runtime/hsa-runtime/core/inc/default_signal.h

@@ -59,7 +59,7 @@ class BusyWaitSignal : public Signal {
   /// @brief Determines if a Signal* can be safely converted to BusyWaitSignal*
   /// via static_cast.
   static __forceinline bool IsType(Signal* ptr) {
-    return ptr->IsType(&rtti_id_);
+    return ptr->IsType(&rtti_id());
   }
 
   /// @brief See base class Signal.
@@ -154,10 +154,13 @@ class BusyWaitSignal : public Signal {
   __forceinline HsaEvent* EopEvent() { return NULL; }
 
  protected:
-  bool _IsA(rtti_t id) const { return id == &rtti_id_; }
+  bool _IsA(rtti_t id) const { return id == &rtti_id(); }
 
  private:
-  static int rtti_id_;
+  static __forceinline int& rtti_id() {
+    static int rtti_id_ = 0;
+    return rtti_id_;
+  }
 
   DISALLOW_COPY_AND_ASSIGN(BusyWaitSignal);
 };
@@ -167,20 +170,23 @@ class DefaultSignal : private LocalSignal, public BusyWaitSignal {
  public:
   /// @brief Determines if a Signal* can be safely converted to BusyWaitSignal*
   /// via static_cast.
-  static __forceinline bool IsType(Signal* ptr) { return ptr->IsType(&rtti_id_); }
+  static __forceinline bool IsType(Signal* ptr) { return ptr->IsType(&rtti_id()); }
 
   /// @brief See base class Signal.
   explicit DefaultSignal(hsa_signal_value_t initial_value, bool enableIPC = false)
       : LocalSignal(initial_value, enableIPC), BusyWaitSignal(signal(), enableIPC) {}
 
  protected:
   bool _IsA(rtti_t id) const {
-    if (id == &rtti_id_) return true;
+    if (id == &rtti_id()) return true;
     return BusyWaitSignal::_IsA(id);
   }
 
  private:
-  static int rtti_id_;
+  static __forceinline int& rtti_id() {
+    static int rtti_id_ = 0;
+    return rtti_id_;
+  }
 
   DISALLOW_COPY_AND_ASSIGN(DefaultSignal);
 };

runtime/hsa-runtime/core/inc/host_queue.h

@@ -52,7 +52,7 @@ namespace rocr {
 namespace core {
 class HostQueue : public Queue {
  public:
-  static __forceinline bool IsType(core::Queue* queue) { return queue->IsType(&rtti_id_); }
+  static __forceinline bool IsType(core::Queue* queue) { return queue->IsType(&rtti_id()); }
 
   HostQueue(hsa_region_t region, uint32_t ring_size, hsa_queue_type32_t type,
             uint32_t features, hsa_signal_t doorbell_signal);
@@ -175,17 +175,23 @@ class HostQueue : public Queue {
   void operator delete(void*, void*) {}
 
  protected:
-  bool _IsA(Queue::rtti_t id) const override { return id == &rtti_id_; }
+  bool _IsA(Queue::rtti_t id) const override { return id == &rtti_id(); }
 
  private:
-  static int rtti_id_;
+  static __forceinline int& rtti_id() {
+    static int rtti_id_ = 0;
+    return rtti_id_;
+  }
   static const size_t kRingAlignment = 256;
   const uint32_t size_;
   void* ring_;
 
   // Host queue id counter, starting from 0x80000000 to avoid overlaping
   // with aql queue id.
-  static std::atomic<uint32_t> queue_count_;
+  static __forceinline std::atomic<uint32_t>& queue_count() {
+    static std::atomic<uint32_t> queue_count_;
+    return queue_count_;
+  }
 
   DISALLOW_COPY_AND_ASSIGN(HostQueue);
 };