Use std::atomic

lmoriche · saleelk · commit bdac18e74d43 · 2020-09-09T14:55:29.000-04:00
Replace amd::Atomic with std::atomic. Remove make_atomic uses by
converting the variable to std::atomic and making sure the memory
order is relaxed when synchronizes-with is not needed.

Delete utils/atomic.hpp.

Change-Id: I0b36db8d604a8510ac6e36b32885fd16a1b8ccfa
diff --git a/device/device.cpp b/device/device.cpp
@@ -19,7 +19,6 @@
  THE SOFTWARE. */
 
 #include "device/device.hpp"
-#include "thread/atomic.hpp"
 #include "thread/monitor.hpp"
 #include "utils/options.hpp"
 #include "comgrctx.hpp"
diff --git a/device/devprogram.cpp b/device/devprogram.cpp
@@ -30,6 +30,7 @@
 #include "comgrctx.hpp"
 
 #include <algorithm>
+#include <atomic>
 #include <cstdio>
 #include <fstream>
 #include <iostream>
@@ -1345,7 +1346,7 @@ bool Program::initBuild(amd::option::Options* options) {
   programOptions_ = options;
 
   if (options->oVariables->DumpFlags > 0) {
-    static amd::Atomic<unsigned> build_num = 0;
+    static std::atomic<uint> build_num{0};
     options->setBuildNo(build_num++);
   }
   buildLog_.clear();
diff --git a/device/gpu/gpudevice.hpp b/device/gpu/gpudevice.hpp
@@ -37,6 +37,7 @@
 #include "device/gpu/gpusettings.hpp"
 #include "device/gpu/gpuappprofile.hpp"
 
+#include <atomic>
 
 #include "acl.h"
 #include "vaminterface.h"
@@ -316,7 +317,7 @@ class Device : public NullDevice, public CALGSLDevice {
     Resource::MemoryType type_;       //!< The buffer's type
     size_t bufSize_;                  //!< Staged buffer size
     std::list<Memory*> freeBuffers_;  //!< The list of free buffers
-    amd::Atomic<uint> acquiredCnt_;   //!< The total number of acquired buffers
+    std::atomic<uint> acquiredCnt_;   //!< The total number of acquired buffers
     amd::Monitor lock_;               //!< Staged buffer acquire/release lock
     const Device& gpuDevice_;         //!< GPU device object
   };
diff --git a/device/gpu/gpumemory.hpp b/device/gpu/gpumemory.hpp
@@ -22,7 +22,6 @@
 #define GPUMEMORY_HPP_
 
 #include "top.hpp"
-#include "thread/atomic.hpp"
 #include "device/gpu/gpuresource.hpp"
 #include <map>
 
diff --git a/device/gpu/gpuresource.cpp b/device/gpu/gpuresource.cpp
@@ -34,7 +34,6 @@
 #include "device/gpu/gpudevice.hpp"
 #include "device/gpu/gpublit.hpp"
 #include "device/gpu/gputimestamp.hpp"
-#include "thread/atomic.hpp"
 #include "hsa_ext_image.h"
 
 #include <string>
diff --git a/device/gpu/gpuresource.hpp b/device/gpu/gpuresource.hpp
@@ -25,6 +25,8 @@
 #include "platform/program.hpp"
 #include "device/gpu/gpudefs.hpp"
 
+#include <atomic>
+
 //! \namespace gpu GPU Resource Implementation
 namespace gpu {
 
@@ -412,7 +414,7 @@ class Resource : public amd::HeapObject {
 
   const Device& gpuDevice_;       //!< GPU device
   CalResourceDesc cal_;           //!< CAL descriptor for this resource
-  amd::Atomic<int> mapCount_;     //!< Total number of maps
+  std::atomic<int> mapCount_;     //!< Total number of maps
   void* address_;                 //!< Physical address of this resource
   size_t offset_;                 //!< Resource offset
   size_t curRename_;              //!< Current active rename in the list
diff --git a/device/pal/paldevice.hpp b/device/pal/paldevice.hpp
@@ -39,6 +39,7 @@
 #include "acl.h"
 #include "memory"
 
+#include <atomic>
 #include <unordered_set>
 
 /*! \addtogroup PAL
@@ -258,7 +259,7 @@ class Device : public NullDevice {
     Resource::MemoryType type_;       //!< The buffer's type
     size_t bufSize_;                  //!< Staged buffer size
     std::list<Memory*> freeBuffers_;  //!< The list of free buffers
-    amd::Atomic<uint> acquiredCnt_;   //!< The total number of acquired buffers
+    std:atomic<uint> acquiredCnt_;   //!< The total number of acquired buffers
     amd::Monitor lock_;               //!< Stgaed buffer acquire/release lock
     const Device& gpuDevice_;         //!< GPU device object
   };
diff --git a/device/pal/palmemory.hpp b/device/pal/palmemory.hpp
@@ -21,7 +21,6 @@
 #pragma once
 
 #include "top.hpp"
-#include "thread/atomic.hpp"
 #include "device/pal/palresource.hpp"
 #include <map>
 
diff --git a/device/pal/palresource.cpp b/device/pal/palresource.cpp
@@ -28,7 +28,6 @@
 #include "device/pal/paldevice.hpp"
 #include "device/pal/palblit.hpp"
 #include "device/pal/paltimestamp.hpp"
-#include "thread/atomic.hpp"
 #include "hsa_ext_image.h"
 #ifdef _WIN32
 #include <d3d10_1.h>
diff --git a/device/pal/palresource.hpp b/device/pal/palresource.hpp
@@ -25,6 +25,7 @@
 #include "device/pal/paldefs.hpp"
 #include "util/palBuddyAllocatorImpl.h"
 
+#include <atomic>
 #include <unordered_map>
 
 //! \namespace pal PAL Resource Implementation
@@ -479,7 +480,7 @@ class Resource : public amd::HeapObject {
 
   const Device& gpuDevice_;     //!< GPU device
   Descriptor desc_;             //!< Descriptor for this resource
-  amd::Atomic<int> mapCount_;   //!< Total number of maps
+  std::atomic<int> mapCount_;   //!< Total number of maps
   void* address_;               //!< Physical address of this resource
   size_t offset_;               //!< Resource offset
   GpuMemoryReference* memRef_;  //!< PAL resource reference
diff --git a/device/rocm/rocdevice.hpp b/device/rocm/rocdevice.hpp
@@ -45,6 +45,7 @@
 #include "hsa_ext_amd.h"
 #include "hsa_ven_amd_loader.h"
 
+#include <atomic>
 #include <iostream>
 #include <vector>
 #include <memory>
@@ -281,7 +282,7 @@ class Device : public NullDevice {
 
     size_t bufSize_;                  //!< Staged buffer size
     std::list<Memory*> freeBuffers_;  //!< The list of free buffers
-    amd::Atomic<uint> acquiredCnt_;   //!< The total number of acquired buffers
+    std::atomic_uint acquiredCnt_;   //!< The total number of acquired buffers
     amd::Monitor lock_;               //!< Stgaed buffer acquire/release lock
     const Device& gpuDevice_;         //!< GPU device object
   };
@@ -535,7 +536,7 @@ class Device : public NullDevice {
                             std::vector<LinkAttrType>* link_attr);
 
  public:
-  amd::Atomic<uint> numOfVgpus_;  //!< Virtual gpu unique index
+  std::atomic<uint> numOfVgpus_;  //!< Virtual gpu unique index
 
   //! enum for keeping the total and available queue priorities
   enum QueuePriority : uint { Low = 0, Normal = 1, High = 2, Total = 3};
diff --git a/include/top.hpp b/include/top.hpp
@@ -45,6 +45,7 @@
 #include <xmmintrin.h>
 #endif /*!ATI_ARCH_ARM*/
 
+#include <atomic>
 #include <cstdint>
 #include <cstddef>
 #include <new>
@@ -169,7 +170,7 @@ class HeapObject {
 /*! \brief For all reference counted objects.
  */
 class ReferenceCountedObject {
-  volatile uint referenceCount_;
+  std::atomic<uint> referenceCount_;
 
  protected:
   virtual ~ReferenceCountedObject() {}
@@ -185,7 +186,7 @@ class ReferenceCountedObject {
   };
   void operator delete(void* obj, size_t extSize) { ReferenceCountedObject::operator delete(obj); }
 
-  uint referenceCount() const { return referenceCount_; }
+  uint referenceCount() const { return referenceCount_.load(std::memory_order_relaxed); }
 
   uint retain();
   uint release();
diff --git a/os/os_win32.cpp b/os/os_win32.cpp
@@ -29,6 +29,7 @@
 #include <time.h>
 #include <intrin.h>
 
+#include <atomic>
 #include <vector>
 #include <string>
 #include <sstream>
diff --git a/platform/activity.hpp b/platform/activity.hpp
@@ -20,12 +20,12 @@
 
 #pragma once
 
+#include "thread/monitor.hpp"
+
 #include <atomic>
 #include <mutex>
 #include <thread>
 
-#include "thread/monitor.hpp"
-
 #define USE_PROF_API 1
 
 #if USE_PROF_API
diff --git a/platform/command.cpp b/platform/command.cpp
@@ -36,6 +36,7 @@
 #include "platform/agent.hpp"
 #include "os/alloc.hpp"
 
+#include <atomic>
 #include <cstring>
 #include <algorithm>
 
@@ -91,7 +92,7 @@ uint64_t epoch = 0;
 bool Event::setStatus(int32_t status, uint64_t timeStamp) {
   assert(status <= CL_QUEUED && "invalid status");
 
-  int32_t currentStatus = status_;
+  int32_t currentStatus = this->status();
   if (currentStatus <= CL_COMPLETE || currentStatus <= status) {
     // We can only move forward in the execution status.
     return false;
@@ -104,7 +105,7 @@ bool Event::setStatus(int32_t status, uint64_t timeStamp) {
     }
   }
 
-  if (!make_atomic(status_).compareAndSet(currentStatus, status)) {
+  if (!status_.compare_exchange_strong(currentStatus, status, std::memory_order_relaxed)) {
     // Somebody else beat us to it, let them deal with the release/signal.
     return false;
   }
@@ -154,7 +155,7 @@ bool Event::setCallback(int32_t status, Event::CallBackFunction callback, void*
     ;  // Someone else is also updating the head of the linked list! reload.
 
   // Check if the event has already reached 'status'
-  if (status_ <= status && entry->callback_ != CallBackFunction(0)) {
+  if (this->status() <= status && entry->callback_ != CallBackFunction(0)) {
     if (entry->callback_.exchange(NULL) != NULL) {
       callback(as_cl(this), status, entry->data_);
     }
@@ -183,30 +184,30 @@ void Event::processCallbacks(int32_t status) const {
 }
 
 bool Event::awaitCompletion() {
-  if (status_ > CL_COMPLETE) {
+  if (status() > CL_COMPLETE) {
     // Notifies current command queue about waiting
     if (!notifyCmdQueue()) {
       return false;
     }
 
-    ClPrint(LOG_DEBUG, LOG_WAIT, "waiting for event %p to complete, current status %d", this, status_);
+    ClPrint(LOG_DEBUG, LOG_WAIT, "waiting for event %p to complete, current status %d", this, status());
     auto* queue = command().queue();
     if ((queue != nullptr) && queue->vdev()->ActiveWait()) {
-      while (status_ > CL_COMPLETE) {
+      while (status() > CL_COMPLETE) {
         amd::Os::yield();
       }
     } else {
       ScopedLock lock(lock_);
 
       // Wait until the status becomes CL_COMPLETE or negative.
-      while (status_ > CL_COMPLETE) {
+      while (status() > CL_COMPLETE) {
         lock_.wait();
       }
     }
     ClPrint(LOG_DEBUG, LOG_WAIT, "event %p wait completed", this);
   }
 
-  return status_ == CL_COMPLETE;
+  return status() == CL_COMPLETE;
 }
 
 bool Event::notifyCmdQueue() {
diff --git a/platform/command.hpp b/platform/command.hpp
@@ -90,7 +90,7 @@ class Event : public RuntimeObject {
   Monitor lock_;
 
   std::atomic<CallBackEntry*> callbacks_;  //!< linked list of callback entries.
-  volatile int32_t status_;                 //!< current execution status.
+  std::atomic<int32_t> status_;            //!< current execution status.
   std::atomic_flag notified_;              //!< Command queue was notified
 
  protected:
@@ -163,7 +163,7 @@ class Event : public RuntimeObject {
   const ProfilingInfo& profilingInfo() const { return profilingInfo_; }
 
   //! Return this command's execution status.
-  int32_t status() const { return status_; }
+  int32_t status() const { return status_.load(std::memory_order_relaxed); }
 
   //! Insert the given \a callback into the callback stack.
   bool setCallback(int32_t status, CallBackFunction callback, void* data);
diff --git a/platform/memory.cpp b/platform/memory.cpp
@@ -26,6 +26,8 @@
 #include "platform/memory.hpp"
 #include "device/device.hpp"
 
+#include <atomic>
+
 namespace amd {
 
 bool BufferRect::create(const size_t* bufferOrigin, const size_t* region, size_t bufferRowPitch,
@@ -277,7 +279,7 @@ bool Memory::create(void* initFrom, bool sysMemAlloc, bool skipAlloc, bool force
 
   // Create memory on all available devices
   for (size_t i = 0; i < devices.size(); i++) {
-    deviceAlloced_[devices[i]] = AllocInit;
+    deviceAlloced_[devices[i]].store(AllocInit, std::memory_order_relaxed);
 
     deviceMemories_[i].ref_ = devices[i];
     deviceMemories_[i].value_ = NULL;
@@ -301,11 +303,11 @@ bool Memory::addDeviceMemory(const Device* dev) {
   AllocState create = AllocCreate;
   AllocState init = AllocInit;
 
-  if (make_atomic(deviceAlloced_[dev]).compareAndSet(init, create)) {
+  if (deviceAlloced_[dev].compare_exchange_strong(init, create, std::memory_order_acq_rel)) {
     // Check if runtime already allocated all available slots for device memory
     if (numDevices() == NumDevicesWithP2P()) {
       // Mark the allocation as an empty
-      deviceAlloced_[dev] = AllocInit;
+      deviceAlloced_[dev].store(AllocInit, std::memory_order_release);
       return false;
     }
     device::Memory* dm = dev->createMemory(*this);
@@ -325,17 +327,17 @@ bool Memory::addDeviceMemory(const Device* dev) {
     } else {
       LogError("Video memory allocation failed!");
       // Mark the allocation as an empty
-      deviceAlloced_[dev] = AllocInit;
+      deviceAlloced_[dev].store(AllocInit, std::memory_order_release);
     }
   }
 
   // Make sure runtime finished memory allocation.
   // Loop if in the create state
-  while (deviceAlloced_[dev] == AllocCreate) {
+  while (deviceAlloced_[dev].load(std::memory_order_acquire) == AllocCreate) {
     Os::yield();
   }
 
-  if (deviceAlloced_[dev] == AllocComplete) {
+  if (deviceAlloced_[dev].load(std::memory_order_acquire) == AllocComplete) {
     result = true;
   }
 
@@ -357,7 +359,7 @@ void Memory::replaceDeviceMemory(const Device* dev, device::Memory* dm) {
   }
 
   deviceMemories_[i].value_ = dm;
-  deviceAlloced_[dev] = AllocRealloced;
+  deviceAlloced_[dev].store(AllocRealloced, std::memory_order_release);
 }
 
 device::Memory* Memory::getDeviceMemory(const Device& dev, bool alloc) {
diff --git a/platform/memory.hpp b/platform/memory.hpp
@@ -146,7 +146,7 @@ class Memory : public amd::RuntimeObject {
   DeviceMemory* deviceMemories_;
 
   //! The device alloced state
-  std::unordered_map<const Device*, AllocState> deviceAlloced_;
+  std::unordered_map<const Device*, std::atomic<AllocState>> deviceAlloced_;
 
   //! Linked list of destructor callbacks.
   std::atomic<DestructorCallBackEntry*> destructorCallbacks_;
diff --git a/platform/runtime.cpp b/platform/runtime.cpp
@@ -102,10 +102,12 @@ class RuntimeTearDown : public amd::HeapObject {
   ~RuntimeTearDown() { /*Runtime::tearDown();*/ }
 } runtime_tear_down;
 
-uint ReferenceCountedObject::retain() { return ++make_atomic(referenceCount_); }
+uint ReferenceCountedObject::retain() {
+  return referenceCount_.fetch_add(1, std::memory_order_relaxed) + 1;
+}
 
 uint ReferenceCountedObject::release() {
-  uint newCount = --make_atomic(referenceCount_);
+  uint newCount = referenceCount_.fetch_sub(1, std::memory_order_relaxed) - 1;
   if (newCount == 0) {
     if (terminate()) {
       delete this;
diff --git a/thread/atomic.hpp b/thread/atomic.hpp
diff --git a/thread/monitor.cpp b/thread/monitor.cpp
diff --git a/thread/monitor.hpp b/thread/monitor.hpp
