SWDEV-518831 - fix streams' sync issue in mthreads (#123)

* SWDEV-518831 - fix streams' sync issue in mthreads

1. Fix sync issue of null stream and non-null streams in
multithreads.
2. Remove assert(GetSubmissionBatch() == nullptr) as it
is invalid in multithreads.
3. Update getActiveQueues() to deal with the state of 
being terminated.

[ROCm/clr commit: 27aad09]

Author: TomSang

projects/clr/hipamd/src/hip_device.cpp

@@ -201,6 +201,7 @@ void Device::WaitActiveStreams(hip::Stream* blocking_stream, bool wait_null_stre
         // Get the last valid command
         waitForStream(active_stream);
       }
+      command->release();
     }
   }
 

projects/clr/hipamd/src/hip_stream.cpp

@@ -73,6 +73,7 @@ bool Stream::Create() {
 
 // ================================================================================================
 void Stream::Destroy(hip::Stream* stream, bool forceDestroy) {
+  stream->device().removeFromActiveQueues(stream);
   stream->device_->RemoveStream(stream);
   stream->SetForceDestroy(forceDestroy);
   stream->release();

projects/clr/rocclr/device/device.cpp

@@ -1107,6 +1107,21 @@ void Device::IpcDetach(void* dev_ptr) const {
   }
 }
 
+std::vector<amd::CommandQueue*> Device::getActiveQueues() {
+  amd::ScopedLock lock(activeQueuesLock_);
+  for (auto it = activeQueues.begin(); it != activeQueues.end();) {
+    if ((*it)->referenceCount() == 0) {
+      // It is being terminated in HostQueue::terminate().
+      // We should not wait for commands in a queue being terminated.
+      it = activeQueues.erase(it);
+    } else {
+      // In case the queue will be destroyed in Stream::Destroy().
+      (*it)->retain();
+      ++it;
+    }
+  }
+  return std::vector<amd::CommandQueue*>(activeQueues.begin(), activeQueues.end());
+}
 }  // namespace amd
 
 namespace amd::device {

projects/clr/rocclr/device/device.hpp

@@ -2119,10 +2119,7 @@ class Device : public RuntimeObject {
   }
 
   //! Returns the queues that have at least one submitted command
-  std::vector<amd::CommandQueue*> getActiveQueues() {
-     amd::ScopedLock lock(activeQueuesLock_);
-     return std::vector<amd::CommandQueue*>(activeQueues.begin(), activeQueues.end());
-  }
+  std::vector<amd::CommandQueue*> getActiveQueues();
 
   //! Adds the queue to the set of active command queues
   void addToActiveQueues(amd::CommandQueue* commandQueue) {