[SYCL] Fix a thread pool data race during shutdown

The flag that kills the threads waiting for work was an atomic bool,
but that is not enough: any modification of a condition must be made
under the mutex associated with that condition variable. Otherwise,
the bool value flip and the call to notify might happen after the
condition check in the other thread, but before it starts waiting for
notifications.

Author: sergey-semenov

sycl/source/detail/thread_pool.hpp

@@ -29,17 +29,17 @@ class ThreadPool {
   std::queue<std::function<void()>> MJobQueue;
   std::mutex MJobQueueMutex;
   std::condition_variable MDoSmthOrStop;
-  std::atomic_bool MStop;
+  bool MStop = false;
   std::atomic_uint MJobsInPool;
 
   void worker() {
     GlobalHandler::instance().registerSchedulerUsage(/*ModifyCounter*/ false);
     std::unique_lock<std::mutex> Lock(MJobQueueMutex);
     while (true) {
-      MDoSmthOrStop.wait(
-          Lock, [this]() { return !MJobQueue.empty() || MStop.load(); });
+      MDoSmthOrStop.wait(Lock,
+                         [this]() { return !MJobQueue.empty() || MStop; });
 
-      if (MStop.load())
+      if (MStop)
         break;
 
       std::function<void()> Job = std::move(MJobQueue.front());
@@ -57,7 +57,6 @@ class ThreadPool {
   void start() {
     MLaunchedThreads.reserve(MThreadCount);
 
-    MStop.store(false);
     MJobsInPool.store(0);
 
     for (size_t Idx = 0; Idx < MThreadCount; ++Idx)
@@ -83,7 +82,10 @@ class ThreadPool {
   }
 
   void finishAndWait() {
-    MStop.store(true);
+    {
+      std::lock_guard<std::mutex> Lock(MJobQueueMutex);
+      MStop = true;
+    }
 
     MDoSmthOrStop.notify_all();
 

sycl/test-e2e/Regression/unwaited_for_host_task.cpp

@@ -0,0 +1,16 @@
+// RUN: %{build} -o %t.out
+// RUN: %{run} %t.out
+
+#include <sycl/sycl.hpp>
+
+using namespace sycl;
+
+// This test checks that host tasks that haven't been waited for do not cause
+// issues during shutdown.
+int main(int argc, char *argv[]) {
+  queue q;
+
+  q.submit([&](handler &h) { h.host_task([=]() {}); });
+
+  return 0;
+}