[SYCL RTC] Workaround data race related to the VFS containing toolchain

This regressed due to intel#19924 but,
apparently, we didn't have proper tests in place. I'm not sure what's
causing this exactly, but having each compilation create its unique
`ToolchainFS` instead of all of them using the same

`llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> SYCLToolchain::ToolchainFS`

somehow results in the test (added in this PR) passing consistently.

Author: aelovikov-intel

sycl-jit/jit-compiler/lib/rtc/DeviceCompilation.cpp

@@ -69,7 +69,12 @@ using namespace jit_compiler;
 namespace {
 
 class SYCLToolchain {
-  SYCLToolchain() {
+  // TODO: For some reason, moving this to a data member of the single instance
+  // of SYCLToolchain results in some data races leading to memory corruption
+  // (e.g., ::free() report errors).
+  static auto getToolchainFS() {
+    llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> ToolchainFS =
+        llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
     using namespace jit_compiler::resource;
 
     for (size_t i = 0; i < NumToolchainFiles; ++i) {
@@ -78,8 +83,11 @@ class SYCLToolchain {
       std::string_view Content{RF.Content.S, RF.Content.Size};
       ToolchainFS->addFile(Path, 0, llvm::MemoryBuffer::getMemBuffer(Content));
     }
+    return ToolchainFS;
   }
 
+  SYCLToolchain() = default;
+
   struct PrecompiledPreambles {
     using key = std::pair<std::string /*Opts*/, std::string /*Preamble*/>;
     std::mutex Mutex;
@@ -260,7 +268,7 @@ class SYCLToolchain {
 
     auto FS = llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(
         llvm::vfs::getRealFileSystem());
-    FS->pushOverlay(ToolchainFS);
+    FS->pushOverlay(getToolchainFS());
     if (FSOverlay)
       FS->pushOverlay(std::move(FSOverlay));
 
@@ -291,7 +299,7 @@ class SYCLToolchain {
                                           LLVMContext &Context) {
     auto FS = llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(
         llvm::vfs::getRealFileSystem());
-    FS->pushOverlay(ToolchainFS);
+    FS->pushOverlay(getToolchainFS());
 
     auto MemBuf = FS->getBufferForFile(LibPath, /*FileSize*/ -1,
                                        /*RequiresNullTerminator*/ false);
@@ -319,8 +327,6 @@ class SYCLToolchain {
   std::string_view Prefix{jit_compiler::resource::ToolchainPrefix.S,
                           jit_compiler::resource::ToolchainPrefix.Size};
   std::string ClangXXExe = (Prefix + "/bin/clang++").str();
-  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> ToolchainFS =
-      llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
 
   PrecompiledPreambles Preambles;
 };

sycl/test-e2e/KernelCompiler/multi_threaded_rtc.cpp

@@ -0,0 +1,49 @@
+// RUN: %{build} -o %t.out
+// RUN: %if hip %{ env SYCL_JIT_AMDGCN_PTX_TARGET_CPU=%{amd_arch} %} %{run} %t.out
+
+// Verify that parallel compilations work.
+
+#include <sycl/detail/core.hpp>
+#include <sycl/kernel_bundle.hpp>
+
+#include <thread>
+
+namespace syclexp = sycl::ext::oneapi::experimental;
+int main() {
+  sycl::queue q;
+  constexpr int N = 16;
+  std::string src_str = R"""(
+#include <sycl/ext/oneapi/free_function_queries.hpp>
+#include <sycl/ext/oneapi/kernel_properties/properties.hpp>
+namespace syclext = sycl::ext::oneapi;
+namespace syclexp = sycl::ext::oneapi::experimental;
+
+extern "C"
+SYCL_EXT_ONEAPI_FUNCTION_PROPERTY((syclexp::single_task_kernel))
+void foo(int *p) {
+  *p = 42;
+}
+)""";
+
+  auto Run = [&](auto... args) {
+    auto kb_src = syclexp::create_kernel_bundle_from_source(
+        q.get_context(), syclexp::source_language::sycl, src_str);
+    auto kb_exe = syclexp::build(kb_src, args...);
+  };
+
+  std::thread threads[N];
+
+  for (auto &t : threads)
+    t = std::thread{Run};
+  for (auto &t : threads)
+    t.join();
+
+  // --auto-pch:
+  auto auto_pch = syclexp::properties{
+      syclexp::build_options{std::vector<std::string>{"--auto-pch"}}};
+
+  for (auto &t : threads)
+    t = std::thread{Run, auto_pch};
+  for (auto &t : threads)
+    t.join();
+}