[SYCL] Optimize kernel cache lock (#17613)

This PR replaces `std::mutex` with the `sycl::detail::SpinLock` (that
performs better) in `KernelProgramCache`.

Author: vinser52

sycl/include/sycl/detail/spinlock.hpp

@@ -26,6 +26,8 @@ namespace detail {
 /// std::mutex, that doesn't provide such guarantees).
 class SpinLock {
 public:
+  bool try_lock() { return !MLock.test_and_set(std::memory_order_acquire); }
+
   void lock() {
     while (MLock.test_and_set(std::memory_order_acquire))
       std::this_thread::yield();

sycl/source/detail/kernel_program_cache.hpp

@@ -15,6 +15,7 @@
 #include <sycl/detail/common.hpp>
 #include <sycl/detail/locked.hpp>
 #include <sycl/detail/os_util.hpp>
+#include <sycl/detail/spinlock.hpp>
 #include <sycl/detail/ur.hpp>
 #include <sycl/detail/util.hpp>
 
@@ -421,7 +422,7 @@ class KernelProgramCache {
 
   template <typename KeyT>
   KernelFastCacheValT tryToGetKernelFast(KeyT &&CacheKey) {
-    std::unique_lock<std::mutex> Lock(MKernelFastCacheMutex);
+    KernelFastCacheReadLockT Lock(MKernelFastCacheMutex);
     auto It = MKernelFastCache.find(CacheKey);
     if (It != MKernelFastCache.end()) {
       traceKernel("Kernel fetched.", CacheKey.second, true);
@@ -445,7 +446,7 @@ class KernelProgramCache {
         return;
     }
     // Save reference between the program and the fast cache key.
-    std::unique_lock<std::mutex> Lock(MKernelFastCacheMutex);
+    KernelFastCacheWriteLockT Lock(MKernelFastCacheMutex);
     MProgramToKernelFastCacheKeyMap[Program].emplace_back(CacheKey);
 
     // if no insertion took place, thus some other thread has already inserted
@@ -483,7 +484,7 @@ class KernelProgramCache {
 
       {
         // Remove corresponding entries from KernelFastCache.
-        std::unique_lock<std::mutex> Lock(MKernelFastCacheMutex);
+        KernelFastCacheWriteLockT Lock(MKernelFastCacheMutex);
         if (auto FastCacheKeyItr =
                 MProgramToKernelFastCacheKeyMap.find(NativePrg);
             FastCacheKeyItr != MProgramToKernelFastCacheKeyMap.end()) {
@@ -630,7 +631,7 @@ class KernelProgramCache {
     std::lock_guard<std::mutex> EvictionListLock(MProgramEvictionListMutex);
     std::lock_guard<std::mutex> L1(MProgramCacheMutex);
     std::lock_guard<std::mutex> L2(MKernelsPerProgramCacheMutex);
-    std::lock_guard<std::mutex> L3(MKernelFastCacheMutex);
+    KernelFastCacheWriteLockT L3(MKernelFastCacheMutex);
     MCachedPrograms = ProgramCache{};
     MKernelsPerProgramCache = KernelCacheT{};
     MKernelFastCache = KernelFastCacheT{};
@@ -758,7 +759,10 @@ class KernelProgramCache {
   KernelCacheT MKernelsPerProgramCache;
   ContextPtr MParentContext;
 
-  std::mutex MKernelFastCacheMutex;
+  using KernelFastCacheMutexT = SpinLock;
+  using KernelFastCacheReadLockT = std::lock_guard<KernelFastCacheMutexT>;
+  using KernelFastCacheWriteLockT = std::lock_guard<KernelFastCacheMutexT>;
+  KernelFastCacheMutexT MKernelFastCacheMutex;
   KernelFastCacheT MKernelFastCache;
 
   // Map between fast kernel cache keys and program handle.