diff --git a/sycl/source/detail/sycl_mem_obj_t.cpp b/sycl/source/detail/sycl_mem_obj_t.cpp
index 46f7305a11cc..f9410aa750f0 100644
--- a/sycl/source/detail/sycl_mem_obj_t.cpp
+++ b/sycl/source/detail/sycl_mem_obj_t.cpp
@@ -141,16 +141,25 @@ void SYCLMemObjT::updateHostMemory(void *const Ptr) {
 }
 
 void SYCLMemObjT::updateHostMemory() {
-  if ((MUploadDataFunctor != nullptr) && MNeedWriteBack)
+  // Don't try updating host memory when stutting down.
+  if ((MUploadDataFunctor != nullptr) && MNeedWriteBack &&
+      GlobalHandler::instance().isOkToDefer())
     MUploadDataFunctor();
 
   // If we're attached to a memory record, process the deletion of the memory
   // record. We may get detached before we do this.
   if (MRecord) {
-    bool Result = Scheduler::getInstance().removeMemoryObject(this);
+    // Don't strictly try holding the lock in removeMemoryObject during shutdown
+    // to prevent deadlocks.
+    bool Result = Scheduler::getInstance().removeMemoryObject(
+        this, GlobalHandler::instance().isOkToDefer());
     std::ignore = Result; // for no assert build
+
+    // removeMemoryObject might fail during shutdown because of not being
+    // able to hold write lock. This can happen if shutdown happen due to
+    // exception/termination while holding lock.
     assert(
-        Result &&
+        (Result || !GlobalHandler::instance().isOkToDefer()) &&
         "removeMemoryObject should not return false in mem object destructor");
   }
   releaseHostMem(MShadowCopy);