add a check to verify thread matching between the encryption thread and the thread where we release the attachment. If they match, use a dummy mutex instead of the actual dbb_thread_mutex to avoid a deadlock

Author: AlexeyMochalov

src/common/ThreadStart.cpp

@@ -219,9 +219,14 @@ ThreadId Thread::getId()
 #endif
 }
 
+bool Thread::isCurrent(InternalId iid)
+{
+	return pthread_equal(iid, pthread_self());
+}
+
 bool Thread::isCurrent()
 {
-	return pthread_equal(internalId, pthread_self());
+	return isCurrent(internalId);
 }
 
 void Thread::sleep(unsigned milliseconds)
@@ -363,9 +368,14 @@ ThreadId Thread::getId()
 	return GetCurrentThreadId();
 }
 
+bool Thread::isCurrent(InternalId iid)
+{
+	return GetCurrentThreadId() == iid;
+}
+
 bool Thread::isCurrent()
 {
-	return GetCurrentThreadId() == internalId;
+	return isCurrent(internalId);
 }
 
 void Thread::sleep(unsigned milliseconds)

src/common/ThreadStart.h

@@ -87,6 +87,7 @@ class Thread
 	static void sleep(unsigned milliseconds);
 	static void yield();
 
+	static bool isCurrent(InternalId iid);
 	bool isCurrent();
 
 	Thread()

src/jrd/CryptoManager.h

@@ -301,6 +301,10 @@ class CryptoManager final : public Firebird::PermanentStorage, public BarSync::I
 	UCHAR getCurrentState(thread_db* tdbb) const;
 	const char* getKeyName() const;
 	const char* getPluginName() const;
+	Thread::Handle getCryptThreadHandle() const
+	{
+		return cryptThreadId;
+	}
 
 private:
 	enum IoResult {SUCCESS_ALL, FAILED_CRYPT, FAILED_IO};

src/jrd/jrd.cpp

@@ -7755,14 +7755,21 @@ void release_attachment(thread_db* tdbb, Jrd::Attachment* attachment, XThreadEns
 
 	Sync sync(&dbb->dbb_sync, "jrd.cpp: release_attachment");
 
+	// dummy mutex is used to avoid races with crypto thread
+	XThreadMutex dummy_mutex;
+	XThreadEnsureUnlock dummyGuard(dummy_mutex, FB_FUNCTION);
+
 	// avoid races with special threads
 	// take into an account lock earlier taken in DROP DATABASE
 	XThreadEnsureUnlock threadGuard(dbb->dbb_thread_mutex, FB_FUNCTION);
 	XThreadEnsureUnlock* activeThreadGuard = dropGuard;
 	if (!activeThreadGuard)
 	{
-		threadGuard.enter();
-		activeThreadGuard = &threadGuard;
+		if (dbb->dbb_crypto_manager && Thread::isCurrent(dbb->dbb_crypto_manager->getCryptThreadHandle()))
+			activeThreadGuard = &dummyGuard;
+		else
+			activeThreadGuard = &threadGuard;
+		activeThreadGuard->enter();
 	}
 
 	sync.lock(SYNC_EXCLUSIVE);