MB-31544: Fix race condition adding external auth requests

Lock the task mutex while we've got the external
auth mutex so that the external auth thread can't
get the task mutex before Task::execute returns

Change-Id: I00bf04b9f9faa6ab6f4cb4a3e218aac95572364a
Reviewed-on: http://review.couchbase.org/100302
Tested-by: Build Bot <[email protected]>
Reviewed-by: Daniel Owen <[email protected]>

Author: trondn

daemon/external_auth_manager_thread.cc

@@ -97,7 +97,19 @@ void ExternalAuthManagerThread::remove(Connection& connection) {
 }
 
 void ExternalAuthManagerThread::enqueueRequest(StartSaslAuthTask& request) {
+    // We need to make sure that the lock ordering for these
+    // mutexes is the same. Let's unlock the task (and the executor thread
+    // is currently blocked waiting for this method to return. It won't
+    // touch the mutex until we return.
+    // Then we'll grab the external auth manager mutex and get our mutex
+    // back (no one else knows about that mutex yet so it should never
+    // block). Then we'll just release the external auth manager lock,
+    // and the external auth thread may start processing these events,
+    // but it'll have to wait until we release the request mutex
+    // before it may signal the task.
+    request.getMutex().unlock();
     std::lock_guard<std::mutex> guard(mutex);
+    request.getMutex().lock();
     incomingRequests.push(&request);
     condition_variable.notify_all();
 }

daemon/start_sasl_auth_task.cc

@@ -79,9 +79,7 @@ Task::Status StartSaslAuthTask::internal_auth() {
         // We can't hold this lock when we're trying to enqueue the
         // request
         internal = false;
-        getMutex().unlock();
         externalAuthManager->enqueueRequest(*this);
-        getMutex().lock();
     }
 
     return internal ? Status::Finished : Status::Continue;