#4777 Fix webrtc crashing in worker threads on shutdown

akleshchev · akleshchev · commit f846861f04fe · 2025-10-03T19:07:00.000+03:00
diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp
@@ -374,8 +374,6 @@ void LLWebRTCImpl::terminate()
 
     mSignalingThread->BlockingCall([this]() { mPeerConnectionFactory = nullptr; });
 
-    mPeerConnections.clear();
-
     mWorkerThread->BlockingCall(
         [this]()
         {
@@ -386,6 +384,14 @@ void LLWebRTCImpl::terminate()
             mDeviceModule     = nullptr;
             mTaskQueueFactory = nullptr;
         });
+
+    // In case peer connections still somehow have jobs in workers,
+    // only clear connections up after clearing workers.
+    mNetworkThread = nullptr;
+    mWorkerThread = nullptr;
+    mSignalingThread = nullptr;
+
+    mPeerConnections.clear();
     webrtc::LogMessage::RemoveLogToStream(mLogSink);
 }
 
@@ -766,6 +772,7 @@ void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnectionInterface* peer_conn
     std::find(mPeerConnections.begin(), mPeerConnections.end(), peer_connection);
     if (it != mPeerConnections.end())
     {
+        // Todo: make sure conection had no jobs in workers
         mPeerConnections.erase(it);
         if (mPeerConnections.empty())
         {
@@ -785,14 +792,19 @@ LLWebRTCPeerConnectionImpl::LLWebRTCPeerConnectionImpl() :
     mWebRTCImpl(nullptr),
     mPeerConnection(nullptr),
     mMute(MUTE_INITIAL),
-    mAnswerReceived(false)
+    mAnswerReceived(false),
+    mPendingJobs(0)
 {
 }
 
 LLWebRTCPeerConnectionImpl::~LLWebRTCPeerConnectionImpl()
 {
     mSignalingObserverList.clear();
     mDataObserverList.clear();
+    if (mPendingJobs > 0)
+    {
+        RTC_LOG(LS_ERROR) << __FUNCTION__ << "Destroying a connection that has " << std::to_string(mPendingJobs) << " unfinished jobs that might cause workers to crash";
+    }
 }
 
 //
@@ -804,8 +816,10 @@ void LLWebRTCPeerConnectionImpl::init(LLWebRTCImpl * webrtc_impl)
     mWebRTCImpl = webrtc_impl;
     mPeerConnectionFactory = mWebRTCImpl->getPeerConnectionFactory();
 }
+
 void LLWebRTCPeerConnectionImpl::terminate()
 {
+    mPendingJobs++;
     mWebRTCImpl->PostSignalingTask(
         [this]()
         {
@@ -848,7 +862,9 @@ void LLWebRTCPeerConnectionImpl::terminate()
                     observer->OnPeerConnectionClosed();
                 }
             }
+            mPendingJobs--;
         });
+    mPeerConnectionFactory.release();
 }
 
 void LLWebRTCPeerConnectionImpl::setSignalingObserver(LLWebRTCSignalingObserver *observer) { mSignalingObserverList.emplace_back(observer); }
@@ -869,6 +885,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection(const LLWebRTCPeerConnecti
     RTC_DCHECK(!mPeerConnection);
     mAnswerReceived = false;
 
+    mPendingJobs++;
     mWebRTCImpl->PostSignalingTask(
         [this,options]()
         {
@@ -902,6 +919,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection(const LLWebRTCPeerConnecti
                 {
                     observer->OnRenegotiationNeeded();
                 }
+                mPendingJobs--;
                 return;
             }
 
@@ -964,6 +982,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection(const LLWebRTCPeerConnecti
 
             webrtc::PeerConnectionInterface::RTCOfferAnswerOptions offerOptions;
             mPeerConnection->CreateOffer(this, offerOptions);
+            mPendingJobs--;
         });
 
     return true;
@@ -1006,6 +1025,7 @@ void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp)
 {
     RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp;
 
+    mPendingJobs++;
     mWebRTCImpl->PostSignalingTask(
                                [this, sdp]()
                                {
@@ -1015,6 +1035,7 @@ void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp)
                                        mPeerConnection->SetRemoteDescription(webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp),
                                                                              webrtc::scoped_refptr<webrtc::SetRemoteDescriptionObserverInterface>(this));
                                    }
+                                   mPendingJobs--;
                                });
 }
 
@@ -1037,6 +1058,7 @@ void LLWebRTCPeerConnectionImpl::setMute(bool mute)
     mMute = new_state;
 
 
+    mPendingJobs++;
     mWebRTCImpl->PostSignalingTask(
         [this, force_reset, enable]()
         {
@@ -1060,6 +1082,7 @@ void LLWebRTCPeerConnectionImpl::setMute(bool mute)
                     track->set_enabled(enable);
                 }
             }
+            mPendingJobs--;
         }
     });
 }
@@ -1081,6 +1104,7 @@ void LLWebRTCPeerConnectionImpl::resetMute()
 
 void LLWebRTCPeerConnectionImpl::setReceiveVolume(float volume)
 {
+    mPendingJobs++;
     mWebRTCImpl->PostSignalingTask(
         [this, volume]()
         {
@@ -1099,11 +1123,13 @@ void LLWebRTCPeerConnectionImpl::setReceiveVolume(float volume)
                     }
                 }
             }
+            mPendingJobs--;
         });
 }
 
 void LLWebRTCPeerConnectionImpl::setSendVolume(float volume)
 {
+    mPendingJobs++;
     mWebRTCImpl->PostSignalingTask(
         [this, volume]()
         {
@@ -1114,6 +1140,7 @@ void LLWebRTCPeerConnectionImpl::setSendVolume(float volume)
                     track->GetSource()->SetVolume(volume*5.0);
                 }
             }
+            mPendingJobs--;
         });
 }
 
@@ -1190,11 +1217,13 @@ void LLWebRTCPeerConnectionImpl::OnConnectionChange(webrtc::PeerConnectionInterf
     {
         case webrtc::PeerConnectionInterface::PeerConnectionState::kConnected:
         {
+            mPendingJobs++;
             mWebRTCImpl->PostWorkerTask([this]() {
                 for (auto &observer : mSignalingObserverList)
                 {
                     observer->OnAudioEstablished(this);
                 }
+                mPendingJobs--;
             });
             break;
         }
@@ -1452,11 +1481,13 @@ void LLWebRTCPeerConnectionImpl::sendData(const std::string& data, bool binary)
     {
         webrtc::CopyOnWriteBuffer cowBuffer(data.data(), data.length());
         webrtc::DataBuffer     buffer(cowBuffer, binary);
+        mPendingJobs++;
         mWebRTCImpl->PostNetworkTask([this, buffer]() {
                 if (mDataChannel)
                 {
                     mDataChannel->Send(buffer);
                 }
+                mPendingJobs--;
             });
     }
 }
diff --git a/indra/llwebrtc/llwebrtc_impl.h b/indra/llwebrtc/llwebrtc_impl.h
@@ -422,6 +422,9 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceO
     ~LLWebRTCImpl()
     {
         delete mLogSink;
+
+        // Explicit cleanup for the sake of debugging and crash stacks
+        mPeerCustomProcessor = nullptr;
     }
 
     void init();
@@ -669,6 +672,8 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnectionInterface,
     // data
     std::vector<LLWebRTCDataObserver *> mDataObserverList;
     webrtc::scoped_refptr<webrtc::DataChannelInterface> mDataChannel;
+
+    std::atomic<int> mPendingJobs;
 };
 
 }
diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp
@@ -291,9 +291,8 @@ void LLWebRTCVoiceClient::terminate()
     LL_INFOS("Voice") << "Terminating WebRTC" << LL_ENDL;
 
     mVoiceEnabled = false;
+    sShuttingDown = true; // so that coroutines won't post more work.
     llwebrtc::terminate();
-
-    sShuttingDown = true;
 }
 
 //---------------------------------------------------
@@ -2659,6 +2658,11 @@ void LLVoiceWebRTCConnection::breakVoiceConnectionCoro(connectionPtr_t connectio
 void LLVoiceWebRTCSpatialConnection::requestVoiceConnection()
 {
     LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE;
+    if (LLWebRTCVoiceClient::isShuttingDown())
+    {
+        mOutstandingRequests--;
+        return;
+    }
 
     LLViewerRegion *regionp = LLWorld::instance().getRegionFromID(mRegionID);
 
@@ -3266,6 +3270,12 @@ void LLVoiceWebRTCAdHocConnection::requestVoiceConnection()
 {
     LL_PROFILE_ZONE_SCOPED_CATEGORY_VOICE;
 
+    if (LLWebRTCVoiceClient::isShuttingDown())
+    {
+        mOutstandingRequests--;
+        return;
+    }
+
     LLViewerRegion *regionp = LLWorld::instance().getRegionFromID(mRegionID);
 
     LL_DEBUGS("Voice") << "Requesting voice connection." << LL_ENDL;

Original file line number	Diff line number	Diff line change
`@@ -374,8 +374,6 @@ void LLWebRTCImpl::terminate()`
`374`	`374`
`375`	`375`	`mSignalingThread->BlockingCall([this]() { mPeerConnectionFactory = nullptr; });`
`376`	`376`
`377`		`- mPeerConnections.clear();`
`378`		`-`
`379`	`377`	`mWorkerThread->BlockingCall(`
`380`	`378`	`[this]()`
`381`	`379`	`{`
`@@ -386,6 +384,14 @@ void LLWebRTCImpl::terminate()`
`386`	`384`	`mDeviceModule = nullptr;`
`387`	`385`	`mTaskQueueFactory = nullptr;`
`388`	`386`	`});`
	`387`	`+`
	`388`	`+ // In case peer connections still somehow have jobs in workers,`
	`389`	`+ // only clear connections up after clearing workers.`
	`390`	`+ mNetworkThread = nullptr;`
	`391`	`+ mWorkerThread = nullptr;`
	`392`	`+ mSignalingThread = nullptr;`
	`393`	`+`
	`394`	`+ mPeerConnections.clear();`
`389`	`395`	`webrtc::LogMessage::RemoveLogToStream(mLogSink);`
`390`	`396`	`}`
`391`	`397`
`@@ -766,6 +772,7 @@ void LLWebRTCImpl::freePeerConnection(LLWebRTCPeerConnectionInterface* peer_conn`
`766`	`772`	`std::find(mPeerConnections.begin(), mPeerConnections.end(), peer_connection);`
`767`	`773`	`if (it != mPeerConnections.end())`
`768`	`774`	`{`
	`775`	`+ // Todo: make sure conection had no jobs in workers`
`769`	`776`	`mPeerConnections.erase(it);`
`770`	`777`	`if (mPeerConnections.empty())`
`771`	`778`	`{`
`@@ -785,14 +792,19 @@ LLWebRTCPeerConnectionImpl::LLWebRTCPeerConnectionImpl() :`
`785`	`792`	`mWebRTCImpl(nullptr),`
`786`	`793`	`mPeerConnection(nullptr),`
`787`	`794`	`mMute(MUTE_INITIAL),`
`788`		`- mAnswerReceived(false)`
	`795`	`+ mAnswerReceived(false),`
	`796`	`+ mPendingJobs(0)`
`789`	`797`	`{`
`790`	`798`	`}`
`791`	`799`
`792`	`800`	`LLWebRTCPeerConnectionImpl::~LLWebRTCPeerConnectionImpl()`
`793`	`801`	`{`
`794`	`802`	`mSignalingObserverList.clear();`
`795`	`803`	`mDataObserverList.clear();`
	`804`	`+ if (mPendingJobs > 0)`
	`805`	`+ {`
	`806`	`+ RTC_LOG(LS_ERROR) << __FUNCTION__ << "Destroying a connection that has " << std::to_string(mPendingJobs) << " unfinished jobs that might cause workers to crash";`
	`807`	`+ }`
`796`	`808`	`}`
`797`	`809`
`798`	`810`	`//`
`@@ -804,8 +816,10 @@ void LLWebRTCPeerConnectionImpl::init(LLWebRTCImpl * webrtc_impl)`
`804`	`816`	`mWebRTCImpl = webrtc_impl;`
`805`	`817`	`mPeerConnectionFactory = mWebRTCImpl->getPeerConnectionFactory();`
`806`	`818`	`}`
	`819`	`+`
`807`	`820`	`void LLWebRTCPeerConnectionImpl::terminate()`
`808`	`821`	`{`
	`822`	`+ mPendingJobs++;`
`809`	`823`	`mWebRTCImpl->PostSignalingTask(`
`810`	`824`	`[this]()`
`811`	`825`	`{`
`@@ -848,7 +862,9 @@ void LLWebRTCPeerConnectionImpl::terminate()`
`848`	`862`	`observer->OnPeerConnectionClosed();`
`849`	`863`	`}`
`850`	`864`	`}`
	`865`	`+ mPendingJobs--;`
`851`	`866`	`});`
	`867`	`+ mPeerConnectionFactory.release();`
`852`	`868`	`}`
`853`	`869`
`854`	`870`	`void LLWebRTCPeerConnectionImpl::setSignalingObserver(LLWebRTCSignalingObserver *observer) { mSignalingObserverList.emplace_back(observer); }`
`@@ -869,6 +885,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection(const LLWebRTCPeerConnecti`
`869`	`885`	`RTC_DCHECK(!mPeerConnection);`
`870`	`886`	`mAnswerReceived = false;`
`871`	`887`
	`888`	`+ mPendingJobs++;`
`872`	`889`	`mWebRTCImpl->PostSignalingTask(`
`873`	`890`	`[this,options]()`
`874`	`891`	`{`
`@@ -902,6 +919,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection(const LLWebRTCPeerConnecti`
`902`	`919`	`{`
`903`	`920`	`observer->OnRenegotiationNeeded();`
`904`	`921`	`}`
	`922`	`+ mPendingJobs--;`
`905`	`923`	`return;`
`906`	`924`	`}`
`907`	`925`
`@@ -964,6 +982,7 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection(const LLWebRTCPeerConnecti`
`964`	`982`
`965`	`983`	`webrtc::PeerConnectionInterface::RTCOfferAnswerOptions offerOptions;`
`966`	`984`	`mPeerConnection->CreateOffer(this, offerOptions);`
	`985`	`+ mPendingJobs--;`
`967`	`986`	`});`
`968`	`987`
`969`	`988`	`return true;`
`@@ -1006,6 +1025,7 @@ void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp)`
`1006`	`1025`	`{`
`1007`	`1026`	`RTC_LOG(LS_INFO) << __FUNCTION__ << " Remote SDP: " << sdp;`
`1008`	`1027`
	`1028`	`+ mPendingJobs++;`
`1009`	`1029`	`mWebRTCImpl->PostSignalingTask(`
`1010`	`1030`	`[this, sdp]()`
`1011`	`1031`	`{`
`@@ -1015,6 +1035,7 @@ void LLWebRTCPeerConnectionImpl::AnswerAvailable(const std::string &sdp)`
`1015`	`1035`	`mPeerConnection->SetRemoteDescription(webrtc::CreateSessionDescription(webrtc::SdpType::kAnswer, sdp),`
`1016`	`1036`	`webrtc::scoped_refptr<webrtc::SetRemoteDescriptionObserverInterface>(this));`
`1017`	`1037`	`}`
	`1038`	`+ mPendingJobs--;`
`1018`	`1039`	`});`
`1019`	`1040`	`}`
`1020`	`1041`
`@@ -1037,6 +1058,7 @@ void LLWebRTCPeerConnectionImpl::setMute(bool mute)`
`1037`	`1058`	`mMute = new_state;`
`1038`	`1059`
`1039`	`1060`
	`1061`	`+ mPendingJobs++;`
`1040`	`1062`	`mWebRTCImpl->PostSignalingTask(`
`1041`	`1063`	`[this, force_reset, enable]()`
`1042`	`1064`	`{`
`@@ -1060,6 +1082,7 @@ void LLWebRTCPeerConnectionImpl::setMute(bool mute)`
`1060`	`1082`	`track->set_enabled(enable);`
`1061`	`1083`	`}`
`1062`	`1084`	`}`
	`1085`	`+ mPendingJobs--;`
`1063`	`1086`	`}`
`1064`	`1087`	`});`
`1065`	`1088`	`}`
`@@ -1081,6 +1104,7 @@ void LLWebRTCPeerConnectionImpl::resetMute()`
`1081`	`1104`
`1082`	`1105`	`void LLWebRTCPeerConnectionImpl::setReceiveVolume(float volume)`
`1083`	`1106`	`{`
	`1107`	`+ mPendingJobs++;`
`1084`	`1108`	`mWebRTCImpl->PostSignalingTask(`
`1085`	`1109`	`[this, volume]()`
`1086`	`1110`	`{`
`@@ -1099,11 +1123,13 @@ void LLWebRTCPeerConnectionImpl::setReceiveVolume(float volume)`
`1099`	`1123`	`}`
`1100`	`1124`	`}`
`1101`	`1125`	`}`
	`1126`	`+ mPendingJobs--;`
`1102`	`1127`	`});`
`1103`	`1128`	`}`
`1104`	`1129`
`1105`	`1130`	`void LLWebRTCPeerConnectionImpl::setSendVolume(float volume)`
`1106`	`1131`	`{`
	`1132`	`+ mPendingJobs++;`
`1107`	`1133`	`mWebRTCImpl->PostSignalingTask(`
`1108`	`1134`	`[this, volume]()`
`1109`	`1135`	`{`
`@@ -1114,6 +1140,7 @@ void LLWebRTCPeerConnectionImpl::setSendVolume(float volume)`
`1114`	`1140`	`track->GetSource()->SetVolume(volume*5.0);`
`1115`	`1141`	`}`
`1116`	`1142`	`}`
	`1143`	`+ mPendingJobs--;`
`1117`	`1144`	`});`
`1118`	`1145`	`}`
`1119`	`1146`
`@@ -1190,11 +1217,13 @@ void LLWebRTCPeerConnectionImpl::OnConnectionChange(webrtc::PeerConnectionInterf`
`1190`	`1217`	`{`
`1191`	`1218`	`case webrtc::PeerConnectionInterface::PeerConnectionState::kConnected:`
`1192`	`1219`	`{`
	`1220`	`+ mPendingJobs++;`
`1193`	`1221`	`mWebRTCImpl->PostWorkerTask([this]() {`
`1194`	`1222`	`for (auto &observer : mSignalingObserverList)`
`1195`	`1223`	`{`
`1196`	`1224`	`observer->OnAudioEstablished(this);`
`1197`	`1225`	`}`
	`1226`	`+ mPendingJobs--;`
`1198`	`1227`	`});`
`1199`	`1228`	`break;`
`1200`	`1229`	`}`
`@@ -1452,11 +1481,13 @@ void LLWebRTCPeerConnectionImpl::sendData(const std::string& data, bool binary)`
`1452`	`1481`	`{`
`1453`	`1482`	`webrtc::CopyOnWriteBuffer cowBuffer(data.data(), data.length());`
`1454`	`1483`	`webrtc::DataBuffer buffer(cowBuffer, binary);`
	`1484`	`+ mPendingJobs++;`
`1455`	`1485`	`mWebRTCImpl->PostNetworkTask([this, buffer]() {`
`1456`	`1486`	`if (mDataChannel)`
`1457`	`1487`	`{`
`1458`	`1488`	`mDataChannel->Send(buffer);`
`1459`	`1489`	`}`
	`1490`	`+ mPendingJobs--;`
`1460`	`1491`	`});`
`1461`	`1492`	`}`
`1462`	`1493`	`}`
Original file line number	Diff line number	Diff line change
`@@ -422,6 +422,9 @@ class LLWebRTCImpl : public LLWebRTCDeviceInterface, public webrtc::AudioDeviceO`
`422`	`422`	`~LLWebRTCImpl()`
`423`	`423`	`{`
`424`	`424`	`delete mLogSink;`
	`425`	`+`
	`426`	`+ // Explicit cleanup for the sake of debugging and crash stacks`
	`427`	`+ mPeerCustomProcessor = nullptr;`
`425`	`428`	`}`
`426`	`429`
`427`	`430`	`void init();`
`@@ -669,6 +672,8 @@ class LLWebRTCPeerConnectionImpl : public LLWebRTCPeerConnectionInterface,`
`669`	`672`	`// data`
`670`	`673`	`std::vector<LLWebRTCDataObserver *> mDataObserverList;`
`671`	`674`	`webrtc::scoped_refptr<webrtc::DataChannelInterface> mDataChannel;`
	`675`	`+`
	`676`	`+ std::atomic<int> mPendingJobs;`
`672`	`677`	`};`
`673`	`678`
`674`	`679`	`}`