diff --git a/src/game/client/game_controls/basemodel_panel.cpp b/src/game/client/game_controls/basemodel_panel.cpp
index 0fed0085787..787a05f60c4 100644
--- a/src/game/client/game_controls/basemodel_panel.cpp
+++ b/src/game/client/game_controls/basemodel_panel.cpp
@@ -516,6 +516,10 @@ void CBaseModelPanel::OnMouseReleased( vgui::MouseCode code )
 	if ( !m_bAllowRotation && !m_bAllowPitch )
 		return;
 
+	// Only accept manipulation if we were capturing the mouse
+	if ( input()->GetMouseCapture() != GetVPanel() )
+		return;
+
 	EnableMouseCapture( false );
 	m_bMousePressed = false;