Boca: Plumb CRD AudioPackets to the WebUI page handler

This change sets up the necessary plumbing to forward audio packets from
the Boca CRD client stack to the Boca WebUI's page handler.

Bug: 450986461
Change-Id: Ic17140afd49e0638ca39912ff2180a95cfff9dfd
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/7046787
Reviewed-by: Ahmed Nasr <[email protected]>
Commit-Queue: Gavin Williams <[email protected]>
Cr-Commit-Position: refs/heads/main@{#1532388}

Author: Gavin Williams

ash/webui/boca_receiver_app_ui/BUILD.gn

@@ -37,6 +37,7 @@ static_library("boca_receiver_app_ui") {
     "//content/public/common",
     "//google_apis/common",
     "//net/traffic_annotation",
+    "//remoting/proto",
     "//ui/webui",
   ]
 }

ash/webui/boca_receiver_app_ui/DEPS

@@ -1,5 +1,6 @@
 include_rules = [
   "+google_apis/common",
+  "+remoting/proto/audio.pb.h",
   "+third_party/skia/include/core/SkBitmap.h",
   "+third_party/webrtc/modules/desktop_capture/desktop_frame.h",
 ]

ash/webui/boca_receiver_app_ui/boca_receiver_untrusted_page_handler.cc

@@ -31,6 +31,7 @@
 #include "google_apis/common/request_sender.h"
 #include "mojo/public/cpp/bindings/pending_remote.h"
 #include "net/traffic_annotation/network_traffic_annotation.h"
+#include "remoting/proto/audio.pb.h"
 #include "third_party/skia/include/core/SkBitmap.h"
 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
 
@@ -283,6 +284,9 @@ void BocaReceiverUntrustedPageHandler::MaybeStartConnection(
                      weak_ptr_factory_.GetWeakPtr()),
       base::BindRepeating(&BocaReceiverUntrustedPageHandler::OnCrdFrameReceived,
                           weak_ptr_factory_.GetWeakPtr()),
+      base::BindRepeating(
+          &BocaReceiverUntrustedPageHandler::OnCrdAudioPacketReceived,
+          weak_ptr_factory_.GetWeakPtr()),
       base::BindRepeating(
           &BocaReceiverUntrustedPageHandler::OnCrdConnectionStateUpdated,
           weak_ptr_factory_.GetWeakPtr()));
@@ -329,6 +333,11 @@ void BocaReceiverUntrustedPageHandler::OnCrdFrameReceived(
                    ::boca::ReceiverConnectionState::CONNECTED);
 }
 
+void BocaReceiverUntrustedPageHandler::OnCrdAudioPacketReceived(
+    std::unique_ptr<remoting::AudioPacket> packet) {
+  // TODO(crbug.com/450986461): Transform packet and pass it to UI through mojo.
+}
+
 void BocaReceiverUntrustedPageHandler::OnCrdConnectionStateUpdated(
     boca::CrdConnectionState state) {
   switch (state) {

ash/webui/boca_receiver_app_ui/boca_receiver_untrusted_page_handler.h

@@ -39,6 +39,10 @@ namespace net {
 struct NetworkTrafficAnnotationTag;
 }  // namespace net
 
+namespace remoting {
+class AudioPacket;
+}  // namespace remoting
+
 namespace webrtc {
 class DesktopFrame;
 }  // namespace webrtc
@@ -104,6 +108,7 @@ class BocaReceiverUntrustedPageHandler
   void OnCrdSessionEnded();
   void OnCrdFrameReceived(SkBitmap bitmap,
                           std::unique_ptr<webrtc::DesktopFrame>);
+  void OnCrdAudioPacketReceived(std::unique_ptr<remoting::AudioPacket> packet);
   void OnCrdConnectionStateUpdated(boca::CrdConnectionState state);
 
   mojo::Remote<mojom::UntrustedPage> page_;

ash/webui/boca_receiver_app_ui/boca_receiver_untrusted_page_handler_unittest.cc

@@ -27,6 +27,7 @@
 #include "chromeos/ash/components/boca/receiver/receiver_handler_delegate.h"
 #include "chromeos/ash/components/boca/receiver/register_receiver_request.h"
 #include "chromeos/ash/components/boca/receiver/update_kiosk_receiver_state_request.h"
+#include "chromeos/ash/components/boca/spotlight/spotlight_audio_stream_consumer.h"
 #include "chromeos/ash/components/boca/spotlight/spotlight_remoting_client_manager.h"
 #include "chromeos/ash/components/boca/util.h"
 #include "google_apis/common/dummy_auth_service.h"
@@ -163,6 +164,8 @@ class MockSpotlightRemotingClientManager
                base::OnceClosure crd_session_ended_callback,
                boca::SpotlightFrameConsumer::FrameReceivedCallback
                    frame_received_callback,
+               boca::SpotlightAudioStreamConsumer::AudioPacketReceivedCallback
+                   audio_packet_received_callback,
                boca::SpotlightCrdStateUpdatedCallback status_updated_callback),
               (override));
 
@@ -354,7 +357,7 @@ TEST_F(BocaReceiverUntrustedPageHandlerTest, StartRequestedNoCodeThenWithCode) {
   auto remoting_client =
       std::make_unique<NiceMock<MockSpotlightRemotingClientManager>>();
   EXPECT_CALL(*remoting_client,
-              StartCrdClient(std::string(kConnectionCode), _, _, _))
+              StartCrdClient(std::string(kConnectionCode), _, _, _, _))
       .Times(1);
   EXPECT_CALL(handler_delegate_, CreateRemotingClientManager)
       .WillOnce(Return(ByMove(std::move(remoting_client))));
@@ -385,7 +388,7 @@ TEST_F(BocaReceiverUntrustedPageHandlerTest,
   auto remoting_client =
       std::make_unique<NiceMock<MockSpotlightRemotingClientManager>>();
   EXPECT_CALL(*remoting_client,
-              StartCrdClient(std::string(kConnectionCode), _, _, _))
+              StartCrdClient(std::string(kConnectionCode), _, _, _, _))
       .Times(1);
   EXPECT_CALL(handler_delegate_, CreateRemotingClientManager)
       .WillOnce(Return(ByMove(std::move(remoting_client))));
@@ -407,11 +410,11 @@ TEST_F(BocaReceiverUntrustedPageHandlerTest, FrameReceived) {
   auto remoting_client =
       std::make_unique<NiceMock<MockSpotlightRemotingClientManager>>();
   EXPECT_CALL(*remoting_client,
-              StartCrdClient(std::string(kConnectionCode), _, _, _))
-      .WillOnce(
-          [&frame_received_cb](auto, auto, auto frame_received_cb_param, auto) {
-            frame_received_cb = std::move(frame_received_cb_param);
-          });
+              StartCrdClient(std::string(kConnectionCode), _, _, _, _))
+      .WillOnce([&frame_received_cb](auto, auto, auto frame_received_cb_param,
+                                     auto, auto) {
+        frame_received_cb = std::move(frame_received_cb_param);
+      });
   EXPECT_CALL(handler_delegate_, CreateRemotingClientManager)
       .WillOnce(Return(ByMove(std::move(remoting_client))));
   WaitForTokenUpload();
@@ -461,10 +464,10 @@ TEST_F(BocaReceiverUntrustedPageHandlerTest, CrdSessionEnded) {
   auto remoting_client =
       std::make_unique<NiceMock<MockSpotlightRemotingClientManager>>();
   EXPECT_CALL(*remoting_client,
-              StartCrdClient(std::string(kConnectionCode), _, _, _))
+              StartCrdClient(std::string(kConnectionCode), _, _, _, _))
       .WillOnce([&session_ended_cb](auto,
                                     base::OnceClosure session_ended_cb_param,
-                                    auto, auto) {
+                                    auto, auto, auto) {
         session_ended_cb = std::move(session_ended_cb_param);
       });
   EXPECT_CALL(handler_delegate_, CreateRemotingClientManager)
@@ -691,11 +694,11 @@ TEST_P(BocaReceiverUntrustedPageHandlerCrdStateTest,
       std::make_unique<NiceMock<MockSpotlightRemotingClientManager>>();
   auto* remoting_client_ptr = remoting_client.get();
   EXPECT_CALL(*remoting_client,
-              StartCrdClient(std::string(kConnectionCode), _, _, _))
-      .WillOnce(
-          [&state_updated_cb](auto, auto, auto, auto state_updated_cb_param) {
-            state_updated_cb = std::move(state_updated_cb_param);
-          });
+              StartCrdClient(std::string(kConnectionCode), _, _, _, _))
+      .WillOnce([&state_updated_cb](auto, auto, auto, auto,
+                                    auto state_updated_cb_param) {
+        state_updated_cb = std::move(state_updated_cb_param);
+      });
   EXPECT_CALL(handler_delegate_, CreateRemotingClientManager)
       .WillOnce(Return(ByMove(std::move(remoting_client))));
   WaitForTokenUpload();

chromeos/ash/components/boca/boca_session_manager.cc

@@ -46,6 +46,7 @@
 #include "components/session_manager/core/session_manager.h"
 #include "components/user_manager/user_manager.h"
 #include "google_apis/common/api_error_codes.h"
+#include "remoting/proto/audio.pb.h"
 #include "services/network/public/cpp/shared_url_loader_factory.h"
 #include "ui/base/l10n/l10n_util.h"
 #include "ui/message_center/message_center.h"
@@ -422,7 +423,9 @@ void BocaSessionManager::StartCrdClient(
 
   remoting_client_manager_->StartCrdClient(
       crd_connection_code, std::move(done_callback),
-      std::move(frame_received_callback), std::move(crd_state_callback));
+      std::move(frame_received_callback),
+      /* audio_packet_received_callback= */ base::DoNothing(),
+      std::move(crd_state_callback));
 }
 
 void BocaSessionManager::EndSpotlightSession(

chromeos/ash/components/boca/spotlight/remoting_client_io_proxy.cc

@@ -28,9 +28,13 @@ RemotingClientIOProxyImpl::RemotingClientIOProxyImpl(
     std::unique_ptr<network::PendingSharedURLLoaderFactory>
         pending_url_loader_factory,
     SpotlightFrameConsumer::FrameReceivedCallback frame_received_callback,
+    SpotlightAudioStreamConsumer::AudioPacketReceivedCallback
+        audio_packet_received_callback,
     SpotlightCrdStateUpdatedCallback status_updated_callback)
     : pending_url_loader_factory_(std::move(pending_url_loader_factory)),
       frame_received_callback_(std::move(frame_received_callback)),
+      audio_packet_received_callback_(
+          std::move(audio_packet_received_callback)),
       status_updated_callback_(std::move(status_updated_callback)) {
   DETACH_FROM_SEQUENCE(sequence_checker_);
 }
@@ -138,7 +142,7 @@ void RemotingClientIOProxyImpl::OnFrameReceived(
 void RemotingClientIOProxyImpl::OnAudioPacketReceived(
     std::unique_ptr<remoting::AudioPacket> packet) {
   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
-  // TODO(crbug.com/450986461): Send `packet` to a callback.
+  audio_packet_received_callback_.Run(std::move(packet));
 }
 
 void RemotingClientIOProxyImpl::ResetRemotingClient(

chromeos/ash/components/boca/spotlight/remoting_client_io_proxy.h

@@ -14,6 +14,7 @@
 #include "base/sequence_checker.h"
 #include "base/task/sequenced_task_runner.h"
 #include "base/task/single_thread_task_runner.h"
+#include "chromeos/ash/components/boca/spotlight/spotlight_audio_stream_consumer.h"
 #include "chromeos/ash/components/boca/spotlight/spotlight_constants.h"
 #include "chromeos/ash/components/boca/spotlight/spotlight_frame_consumer.h"
 #include "remoting/client/common/client_status_observer.h"
@@ -63,6 +64,8 @@ class RemotingClientIOProxyImpl : public RemotingClientIOProxy,
       std::unique_ptr<network::PendingSharedURLLoaderFactory>
           pending_url_loader_factory,
       SpotlightFrameConsumer::FrameReceivedCallback frame_received_callback,
+      SpotlightAudioStreamConsumer::AudioPacketReceivedCallback
+          audio_packet_received_callback,
       SpotlightCrdStateUpdatedCallback status_updated_callback);
   RemotingClientIOProxyImpl(const RemotingClientIOProxyImpl&) = delete;
   RemotingClientIOProxyImpl& operator=(const RemotingClientIOProxyImpl&) =
@@ -113,6 +116,8 @@ class RemotingClientIOProxyImpl : public RemotingClientIOProxy,
   base::OnceClosure crd_session_ended_callback_;
   // Callback for receiving a completed frame from `SpotlightFrameConsumer`.
   SpotlightFrameConsumer::FrameReceivedCallback frame_received_callback_;
+  SpotlightAudioStreamConsumer::AudioPacketReceivedCallback
+      audio_packet_received_callback_;
   // Callback for `CrdConnectionState` updates.
   SpotlightCrdStateUpdatedCallback status_updated_callback_;
   std::unique_ptr<SpotlightFrameConsumer> frame_consumer_;

chromeos/ash/components/boca/spotlight/spotlight_remoting_client_manager.cc

@@ -27,6 +27,7 @@
 #include "chromeos/ash/components/boca/spotlight/remoting_client_io_proxy.h"
 #include "chromeos/ash/components/boca/spotlight/spotlight_constants.h"
 #include "chromeos/ash/components/boca/spotlight/spotlight_oauth_token_fetcher.h"
+#include "remoting/proto/audio.pb.h"
 #include "services/network/public/cpp/shared_url_loader_factory.h"
 #include "third_party/skia/include/core/SkBitmap.h"
 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
@@ -51,6 +52,9 @@ SpotlightRemotingClientManagerImpl::SpotlightRemotingClientManagerImpl(
           base::BindPostTaskToCurrentDefault(base::BindRepeating(
               &SpotlightRemotingClientManagerImpl::HandleFrameReceived,
               weak_factory_.GetWeakPtr())),
+          base::BindPostTaskToCurrentDefault(base::BindRepeating(
+              &SpotlightRemotingClientManagerImpl::HandleAudioPacketReceived,
+              weak_factory_.GetWeakPtr())),
           base::BindPostTaskToCurrentDefault(base::BindRepeating(
               &SpotlightRemotingClientManagerImpl::UpdateState,
               weak_factory_.GetWeakPtr()))));
@@ -69,6 +73,8 @@ void SpotlightRemotingClientManagerImpl::StartCrdClient(
     std::string crd_connection_code,
     base::OnceClosure crd_session_ended_callback,
     SpotlightFrameConsumer::FrameReceivedCallback frame_received_callback,
+    SpotlightAudioStreamConsumer::AudioPacketReceivedCallback
+        audio_packet_received_callback,
     SpotlightCrdStateUpdatedCallback status_updated_callback) {
   CHECK(ash::features::IsBocaSpotlightRobotRequesterEnabled());
   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
@@ -91,6 +97,7 @@ void SpotlightRemotingClientManagerImpl::StartCrdClient(
 
   crd_session_ended_callback_ = std::move(crd_session_ended_callback);
   frame_received_callback_ = std::move(frame_received_callback);
+  audio_packet_received_callback_ = std::move(audio_packet_received_callback);
   status_updated_callback_ = std::move(status_updated_callback);
 
   token_fetcher_->Start((base::BindOnce(
@@ -122,9 +129,12 @@ SpotlightRemotingClientManagerImpl::CreateRemotingIOProxy(
     std::unique_ptr<network::PendingSharedURLLoaderFactory>
         pending_url_loader_factory,
     SpotlightFrameConsumer::FrameReceivedCallback frame_received_callback,
+    SpotlightAudioStreamConsumer::AudioPacketReceivedCallback
+        audio_packet_received_callback,
     SpotlightCrdStateUpdatedCallback status_updated_callback) {
   return std::make_unique<RemotingClientIOProxyImpl>(
       std::move(pending_url_loader_factory), std::move(frame_received_callback),
+      std::move(audio_packet_received_callback),
       std::move(status_updated_callback));
 }
 
@@ -183,10 +193,20 @@ void SpotlightRemotingClientManagerImpl::HandleFrameReceived(
   frame_received_callback_.Run(std::move(bitmap), std::move(frame));
 }
 
+void SpotlightRemotingClientManagerImpl::HandleAudioPacketReceived(
+    std::unique_ptr<remoting::AudioPacket> packet) {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+  if (!audio_packet_received_callback_) {
+    return;
+  }
+  audio_packet_received_callback_.Run(std::move(packet));
+}
+
 void SpotlightRemotingClientManagerImpl::Reset() {
   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
   crd_session_ended_callback_.Reset();
   frame_received_callback_.Reset();
+  audio_packet_received_callback_.Reset();
   status_updated_callback_.Reset();
   frame_timeout_timer_.Stop();
   session_in_progress_ = false;

chromeos/ash/components/boca/spotlight/spotlight_remoting_client_manager.h

@@ -17,6 +17,7 @@
 #include "base/threading/thread.h"
 #include "base/timer/timer.h"
 #include "chromeos/ash/components/boca/spotlight/remoting_client_io_proxy.h"
+#include "chromeos/ash/components/boca/spotlight/spotlight_audio_stream_consumer.h"
 #include "chromeos/ash/components/boca/spotlight/spotlight_constants.h"
 #include "chromeos/ash/components/boca/spotlight/spotlight_frame_consumer.h"
 #include "chromeos/ash/components/boca/spotlight/spotlight_oauth_token_fetcher.h"
@@ -27,6 +28,7 @@ class SharedURLLoaderFactory;
 }  // namespace network
 
 namespace remoting {
+class AudioPacket;
 class RemotingClient;
 }  // namespace remoting
 
@@ -49,6 +51,8 @@ class SpotlightRemotingClientManager {
       std::string crd_connection_code,
       base::OnceClosure crd_session_ended_callback,
       SpotlightFrameConsumer::FrameReceivedCallback frame_received_callback,
+      SpotlightAudioStreamConsumer::AudioPacketReceivedCallback
+          audio_packet_received_callback,
       SpotlightCrdStateUpdatedCallback status_updated_callback) = 0;
 
   virtual void StopCrdClient(base::OnceClosure on_stopped_callback) = 0;
@@ -66,6 +70,7 @@ class SpotlightRemotingClientManagerImpl
       base::RepeatingCallback<std::unique_ptr<RemotingClientIOProxy>(
           std::unique_ptr<network::PendingSharedURLLoaderFactory>,
           SpotlightFrameConsumer::FrameReceivedCallback,
+          SpotlightAudioStreamConsumer::AudioPacketReceivedCallback,
           SpotlightCrdStateUpdatedCallback)>;
 
   SpotlightRemotingClientManagerImpl(
@@ -86,6 +91,8 @@ class SpotlightRemotingClientManagerImpl
       std::string crd_connection_code,
       base::OnceClosure crd_session_ended_callback,
       SpotlightFrameConsumer::FrameReceivedCallback frame_received_callback,
+      SpotlightAudioStreamConsumer::AudioPacketReceivedCallback
+          audio_packet_received_callback,
       SpotlightCrdStateUpdatedCallback status_updated_callback) override;
   // Forwards the request to stop the crd client to the
   // `remoting_client_io_proxy_`.
@@ -100,6 +107,8 @@ class SpotlightRemotingClientManagerImpl
       std::unique_ptr<network::PendingSharedURLLoaderFactory>
           pending_url_loader_factory,
       SpotlightFrameConsumer::FrameReceivedCallback frame_received_callback,
+      SpotlightAudioStreamConsumer::AudioPacketReceivedCallback
+          audio_packet_received_callback,
       SpotlightCrdStateUpdatedCallback status_updated_callback);
 
   // Receives the OAuth token on the main/UI thread and calls the
@@ -117,6 +126,9 @@ class SpotlightRemotingClientManagerImpl
   void HandleFrameReceived(SkBitmap bitmap,
                            std::unique_ptr<webrtc::DesktopFrame> frame);
 
+  // Forwards the audio packet to the `audio_packet_received_callback_`;
+  void HandleAudioPacketReceived(std::unique_ptr<remoting::AudioPacket> packet);
+
   void Reset();
 
   SEQUENCE_CHECKER(sequence_checker_);
@@ -127,6 +139,8 @@ class SpotlightRemotingClientManagerImpl
   base::Thread io_thread_;
   base::OnceClosure crd_session_ended_callback_;
   SpotlightFrameConsumer::FrameReceivedCallback frame_received_callback_;
+  SpotlightAudioStreamConsumer::AudioPacketReceivedCallback
+      audio_packet_received_callback_;
   SpotlightCrdStateUpdatedCallback status_updated_callback_;
   std::unique_ptr<SpotlightOAuthTokenFetcher> token_fetcher_;
   // The `SpotlightRemotingClientManagerImpl` is owned by the main/UI thread