More renderer optmizations

Author: taste1981

talk/owt/sdk/base/cameravideocapturer.cc

@@ -36,7 +36,7 @@ void CameraVideoCapturer::OnFrame(const webrtc::VideoFrame& frame) {
     return;
   }
 
-  if (out_height != frame.height() || out_width != frame.width()) {
+  if (false/*out_height != frame.height() || out_width != frame.width()*/) {
     // Video adapter has requested a down-scale. Allocate a new buffer and
     // return scaled version.
     rtc::scoped_refptr<webrtc::I420Buffer> scaled_buffer =

talk/owt/sdk/base/connectionstats.cc

@@ -694,6 +694,7 @@ RTCInboundRTPStreamStats::RTCInboundRTPStreamStats(
     double gap_loss_rate,
     double gap_discard_rate,
     uint32_t frames_decoded,
+    uint32_t frames_rendered,
     uint32_t key_frames_decoded,
     double total_decode_time,
     double total_inter_frame_delay,
@@ -724,6 +725,7 @@ RTCInboundRTPStreamStats::RTCInboundRTPStreamStats(
       gap_loss_rate(gap_loss_rate),
       gap_discard_rate(gap_discard_rate),
       frames_decoded(frames_decoded),
+      frames_rendered(frames_rendered),
       key_frames_decoded(key_frames_decoded),
       total_decode_time(total_decode_time),
       total_inter_frame_delay(total_inter_frame_delay),
@@ -755,6 +757,7 @@ RTCInboundRTPStreamStats::RTCInboundRTPStreamStats(
       gap_loss_rate(other.gap_loss_rate),
       gap_discard_rate(other.gap_discard_rate),
       frames_decoded(other.frames_decoded),
+      frames_rendered(other.frames_rendered),
       key_frames_decoded(other.key_frames_decoded),
       total_decode_time(other.total_decode_time),
       total_inter_frame_delay(other.total_inter_frame_delay),

talk/owt/sdk/base/functionalobserver.cc

@@ -612,6 +612,8 @@ void FunctionalStandardRTCStatsCollectorCallback::OnStatsDelivered(
                                          0),
               OWT_STATS_VALUE_OR_DEFAULT(webrtc_stats, frames_decoded, uint32_t,
                                          0),
+              OWT_STATS_VALUE_OR_DEFAULT(webrtc_stats, frames_rendered, uint32_t,
+                                         0),
               OWT_STATS_VALUE_OR_DEFAULT(webrtc_stats, key_frames_decoded,
                                          uint32_t, 0),
               OWT_STATS_VALUE_OR_DEFAULT(webrtc_stats, total_decode_time,

talk/owt/sdk/base/peerconnectionchannel.cc

@@ -4,6 +4,7 @@
 #include "talk/owt/sdk/base/peerconnectionchannel.h"
 #include <vector>
 #include "talk/owt/sdk/base/sdputils.h"
+#include "webrtc/api/peer_connection_interface.h"
 #include "webrtc/rtc_base/logging.h"
 #include "webrtc/rtc_base/thread.h"
 using namespace rtc;
@@ -30,6 +31,7 @@ bool PeerConnectionChannel::InitializePeerConnection() {
   configuration_.enable_dtls_srtp = true;
   configuration_.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan;
   configuration_.media_config.enable_dscp = true;
+  configuration_.bundle_policy = webrtc::PeerConnectionInterface::BundlePolicy::kBundlePolicyMaxBundle;
   peer_connection_ =
       (factory_->CreatePeerConnection(configuration_, this)).get();
   if (!peer_connection_.get()) {

talk/owt/sdk/base/win/msdkvideodecoder.cc

@@ -393,6 +393,7 @@ int32_t MSDKVideoDecoder::Decode(
           decoded_frame.set_ntp_time_ms(inputImage.ntp_time_ms_);
           decoded_frame.set_timestamp(inputImage.Timestamp());
           callback_->Decoded(decoded_frame);
+          RTC_LOG(LS_ERROR) << "Succeed decoding a frame.";
         }
       }
     } else if (MFX_ERR_MORE_DATA == sts) {

talk/owt/sdk/base/win/msdkvideodecoderfactory.cc

@@ -86,7 +86,7 @@ std::unique_ptr<webrtc::VideoDecoder> MSDKVideoDecoderFactory::CreateVideoDecode
     }
 #endif
   }
-  if (absl::EqualsIgnoreCase(format.name, cricket::kVp9CodecName) /*&& !vp9_hw*/) {
+  if (absl::EqualsIgnoreCase(format.name, cricket::kVp9CodecName) && !vp9_hw) {
     return webrtc::VP9Decoder::Create();
   } else if (absl::EqualsIgnoreCase(format.name, cricket::kVp8CodecName) &&
              !vp8_hw) {

talk/owt/sdk/base/win/videorendererd3d11.cc

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #include "talk/owt/sdk/base/win/videorendererd3d11.h"
+#include <cstdio>
 #include <array>
 #include "rtc_base/logging.h"
 #include "talk/owt/sdk/base/nativehandlebuffer.h"
@@ -18,12 +19,13 @@ struct D3dCustomVertex {
   float u, v;
 };
 
+static int frame_count = 0;
 using namespace rtc;
 namespace owt {
 namespace base {
 
 WebrtcVideoRendererD3D11Impl::WebrtcVideoRendererD3D11Impl(HWND wnd)
-    : wnd_(wnd), width_(0), height_(0) {
+    : wnd_(wnd), width_(0), height_(0), clock_(Clock::GetRealTimeClock()) {
   CreateDXGIFactory(__uuidof(IDXGIFactory2), (void**)(&dxgi_factory_));
 }
 
@@ -50,6 +52,7 @@ void WebrtcVideoRendererD3D11Impl::OnFrame(
   uint16_t width = video_frame.video_frame_buffer()->width();
   uint16_t height = video_frame.video_frame_buffer()->height();
 
+  RTC_LOG(LS_ERROR) << "On frame." << width << "x" << height;
   if (width == 0 || height == 0)
     return;
 
@@ -549,6 +552,11 @@ void WebrtcVideoRendererD3D11Impl::RenderToBackbuffer(int array_slice) {
   DXGI_PRESENT_PARAMETERS parameters = {0};
   swap_chain_for_hwnd_->Present1(
       0, 0, &parameters);
+  int64_t current_time = clock_->TimeInMilliseconds();
+  frame_count++;
+  FILE* f = fopen("frames.txt", "a");
+  fprintf(f, "timestamp: %lld, frames drawn: %d\r\n", current_time, frame_count);
+  fclose(f);
 }
 
 // Helper method to initialize a shader resource view

talk/owt/sdk/base/win/videorendererd3d11.h

@@ -21,6 +21,7 @@
 #include <vector>
 #include "webrtc/api/video/video_frame.h"
 #include "webrtc/api/video/video_sink_interface.h"
+#include "webrtc/system_wrappers/include/clock.h"
 
 // Holds texture coordinate of the vertex
 typedef struct _TEXCOORD {
@@ -113,6 +114,7 @@ class WebrtcVideoRendererD3D11Impl
   rtc::scoped_refptr<IDirect3DTexture9> m_texture_;
   rtc::scoped_refptr<IDirect3DVertexBuffer9> m_vertex_buffer_;
   UINT views_count = 0;
+  webrtc::Clock* clock_;
 
 };
 }  // namespace base

talk/owt/sdk/include/cpp/owt/base/connectionstats.h

@@ -717,6 +717,7 @@ class RTCInboundRTPStreamStats final : public RTCRTPStreamStats {
                            double gap_loss_rate,
                            double gap_discard_rate,
                            uint32_t frames_decoded,
+                           uint32_t frames_rendered,
                            uint32_t key_frames_decoded,
                            double total_decode_time,
                            double total_inter_frame_delay,
@@ -760,6 +761,7 @@ class RTCInboundRTPStreamStats final : public RTCRTPStreamStats {
   // TODO: Collect and populate this value.
   double gap_discard_rate;
   uint32_t frames_decoded;
+  uint32_t frames_rendered;
   uint32_t key_frames_decoded;
   double total_decode_time;
   double total_inter_frame_delay;

talk/owt/sdk/p2p/p2pclient.cc

@@ -215,7 +215,8 @@ void P2PClient::OnSignalingMessage(const std::string& message,
       RTC_LOG(LS_WARNING) << "Non-existed chat cannot be stopped.";
       return;
     }
-  } else if (message.find("\"type\":\"offer\"") != std::string::npos) {
+  } 
+  else if (message.find("\"type\":\"offer\"") != std::string::npos) {
     auto pcc = GetPeerConnectionChannel(remote_id);
     if (pcc->HaveLocalOffer() && local_id_.compare(remote_id) > 0) {
       // Make the remote side as the publisher.
@@ -233,7 +234,8 @@ void P2PClient::OnSignalingMessage(const std::string& message,
       new_pcc->Publish(stream, success_callback, failure_callback);
       return;
     }
-  } else if (message.find("\"type\":\"chat-closed\"") != std::string::npos) {
+  } 
+  else if (message.find("\"type\":\"chat-closed\"") != std::string::npos) {
     int code = 0;
     std::string error = "";
     Json::Reader reader;