Enable TCAE information population on client stack. (#160)

Author: taste1981

api/task_queue/test/ptp_clock_sync.h

@@ -0,0 +1,57 @@
+// Copyright (C) <2021> Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef RTC_BASE_TIME_PTP_CLOCK_SYNC_H_
+#define RTC_BASE_TIME_PTP_CLOCK_SYNC_H_
+
+#include <windows.h>
+#include <cstdint>
+
+#define MICROSECONDS_FACTOR 1000000.0
+#define OFFSET_FACTOR 200000
+#define SERVER_FREQUENCY 0.09  // RTP/NTP timestamp runs at 90KHz clock
+
+// A Windows implementation for PTP using timestamp from RTP and local
+// timestamp. We may need to implement something like QueryPerformanceFrequency
+// for this to work on Linux platforms.
+namespace webrtc {
+class PTPClockSync {
+ public:
+  PTPClockSync()
+      : m_server_point(0), m_server_freq(0), m_client_point(0), m_last_ts(0) {
+    uint64_t freq; // Performance counter frequency in a second.
+    QueryPerformanceFrequency((LARGE_INTEGER*)&freq);
+    m_client_freq = (double)freq / MICROSECONDS_FACTOR;
+    m_server_freq = SERVER_FREQUENCY;
+  }
+  ~PTPClockSync() {}
+
+  void Sync(uint32_t ts, uint64_t tc) {
+    if (GetDuration(ts, tc) < 0 ||
+        (ts - m_last_ts) > OFFSET_FACTOR * m_server_freq) {
+      UpdateSync(ts, tc);
+    }
+    m_last_ts = ts;
+  }
+
+  double GetDuration(uint32_t ts, uint64_t tc) {
+    int ds = (int)(ts - m_server_point);
+    int dc = (int)(tc - m_client_point);
+    return (double)dc / m_client_freq - (double)ds / m_server_freq;
+  }
+
+ protected:
+  uint32_t m_server_point;
+  double m_server_freq;  // count per us
+  uint64_t m_client_point;
+  double m_client_freq;  // count per us
+  uint32_t m_last_ts;
+
+  void UpdateSync(uint32_t ts, uint64_t tc) {
+    m_client_point = tc;
+    m_server_point = ts;
+  }
+};
+}  // namespace webrtc
+#endif

api/video/encoded_image.h

@@ -198,6 +198,14 @@ class RTC_EXPORT EncodedImage {
     int64_t receive_finish_ms = 0;
   } timing_;
 
+#if defined(WEBRTC_WIN)
+  struct BWEStats {
+    double start_duration_ = 0;
+    double last_duration_ = 0;
+    int32_t packets_lost_ = 0;
+  }bwe_stats_;
+#endif
+
  private:
   size_t capacity() const { return encoded_data_ ? encoded_data_->size() : 0; }
 

modules/video_coding/frame_object.h

@@ -46,12 +46,17 @@ class RtpFrameObject : public EncodedFrame {
   int64_t RenderTime() const override;
   bool delayed_by_retransmission() const override;
   const RTPVideoHeader& GetRtpVideoHeader() const;
-
-  uint8_t* mutable_data() { return image_buffer_->data(); }
-
+  const FrameMarking& GetFrameMarking() const;
+#if defined(WEBRTC_WIN)
+  void SetBWETiming(double start_duration,
+      double last_duration,
+      int32_t packets_lost) {
+    bwe_stats_.start_duration_ = start_duration;
+    bwe_stats_.last_duration_ = last_duration;
+    bwe_stats_.packets_lost_ = packets_lost;
+  }
+#endif
  private:
-  // Reference for mutable access.
-  rtc::scoped_refptr<EncodedImageBuffer> image_buffer_;
   RTPVideoHeader rtp_video_header_;
   VideoCodecType codec_type_;
   uint16_t first_seq_num_;

modules/video_coding/packet_buffer.cc

@@ -86,6 +86,10 @@ PacketBuffer::InsertResult PacketBuffer::InsertPacket(
     first_seq_num_ = seq_num;
   }
 
+#if defined(WEBRTC_WIN)
+  QueryPerformanceCounter((LARGE_INTEGER*)&packet->time_ticks);
+#endif
+
   if (buffer_[index] != nullptr) {
     // Duplicate packet, just delete the payload.
     if (buffer_[index]->seq_num == packet->seq_num) {

modules/video_coding/packet_buffer.h

@@ -63,6 +63,9 @@ class PacketBuffer {
 
     rtc::CopyOnWriteBuffer video_payload;
     RTPVideoHeader video_header;
+#if defined(WEBRTC_WIN)
+    int64_t time_ticks;
+#endif
   };
   struct InsertResult {
     std::vector<std::unique_ptr<Packet>> packets;

video/rtp_video_stream_receiver.h

@@ -47,6 +47,9 @@
 #include "rtc_base/constructor_magic.h"
 #include "rtc_base/critical_section.h"
 #include "rtc_base/numerics/sequence_number_util.h"
+#if defined(WEBRTC_WIN)
+#include "rtc_base/time/ptp_clock_sync.h"
+#endif
 #include "rtc_base/synchronization/sequence_checker.h"
 #include "rtc_base/thread_annotations.h"
 #include "rtc_base/thread_checker.h"
@@ -272,6 +275,9 @@ class RtpVideoStreamReceiver : public LossNotificationSender,
   void OnAssembledFrame(std::unique_ptr<video_coding::RtpFrameObject> frame);
 
   Clock* const clock_;
+#if defined(WEBRTC_WIN)
+  PTPClockSync clock_sync_;
+#endif
   // Ownership of this object lies with VideoReceiveStream, which owns |this|.
   const VideoReceiveStream::Config& config_;
   PacketRouter* const packet_router_;

video/rtp_video_stream_receiver2.cc

@@ -623,8 +623,11 @@ void RtpVideoStreamReceiver2::OnReceivedPayloadData(
       RTC_LOG(LS_INFO)
           << "LossNotificationController does not support reordering.";
     } else if (generic_descriptor_state == kNoGenericDescriptor) {
+#if 0
+	  // Johny(TODO): check cause of no generic descriptor.
       RTC_LOG(LS_WARNING) << "LossNotificationController requires generic "
                              "frame descriptor, but it is missing.";
+#endif
     } else {
       if (video_header.is_first_packet_in_frame) {
         RTC_DCHECK(video_header.generic);
@@ -810,6 +813,11 @@ void RtpVideoStreamReceiver2::OnInsertedPacket(
   std::vector<rtc::ArrayView<const uint8_t>> payloads;
   RtpPacketInfos::vector_type packet_infos;
 
+  // Add timing information required by sender-side BWE.
+#if defined(WEBRTC_WIN)
+  int64_t max_tc = 0, min_tc = 0;
+  double start_duration = 0, last_duration = 0;
+#endif
   bool frame_boundary = true;
   for (auto& packet : result.packets) {
     // PacketBuffer promisses frame boundaries are correctly set on each
@@ -824,16 +832,28 @@ void RtpVideoStreamReceiver2::OnInsertedPacket(
       max_nack_count = packet->times_nacked;
       min_recv_time = packet_info.receive_time().ms();
       max_recv_time = packet_info.receive_time().ms();
+#if defined(WEBRTC_WIN)
+      max_tc = min_tc = packet->time_ticks;
+#endif
     } else {
       max_nack_count = std::max(max_nack_count, packet->times_nacked);
       min_recv_time = std::min(min_recv_time, packet_info.receive_time().ms());
       max_recv_time = std::max(max_recv_time, packet_info.receive_time().ms());
+#if defined(WEBRTC_WIN)
+      max_tc = std::max(max_tc, packet->time_ticks);
+      min_tc = std::min(min_tc, packet->time_ticks);
+#endif
     }
     payloads.emplace_back(packet->video_payload);
     packet_infos.push_back(packet_info);
 
     frame_boundary = packet->is_last_packet_in_frame();
     if (packet->is_last_packet_in_frame()) {
+#if defined(WEBRTC_WIN)
+      clock_sync_.Sync(packet->timestamp, min_tc);
+      start_duration = clock_sync_.GetDuration(packet->timestamp, min_tc);
+      last_duration = clock_sync_.GetDuration(packet->timestamp, max_tc);
+#endif
       auto depacketizer_it = payload_type_map_.find(first_packet->payload_type);
       RTC_CHECK(depacketizer_it != payload_type_map_.end());
 
@@ -845,24 +865,35 @@ void RtpVideoStreamReceiver2::OnInsertedPacket(
       }
 
       const video_coding::PacketBuffer::Packet& last_packet = *packet;
-      OnAssembledFrame(std::make_unique<RtpFrameObject>(
-          first_packet->seq_num,                             //
-          last_packet.seq_num,                               //
-          last_packet.marker_bit,                            //
-          max_nack_count,                                    //
-          min_recv_time,                                     //
-          max_recv_time,                                     //
-          first_packet->timestamp,                           //
+      std::unique_ptr<video_coding::RtpFrameObject> frame =
+          std::make_unique<video_coding::RtpFrameObject>(
+          first_packet->seq_num,                    //
+          last_packet.seq_num,                      //
+          last_packet.marker_bit,                   //
+          max_nack_count,                           //
+          min_recv_time,                            //
+          max_recv_time,                            //
+          first_packet->timestamp,                  //
           ntp_estimator_.Estimate(first_packet->timestamp),  //
-          last_packet.video_header.video_timing,             //
-          first_packet->payload_type,                        //
-          first_packet->codec(),                             //
-          last_packet.video_header.rotation,                 //
-          last_packet.video_header.content_type,             //
-          first_packet->video_header,                        //
-          last_packet.video_header.color_space,              //
-          RtpPacketInfos(std::move(packet_infos)),           //
-          std::move(bitstream)));
+          last_packet.video_header.video_timing,    //
+          first_packet->payload_type,               //
+          first_packet->codec(),                    //
+          last_packet.video_header.rotation,        //
+          last_packet.video_header.content_type,    //
+          first_packet->video_header,               //
+          last_packet.video_header.color_space,     //
+          RtpPacketInfos(std::move(packet_infos)),  //
+              std::move(bitstream));
+#if defined(WEBRTC_WIN)
+      StreamStatistician* ss =
+          rtp_receive_statistics_->GetStatistician(config_.rtp.remote_ssrc);
+      int32_t packets_lost = 0;
+      if (ss != nullptr) {
+        packets_lost = ss->GetStats().packets_lost;
+        frame->SetBWETiming(start_duration, last_duration, packets_lost);
+      }
+#endif
+      OnAssembledFrame(std::move(frame));
       payloads.clear();
       packet_infos.clear();
     }