Use REMB for RID streams (#361)

Author: starwarfan

scripts/patches/licode/0010-Enable-REMB-for-RID-streams.patch

@@ -0,0 +1,179 @@
+From 15553a66a08795ab3bbbcfdc9ad3547911ac763e Mon Sep 17 00:00:00 2001
+From: Chen Li1 <[email protected]>
+Date: Thu, 31 Oct 2019 15:39:42 +0800
+Subject: [PATCH] Enable REMB for RID streams
+
+---
+ erizo/src/erizo/MediaStream.cpp                    | 16 +++++++++++++---
+ erizo/src/erizo/SdpInfo.cpp                        |  1 +
+ erizo/src/erizo/rtp/BandwidthEstimationHandler.cpp | 20 ++++++++++++++------
+ erizo/src/erizo/rtp/BandwidthEstimationHandler.h   |  4 ++--
+ erizo/src/erizo/rtp/RtpExtensionProcessor.cpp      |  2 +-
+ 5 files changed, 31 insertions(+), 12 deletions(-)
+
+diff --git a/erizo/src/erizo/MediaStream.cpp b/erizo/src/erizo/MediaStream.cpp
+index a78b057..4327c6f 100644
+--- a/erizo/src/erizo/MediaStream.cpp
++++ b/erizo/src/erizo/MediaStream.cpp
+@@ -21,7 +21,7 @@
+ #include "rtp/RtcpForwarder.h"
+ // #include "rtp/RtpSlideShowHandler.h"
+ // #include "rtp/RtpTrackMuteHandler.h"
+-// #include "rtp/BandwidthEstimationHandler.h"
++#include "rtp/BandwidthEstimationHandler.h"
+ // #include "rtp/FecReceiverHandler.h"
+ #include "rtp/RtcpProcessorHandler.h"
+ // #include "rtp/RtpRetransmissionHandler.h"
+@@ -151,6 +151,10 @@ bool MediaStream::setRemoteSdp(std::shared_ptr<SdpInfo> sdp) {
+   if (!sending_) {
+     return true;
+   }
++  if (!sdp) {
++    ELOG_WARN("%s message: setting remote SDP nullptr", toLog());
++    return true;
++  }
+   remote_sdp_ =  std::make_shared<SdpInfo>(*sdp.get());
+   if (remote_sdp_->videoBandwidth != 0) {
+     ELOG_DEBUG("%s message: Setting remote BW, maxVideoBW: %u", toLog(), remote_sdp_->videoBandwidth);
+@@ -193,6 +197,10 @@ bool MediaStream::setRemoteSdp(std::shared_ptr<SdpInfo> sdp) {
+       rtcp_processor_->addSourceSsrc(new_ssrc);
+   });
+ 
++  if (remote_sdp_->rids().size() > 1) {
++    simulcast_ = true;
++  }
++
+   initializePipeline();
+ 
+   initializeStats();
+@@ -348,7 +356,7 @@ void MediaStream::initializePipeline() {
+   pipeline_->addFront(std::make_shared<PacketReader>(this));
+   pipeline_->addFront(WoogeenHandler(this));
+ 
+-  pipeline_->addFront(std::make_shared<RtcpProcessorHandler>());
++  // pipeline_->addFront(std::make_shared<RtcpProcessorHandler>());
+   // pipeline_->addFront(std::make_shared<FecReceiverHandler>());
+   // pipeline_->addFront(std::make_shared<LayerBitrateCalculationHandler>());
+   // pipeline_->addFront(std::make_shared<QualityFilterHandler>());
+@@ -357,7 +365,9 @@ void MediaStream::initializePipeline() {
+   // pipeline_->addFront(std::make_shared<RtpSlideShowHandler>());
+   // pipeline_->addFront(std::make_shared<RtpPaddingGeneratorHandler>());
+   // pipeline_->addFront(std::make_shared<PliPacerHandler>());
+-  // pipeline_->addFront(std::make_shared<BandwidthEstimationHandler>());
++  if (simulcast_ && is_publisher_) {
++    pipeline_->addFront(std::make_shared<BandwidthEstimationHandler>());
++  }
+   // pipeline_->addFront(std::make_shared<RtpPaddingRemovalHandler>());
+   // pipeline_->addFront(std::make_shared<RtcpFeedbackGenerationHandler>());
+   // pipeline_->addFront(std::make_shared<RtpRetransmissionHandler>());
+diff --git a/erizo/src/erizo/SdpInfo.cpp b/erizo/src/erizo/SdpInfo.cpp
+index c4d9ea7..4e13951 100644
+--- a/erizo/src/erizo/SdpInfo.cpp
++++ b/erizo/src/erizo/SdpInfo.cpp
+@@ -69,6 +69,7 @@ namespace erizo {
+   }
+ 
+   bool SdpInfo::initWithSdp(const std::string& sdp, const std::string& media) {
++    rids_.clear();
+     return processSdp(sdp, media);
+   }
+   std::string SdpInfo::addCandidate(const CandidateInfo& info) {
+diff --git a/erizo/src/erizo/rtp/BandwidthEstimationHandler.cpp b/erizo/src/erizo/rtp/BandwidthEstimationHandler.cpp
+index a5a9366..d283ca5 100644
+--- a/erizo/src/erizo/rtp/BandwidthEstimationHandler.cpp
++++ b/erizo/src/erizo/rtp/BandwidthEstimationHandler.cpp
+@@ -97,15 +97,15 @@ void BandwidthEstimationHandler::process() {
+   }, std::chrono::milliseconds(rbe_->TimeUntilNextProcess()));
+ }
+ 
+-void BandwidthEstimationHandler::updateExtensionMaps(std::array<RTPExtensions, 10> video_map,
+-                                                     std::array<RTPExtensions, 10> audio_map) {
++void BandwidthEstimationHandler::updateExtensionMaps(std::array<RTPExtensions, 20> video_map,
++                                                     std::array<RTPExtensions, 20> audio_map) {
+   updateExtensionMap(true, video_map);
+   updateExtensionMap(false, audio_map);
+ }
+ 
+-void BandwidthEstimationHandler::updateExtensionMap(bool is_video, std::array<RTPExtensions, 10> map) {
++void BandwidthEstimationHandler::updateExtensionMap(bool is_video, std::array<RTPExtensions, 20> map) {
+   webrtc::RTPExtensionType type = webrtc::kRtpExtensionNone;
+-  for (uint8_t id = 0; id < 10; id++) {
++  for (uint8_t id = 0; id < 20; id++) {
+     RTPExtensions extension = map[id];
+     switch (extension) {
+       case RTP_ID:
+@@ -130,6 +130,9 @@ void BandwidthEstimationHandler::updateExtensionMap(bool is_video, std::array<RT
+       case PLAYBACK_TIME:
+         type = webrtc::kRtpExtensionPlayoutDelay;
+         break;
++      default:
++        continue;
++        break;
+     }
+     if (type == webrtc::kRtpExtensionNone) {
+       continue;
+@@ -219,6 +222,11 @@ void BandwidthEstimationHandler::pickEstimator() {
+ }
+ 
+ void BandwidthEstimationHandler::sendREMBPacket() {
++  uint32_t mul = 1;
++  auto remoteSdp = stream_->getRemoteSdpInfo();
++  if (remoteSdp && !remoteSdp->rids().empty()) {
++    mul = (1 << remoteSdp->rids().size()) - 1;
++  }
+   remb_packet_.setPacketType(RTCP_PS_Feedback_PT);
+   remb_packet_.setBlockCount(RTCP_AFB);
+   memcpy(&remb_packet_.report.rembPacket.uniqueid, "REMB", 4);
+@@ -229,12 +237,12 @@ void BandwidthEstimationHandler::sendREMBPacket() {
+   remb_packet_.setLength(5);
+   uint32_t capped_bitrate = max_video_bw_ > 0 ? std::min(max_video_bw_, bitrate_) : bitrate_;
+   ELOG_DEBUG("Bitrates min(%u,%u) = %u", bitrate_, max_video_bw_, capped_bitrate);
+-  remb_packet_.setREMBBitRate(capped_bitrate);
++  remb_packet_.setREMBBitRate(bitrate_ * mul);
+   remb_packet_.setREMBNumSSRC(1);
+   remb_packet_.setREMBFeedSSRC(0, stream_->getVideoSourceSSRC());
+   int remb_length = (remb_packet_.getLength() + 1) * 4;
+   if (active_) {
+-    ELOG_DEBUG("BWE Estimation is %d", last_send_bitrate_);
++    ELOG_DEBUG("BWE Estimation is %d", bitrate_ * mul);
+     getContext()->fireWrite(std::make_shared<DataPacket>(0,
+       reinterpret_cast<char*>(&remb_packet_), remb_length, OTHER_PACKET));
+   }
+diff --git a/erizo/src/erizo/rtp/BandwidthEstimationHandler.h b/erizo/src/erizo/rtp/BandwidthEstimationHandler.h
+index c851ab1..166d799 100644
+--- a/erizo/src/erizo/rtp/BandwidthEstimationHandler.h
++++ b/erizo/src/erizo/rtp/BandwidthEstimationHandler.h
+@@ -55,7 +55,7 @@ class BandwidthEstimationHandler: public Handler, public RemoteBitrateObserver,
+   void write(Context *ctx, std::shared_ptr<DataPacket> packet) override;
+   void notifyUpdate() override;
+ 
+-  void updateExtensionMaps(std::array<RTPExtensions, 10> video_map, std::array<RTPExtensions, 10> audio_map);
++  void updateExtensionMaps(std::array<RTPExtensions, 20> video_map, std::array<RTPExtensions, 20> audio_map);
+ 
+  private:
+   void process();
+@@ -65,7 +65,7 @@ class BandwidthEstimationHandler: public Handler, public RemoteBitrateObserver,
+   void pickEstimatorFromHeader();
+   void pickEstimator();
+ 
+-  void updateExtensionMap(bool video, std::array<RTPExtensions, 10> map);
++  void updateExtensionMap(bool video, std::array<RTPExtensions, 20> map);
+ 
+   MediaStream *stream_;
+   std::shared_ptr<Worker> worker_;
+diff --git a/erizo/src/erizo/rtp/RtpExtensionProcessor.cpp b/erizo/src/erizo/rtp/RtpExtensionProcessor.cpp
+index 5ee860e..6ea37e8 100644
+--- a/erizo/src/erizo/rtp/RtpExtensionProcessor.cpp
++++ b/erizo/src/erizo/rtp/RtpExtensionProcessor.cpp
+@@ -153,7 +153,7 @@ uint32_t RtpExtensionProcessor::processRtpExtensions(std::shared_ptr<DataPacket>
+         if (extId != 0 && extMap[extId] != 0) {
+           switch (extMap[extId]) {
+             case ABS_SEND_TIME:
+-              processAbsSendTime(extBuffer);
++              // processAbsSendTime(extBuffer);
+               break;
+             case VIDEO_ORIENTATION:
+               processVideoOrientation(extBuffer);
+-- 
+2.7.4
+

source/agent/webrtc/log4cxx.properties

@@ -63,3 +63,4 @@ log4j.logger.rtp.StatsCalculator=WARN
 log4j.logger.rtp.LayerDetectorHandler=WARN
 log4j.logger.rtp.PliPacerHandler=WARN
 log4j.logger.rtp.RtpPaddingGeneratorHandler=WARN
+log4j.logger.rtp.BandwidthEstimationHandler=WARN

source/agent/webrtc/sdp.js

@@ -422,7 +422,20 @@ exports.getExtId = function (sdp, extUri) {
       }
     }
   }
-  return -1;
+  return 0;
+};
+
+exports.filterExt = function (sdp, uri) {
+  const sdpObj = transform.parse(sdp);
+  for (const media of sdpObj.media) {
+    if (Array.isArray(media.ext)) {
+      media.ext = media.ext.filter(ext => ext.uri !== uri);
+    }
+    if (Array.isArray(media.rtcpFb)) {
+      media.rtcpFb = media.rtcpFb.filter(fb => fb.type !== 'transport-cc');
+    }
+  }
+  return transform.write(sdpObj);
 };
 
 exports.processOffer = function (sdp, preference = {}, direction) {

source/agent/webrtc/webrtcLib/binding.gyp

@@ -33,6 +33,7 @@
       'erizo/src/erizo/rtp/RtpUtils.cpp',
       'erizo/src/erizo/rtp/QualityManager.cpp',
       'erizo/src/erizo/rtp/RtpExtensionProcessor.cpp',
+      'erizo/src/erizo/rtp/BandwidthEstimationHandler.cpp',
       '<!@(find erizo/src/erizo/dtls -name "*.cpp")',
       '<!@(find erizo/src/erizo/dtls -name "*.c")',
       '<!@(find erizo/src/erizo/pipeline -name "*.cpp")',

source/agent/webrtc/wrtcConnection.js

@@ -24,6 +24,7 @@ const {
   getLegacySimulcastInfo,
   hasCodec,
   getExtId,
+  filterExt,
   addAudioSSRC,
   addVideoSSRC,
 } = require('./sdp');
@@ -184,6 +185,10 @@ module.exports = function (spec, on_status, on_mediaUpdate) {
           const vSsrc = videoFramePacketizer.ssrc();
           message = addVideoSSRC(message, vSsrc);
         }
+        if (isSimulcast) {
+          // TODO: enable transport-cc for simulcast if bandwidth works
+          message = filterExt(message, TransportSeqNumUri);
+        }
         log.debug('Answer SDP', message);
         on_status({type: 'answer', sdp: message});
 
@@ -292,10 +297,9 @@ module.exports = function (spec, on_status, on_mediaUpdate) {
 
   var processLegacySimulcast = function (sdp) {
     // Process legacy simulcast info for Safari
-    log.debug('Process legacy simulcast');
     const simInfo = getLegacySimulcastInfo(sdp);
     if (simInfo.length > 1 && video) {
-      isSimulcast = true;
+      log.debug('Process legacy simulcast');
       stream = new WrtcStream({
         audioFrameConstructor,
         videoFrameConstructor,
@@ -337,12 +341,19 @@ module.exports = function (spec, on_status, on_mediaUpdate) {
 
   var setupTransport = function (sdp) {
     if (direction === 'in') {
+      const simulcastInfo = getSimulcastInfo(sdp);
+      const legacySimInfo = getLegacySimulcastInfo(sdp);
+      transportSeqNumExt = getExtId(sdp, TransportSeqNumUri);
+      if (simulcastInfo.length > 0 || legacySimInfo.length > 1) {
+        isSimulcast = true;
+        transportSeqNumExt = 0;
+      }
+
       if (audio) {
         audioFrameConstructor = new AudioFrameConstructor();
         audioFrameConstructor.bindTransport(wrtc.getMediaStream(wrtcId));
       }
       if (video) {
-        transportSeqNumExt = getExtId(sdp, TransportSeqNumUri);
         videoFrameConstructor = new VideoFrameConstructor(on_mediaUpdate, transportSeqNumExt);
         videoFrameConstructor.bindTransport(wrtc.getMediaStream(wrtcId));
       }
@@ -352,10 +363,6 @@ module.exports = function (spec, on_status, on_mediaUpdate) {
       }
       const aSsrc = getAudioSsrc(sdp);
       const vSsrc = getVideoSsrcList(sdp);
-      const simulcastInfo = getSimulcastInfo(sdp);
-      if (simulcastInfo.length > 0) {
-        isSimulcast = true;
-      }
       log.debug('SDP ssrc:', aSsrc, vSsrc);
       wrtc.setRemoteSsrc(aSsrc, vSsrc, '');
       wrtc.setSimulcastInfo(simulcastInfo);

source/core/owt_base/VideoFrameConstructor.cpp

@@ -62,7 +62,6 @@ bool VideoFrameConstructor::init(uint32_t transportccExtId)
     m_remoteBitrateObserver.reset(new VieRemb(Clock::GetRealTimeClock()));
     m_remoteBitrateEstimator.reset(new RemoteEstimatorProxy(Clock::GetRealTimeClock(), m_packetRouter.get()));
     m_videoReceiver.reset(new ViEReceiver(m_video_receiver.get(), m_remoteBitrateEstimator.get(), this));
-    m_videoReceiver->SetReceiveTransportSequenceNumberStatus(true, transportccExtId);
 
     RtpRtcp::Configuration configuration;
     configuration.audio = false;  // Video.
@@ -73,7 +72,10 @@ bool VideoFrameConstructor::init(uint32_t transportccExtId)
     m_rtpRtcp->SetRTCPStatus(webrtc::RtcpMode::kCompound);
     // Since currently our MCU only claims FIR support in SDP, we choose FirRtcp for now.
     m_rtpRtcp->SetKeyFrameRequestMethod(kKeyFrameReqFirRtcp);
-    m_rtpRtcp->RegisterSendRtpHeaderExtension(RTPExtensionType::kRtpExtensionTransportSequenceNumber, transportccExtId);
+    if (transportccExtId > 0) {
+        m_videoReceiver->SetReceiveTransportSequenceNumberStatus(true, transportccExtId);
+        m_rtpRtcp->RegisterSendRtpHeaderExtension(RTPExtensionType::kRtpExtensionTransportSequenceNumber, transportccExtId);
+    }
     m_rtpRtcp->SetREMBStatus(false);
     m_videoReceiver->SetRtpRtcpModule(m_rtpRtcp.get());
     m_remoteBitrateObserver->AddRembSender(m_rtpRtcp.get());

source/core/owt_base/VideoFrameConstructor.h

@@ -52,7 +52,7 @@ class VideoFrameConstructor : public erizo::MediaSink,
     DECLARE_LOGGER();
 
 public:
-    VideoFrameConstructor(VideoInfoListener*, uint32_t transportccExtId = 2);
+    VideoFrameConstructor(VideoInfoListener*, uint32_t transportccExtId = 0);
     virtual ~VideoFrameConstructor();
 
     void bindTransport(erizo::MediaSource* source, erizo::FeedbackSink* fbSink);