Merge to M88: Enable initial frame drop for single stream/layer

This is merge of 4 separate CLs:

1) Enable initial frame drop for SVC 'singlecast'
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/195542
(cherry picked from commit cde4a9f)


2) Reset initial frame dropper if the stream changes for external reasons
External reasons here are simulcast configuration and
source resolution change.
Initial frame dropper should be enabled in these cases because the
client can request way too big resolution for available bitrate and
usual quality scaling would take too long.
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/195004
(cherry picked from commit 84bc348)


3) Adjust min bitrate for the first active stream
Without this change, if the user disables QVGA and VGA streams via |active|
flags in SetParamters, the resulting stream would have too high min bitrate.
This would lead to bad performance and low quality adaptation rate.
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/195325
(cherry picked from commit d381194)


4) Enable initial frame drop for one active simulcast layer.
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/193840
(cherry picked from commit f46723c)

No-Try: True
Bug: chromium:1153693
Change-Id: I7f31f85b5da2afdfd79e9c6f92d9a5a629cf18f7
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/195940
Commit-Queue: Mirko Bonadei <[email protected]>
Reviewed-by: Åsa Persson <[email protected]>
Cr-Commit-Position: refs/branch-heads/4324@{#2}
Cr-Branched-From: daab689-refs/heads/master@{#32599}

Author: Ilya Nikolaevskiy

media/engine/webrtc_video_engine.cc

@@ -3621,6 +3621,32 @@ EncoderStreamFactory::CreateSimulcastOrConferenceModeScreenshareStreams(
     BoostMaxSimulcastLayer(
         webrtc::DataRate::BitsPerSec(encoder_config.max_bitrate_bps), &layers);
   }
+
+  // Sort the layers by max_bitrate_bps, they might not always be from
+  // smallest to biggest
+  std::vector<size_t> index(layers.size());
+  std::iota(index.begin(), index.end(), 0);
+  std::stable_sort(index.begin(), index.end(), [&layers](size_t a, size_t b) {
+    return layers[a].max_bitrate_bps < layers[b].max_bitrate_bps;
+  });
+
+  if (!layers[index[0]].active) {
+    // Adjust min bitrate of the first active layer to allow it to go as low as
+    // the lowest (now inactive) layer could.
+    // Otherwise, if e.g. a single HD stream is active, it would have 600kbps
+    // min bitrate, which would always be allocated to the stream.
+    // This would lead to congested network, dropped frames and overall bad
+    // experience.
+
+    const int min_configured_bitrate = layers[index[0]].min_bitrate_bps;
+    for (size_t i = 0; i < layers.size(); ++i) {
+      if (layers[index[i]].active) {
+        layers[index[i]].min_bitrate_bps = min_configured_bitrate;
+        break;
+      }
+    }
+  }
+
   return layers;
 }
 

media/engine/webrtc_video_engine_unittest.cc

@@ -7878,6 +7878,59 @@ TEST_F(WebRtcVideoChannelTest, SetRtpSendParametersMultipleEncodingsActive) {
   EXPECT_TRUE(channel_->SetVideoSend(primary_ssrc, nullptr, nullptr));
 }
 
+// Tests that when some streams are disactivated then the lowest
+// stream min_bitrate would be reused for the first active stream.
+TEST_F(WebRtcVideoChannelTest,
+       SetRtpSendParametersSetsMinBitrateForFirstActiveStream) {
+  // Create the stream params with multiple ssrcs for simulcast.
+  const size_t kNumSimulcastStreams = 3;
+  std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs3);
+  StreamParams stream_params = CreateSimStreamParams("cname", ssrcs);
+  FakeVideoSendStream* fake_video_send_stream = AddSendStream(stream_params);
+  uint32_t primary_ssrc = stream_params.first_ssrc();
+
+  // Using the FrameForwarder, we manually send a full size
+  // frame. This allows us to test that ReconfigureEncoder is called
+  // appropriately.
+  webrtc::test::FrameForwarder frame_forwarder;
+  VideoOptions options;
+  EXPECT_TRUE(channel_->SetVideoSend(primary_ssrc, &options, &frame_forwarder));
+  channel_->SetSend(true);
+  frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame(
+      1920, 1080, webrtc::VideoRotation::kVideoRotation_0,
+      rtc::kNumMicrosecsPerSec / 30));
+
+  // Check that all encodings are initially active.
+  webrtc::RtpParameters parameters =
+      channel_->GetRtpSendParameters(primary_ssrc);
+  EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size());
+  EXPECT_TRUE(parameters.encodings[0].active);
+  EXPECT_TRUE(parameters.encodings[1].active);
+  EXPECT_TRUE(parameters.encodings[2].active);
+  EXPECT_TRUE(fake_video_send_stream->IsSending());
+
+  // Only turn on the highest stream.
+  parameters.encodings[0].active = false;
+  parameters.encodings[1].active = false;
+  parameters.encodings[2].active = true;
+  EXPECT_TRUE(channel_->SetRtpSendParameters(primary_ssrc, parameters).ok());
+
+  // Check that the VideoSendStream is updated appropriately. This means its
+  // send state was updated and it was reconfigured.
+  EXPECT_TRUE(fake_video_send_stream->IsSending());
+  std::vector<webrtc::VideoStream> simulcast_streams =
+      fake_video_send_stream->GetVideoStreams();
+  EXPECT_EQ(kNumSimulcastStreams, simulcast_streams.size());
+  EXPECT_FALSE(simulcast_streams[0].active);
+  EXPECT_FALSE(simulcast_streams[1].active);
+  EXPECT_TRUE(simulcast_streams[2].active);
+
+  EXPECT_EQ(simulcast_streams[2].min_bitrate_bps,
+            simulcast_streams[0].min_bitrate_bps);
+
+  EXPECT_TRUE(channel_->SetVideoSend(primary_ssrc, nullptr, nullptr));
+}
+
 // Test that if a stream is reconfigured (due to a codec change or other
 // change) while its encoding is still inactive, it doesn't start sending.
 TEST_F(WebRtcVideoChannelTest,

video/adaptation/video_stream_encoder_resource_manager.cc

@@ -10,6 +10,7 @@
 
 #include "video/adaptation/video_stream_encoder_resource_manager.h"
 
+#include <algorithm>
 #include <cmath>
 #include <limits>
 #include <memory>
@@ -58,6 +59,52 @@ std::string ToString(VideoAdaptationReason reason) {
   RTC_CHECK_NOTREACHED();
 }
 
+absl::optional<uint32_t> GetSingleActiveStreamPixels(const VideoCodec& codec) {
+  int num_active = 0;
+  absl::optional<uint32_t> pixels;
+  if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
+    for (int i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) {
+      if (codec.spatialLayers[i].active) {
+        ++num_active;
+        pixels = codec.spatialLayers[i].width * codec.spatialLayers[i].height;
+      }
+    }
+  } else {
+    for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
+      if (codec.simulcastStream[i].active) {
+        ++num_active;
+        pixels =
+            codec.simulcastStream[i].width * codec.simulcastStream[i].height;
+      }
+    }
+  }
+  if (num_active > 1)
+    return absl::nullopt;
+  return pixels;
+}
+
+std::vector<bool> GetActiveLayersFlags(const VideoCodec& codec) {
+  std::vector<bool> flags;
+  if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
+    flags.resize(codec.VP9().numberOfSpatialLayers);
+    for (size_t i = 0; i < flags.size(); ++i) {
+      flags[i] = codec.spatialLayers[i].active;
+    }
+  } else {
+    flags.resize(codec.numberOfSimulcastStreams);
+    for (size_t i = 0; i < flags.size(); ++i) {
+      flags[i] = codec.simulcastStream[i].active;
+    }
+  }
+  return flags;
+}
+
+bool EqualFlags(const std::vector<bool>& a, const std::vector<bool>& b) {
+  if (a.size() != b.size())
+    return false;
+  return std::equal(a.begin(), a.end(), b.begin());
+}
+
 }  // namespace
 
 class VideoStreamEncoderResourceManager::InitialFrameDropper {
@@ -69,7 +116,9 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper {
         has_seen_first_bwe_drop_(false),
         set_start_bitrate_(DataRate::Zero()),
         set_start_bitrate_time_ms_(0),
-        initial_framedrop_(0) {
+        initial_framedrop_(0),
+        last_input_width_(0),
+        last_input_height_(0) {
     RTC_DCHECK(quality_scaler_resource_);
   }
 
@@ -78,6 +127,10 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper {
     return initial_framedrop_ < kMaxInitialFramedrop;
   }
 
+  absl::optional<uint32_t> single_active_stream_pixels() const {
+    return single_active_stream_pixels_;
+  }
+
   // Input signals.
   void SetStartBitrate(DataRate start_bitrate, int64_t now_ms) {
     set_start_bitrate_ = start_bitrate;
@@ -104,9 +157,38 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper {
     }
   }
 
+  void OnEncoderSettingsUpdated(
+      const VideoCodec& codec,
+      const VideoAdaptationCounters& adaptation_counters) {
+    std::vector<bool> active_flags = GetActiveLayersFlags(codec);
+    // Check if the source resolution has changed for the external reasons,
+    // i.e. without any adaptation from WebRTC.
+    const bool source_resolution_changed =
+        (last_input_width_ != codec.width ||
+         last_input_height_ != codec.height) &&
+        adaptation_counters.resolution_adaptations ==
+            last_adaptation_counters_.resolution_adaptations;
+    if (!EqualFlags(active_flags, last_active_flags_) ||
+        source_resolution_changed) {
+      // Streams configuration has changed.
+      // Initial frame drop must be enabled because BWE might be way too low
+      // for the selected resolution.
+      if (quality_scaler_resource_->is_started()) {
+        RTC_LOG(LS_INFO) << "Resetting initial_framedrop_ due to changed "
+                            "stream parameters";
+        initial_framedrop_ = 0;
+      }
+    }
+    last_adaptation_counters_ = adaptation_counters;
+    last_active_flags_ = active_flags;
+    last_input_width_ = codec.width;
+    last_input_height_ = codec.height;
+    single_active_stream_pixels_ = GetSingleActiveStreamPixels(codec);
+  }
+
   void OnFrameDroppedDueToSize() { ++initial_framedrop_; }
 
-  void OnMaybeEncodeFrame() { initial_framedrop_ = kMaxInitialFramedrop; }
+  void Disable() { initial_framedrop_ = kMaxInitialFramedrop; }
 
   void OnQualityScalerSettingsUpdated() {
     if (quality_scaler_resource_->is_started()) {
@@ -130,6 +212,12 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper {
   int64_t set_start_bitrate_time_ms_;
   // Counts how many frames we've dropped in the initial framedrop phase.
   int initial_framedrop_;
+  absl::optional<uint32_t> single_active_stream_pixels_;
+
+  std::vector<bool> last_active_flags_;
+  VideoAdaptationCounters last_adaptation_counters_;
+  int last_input_width_;
+  int last_input_height_;
 };
 
 VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager(
@@ -230,7 +318,7 @@ void VideoStreamEncoderResourceManager::AddResource(
   RTC_DCHECK(resource);
   bool inserted;
   std::tie(std::ignore, inserted) = resources_.emplace(resource, reason);
-  RTC_DCHECK(inserted) << "Resurce " << resource->Name()
+  RTC_DCHECK(inserted) << "Resource " << resource->Name()
                        << " already was inserted";
   adaptation_processor_->AddResource(resource);
 }
@@ -259,6 +347,8 @@ void VideoStreamEncoderResourceManager::SetEncoderSettings(
   RTC_DCHECK_RUN_ON(encoder_queue_);
   encoder_settings_ = std::move(encoder_settings);
   bitrate_constraint_->OnEncoderSettingsUpdated(encoder_settings_);
+  initial_frame_dropper_->OnEncoderSettingsUpdated(
+      encoder_settings_->video_codec(), current_adaptation_counters_);
   MaybeUpdateTargetFrameRate();
 }
 
@@ -339,9 +429,15 @@ bool VideoStreamEncoderResourceManager::DropInitialFrames() const {
   return initial_frame_dropper_->DropInitialFrames();
 }
 
+absl::optional<uint32_t>
+VideoStreamEncoderResourceManager::SingleActiveStreamPixels() const {
+  RTC_DCHECK_RUN_ON(encoder_queue_);
+  return initial_frame_dropper_->single_active_stream_pixels();
+}
+
 void VideoStreamEncoderResourceManager::OnMaybeEncodeFrame() {
   RTC_DCHECK_RUN_ON(encoder_queue_);
-  initial_frame_dropper_->OnMaybeEncodeFrame();
+  initial_frame_dropper_->Disable();
   if (quality_rampup_experiment_ && quality_scaler_resource_->is_started()) {
     DataRate bandwidth = encoder_rates_.has_value()
                              ? encoder_rates_->bandwidth_allocation
@@ -457,6 +553,8 @@ void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated(
     rtc::scoped_refptr<Resource> reason,
     const VideoSourceRestrictions& unfiltered_restrictions) {
   RTC_DCHECK_RUN_ON(encoder_queue_);
+  current_adaptation_counters_ = adaptation_counters;
+
   // TODO(bugs.webrtc.org/11553) Remove reason parameter and add reset callback.
   if (!reason && adaptation_counters.Total() == 0) {
     // Adaptation was manually reset - clear the per-reason counters too.

video/adaptation/video_stream_encoder_resource_manager.h

@@ -121,9 +121,10 @@ class VideoStreamEncoderResourceManager
                    VideoAdaptationReason reason);
   void RemoveResource(rtc::scoped_refptr<Resource> resource);
   std::vector<AdaptationConstraint*> AdaptationConstraints() const;
-  // If true, the VideoStreamEncoder should eexecute its logic to maybe drop
-  // frames baseed on size and bitrate.
+  // If true, the VideoStreamEncoder should execute its logic to maybe drop
+  // frames based on size and bitrate.
   bool DropInitialFrames() const;
+  absl::optional<uint32_t> SingleActiveStreamPixels() const;
 
   // VideoSourceRestrictionsListener implementation.
   // Updates |video_source_restrictions_|.
@@ -183,6 +184,9 @@ class VideoStreamEncoderResourceManager
   VideoSourceRestrictions video_source_restrictions_
       RTC_GUARDED_BY(encoder_queue_);
 
+  VideoAdaptationCounters current_adaptation_counters_
+      RTC_GUARDED_BY(encoder_queue_);
+
   const BalancedDegradationSettings balanced_settings_;
   Clock* clock_ RTC_GUARDED_BY(encoder_queue_);
   const bool experiment_cpu_load_estimator_ RTC_GUARDED_BY(encoder_queue_);