Allow encoders to receive preferred pixel formats from native buffers

Adds a field to EncoderInfo called preferred_pixel_formats which a
software encoder populates with the pixel formats it supports. When a
kNative frame is received for encoding, the VideoStreamEncoder will
first try to get a frame that is accessible by the software encoder in
that pixel format from the kNative frame. If this fails it will fallback
to converting the frame using ToI420.

This minimizes the number of conversions made in the case that the
encoder supports the pixel format of the native buffer or where
conversion can be accelerated. For example, in Chromium, the capturer can
emit an NV12 frame, which can be consumed by libvpx which supports NV12.

Testing: Tested in Chrome with media::VideoFrame adapters.

Bug: webrtc:11977
Change-Id: I9becc4100136b0c0128f4fa06dedf9ee4dc62f37
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/187121
Reviewed-by: Niels Moller <[email protected]>
Reviewed-by: Ilya Nikolaevskiy <[email protected]>
Reviewed-by: Markus Handell <[email protected]>
Commit-Queue: Evan Shrubsole <[email protected]>
Cr-Commit-Position: refs/heads/master@{#32353}

Author: eshrubs

api/video/video_codec_constants.h

@@ -17,6 +17,7 @@ enum : int { kMaxEncoderBuffers = 8 };
 enum : int { kMaxSimulcastStreams = 3 };
 enum : int { kMaxSpatialLayers = 5 };
 enum : int { kMaxTemporalStreams = 4 };
+enum : int { kMaxPreferredPixelFormats = 5 };
 
 }  // namespace webrtc
 

api/video/video_frame_buffer.cc

@@ -55,10 +55,35 @@ const NV12BufferInterface* VideoFrameBuffer::GetNV12() const {
   return static_cast<const NV12BufferInterface*>(this);
 }
 
+rtc::scoped_refptr<VideoFrameBuffer> VideoFrameBuffer::GetMappedFrameBuffer(
+    rtc::ArrayView<Type> types) {
+  RTC_CHECK(type() == Type::kNative);
+  return nullptr;
+}
+
 VideoFrameBuffer::Type I420BufferInterface::type() const {
   return Type::kI420;
 }
 
+const char* VideoFrameBufferTypeToString(VideoFrameBuffer::Type type) {
+  switch (type) {
+    case VideoFrameBuffer::Type::kNative:
+      return "kNative";
+    case VideoFrameBuffer::Type::kI420:
+      return "kI420";
+    case VideoFrameBuffer::Type::kI420A:
+      return "kI420A";
+    case VideoFrameBuffer::Type::kI444:
+      return "kI444";
+    case VideoFrameBuffer::Type::kI010:
+      return "kI010";
+    case VideoFrameBuffer::Type::kNV12:
+      return "kNV12";
+    default:
+      RTC_NOTREACHED();
+  }
+}
+
 int I420BufferInterface::ChromaWidth() const {
   return (width() + 1) / 2;
 }

api/video/video_frame_buffer.h

@@ -13,6 +13,7 @@
 
 #include <stdint.h>
 
+#include "api/array_view.h"
 #include "api/scoped_refptr.h"
 #include "rtc_base/ref_count.h"
 #include "rtc_base/system/rtc_export.h"
@@ -74,6 +75,8 @@ class RTC_EXPORT VideoFrameBuffer : public rtc::RefCountInterface {
   // WebrtcVideoFrameAdapter in Chrome - it's I420 buffer backed by a shared
   // memory buffer. Therefore it must have type kNative. Yet, ToI420()
   // doesn't affect binary data at all. Another example is any I420A buffer.
+  // TODO(https://crbug.com/webrtc/12021): Make this method non-virtual and
+  // behave as the other GetXXX methods below.
   virtual const I420BufferInterface* GetI420() const;
 
   // A format specific scale function. Default implementation works by
@@ -101,10 +104,21 @@ class RTC_EXPORT VideoFrameBuffer : public rtc::RefCountInterface {
   const I010BufferInterface* GetI010() const;
   const NV12BufferInterface* GetNV12() const;
 
+  // From a kNative frame, returns a VideoFrameBuffer with a pixel format in
+  // the list of types that is in the main memory with a pixel perfect
+  // conversion for encoding with a software encoder. Returns nullptr if the
+  // frame type is not supported, mapping is not possible, or if the kNative
+  // frame has not implemented this method. Only callable if type() is kNative.
+  virtual rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
+      rtc::ArrayView<Type> types);
+
  protected:
   ~VideoFrameBuffer() override {}
 };
 
+// Update when VideoFrameBuffer::Type is updated.
+const char* VideoFrameBufferTypeToString(VideoFrameBuffer::Type type);
+
 // This interface represents planar formats.
 class PlanarYuvBuffer : public VideoFrameBuffer {
  public:

api/video_codecs/video_encoder.cc

@@ -103,7 +103,8 @@ VideoEncoder::EncoderInfo::EncoderInfo()
       fps_allocation{absl::InlinedVector<uint8_t, kMaxTemporalStreams>(
           1,
           kMaxFramerateFraction)},
-      supports_simulcast(false) {}
+      supports_simulcast(false),
+      preferred_pixel_formats{VideoFrameBuffer::Type::kI420} {}
 
 VideoEncoder::EncoderInfo::EncoderInfo(const EncoderInfo&) = default;
 
@@ -169,7 +170,15 @@ std::string VideoEncoder::EncoderInfo::ToString() const {
   }
   oss << "] "
          ", supports_simulcast = "
-      << supports_simulcast << "}";
+      << supports_simulcast;
+  oss << ", preferred_pixel_formats = [";
+  for (size_t i = 0; i < preferred_pixel_formats.size(); ++i) {
+    if (i > 0)
+      oss << ", ";
+    oss << VideoFrameBufferTypeToString(preferred_pixel_formats.at(i));
+  }
+  oss << "]";
+  oss << "}";
   return oss.str();
 }
 

api/video_codecs/video_encoder.h

@@ -254,6 +254,12 @@ class RTC_EXPORT VideoEncoder {
     // in such case the encoder should return
     // WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED.
     bool supports_simulcast;
+
+    // The list of pixel formats preferred by the encoder. It is assumed that if
+    // the list is empty and supports_native_handle is false, then {I420} is the
+    // preferred pixel format. The order of the formats does not matter.
+    absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
+        preferred_pixel_formats;
   };
 
   struct RTC_EXPORT RateControlParameters {

modules/video_coding/codecs/av1/libaom_av1_encoder.cc

@@ -603,6 +603,7 @@ VideoEncoder::EncoderInfo LibaomAv1Encoder::GetEncoderInfo() const {
   info.has_trusted_rate_controller = true;
   info.is_hardware_accelerated = false;
   info.scaling_settings = VideoEncoder::ScalingSettings(kMinQindex, kMaxQindex);
+  info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420};
   return info;
 }
 

modules/video_coding/codecs/h264/h264_encoder_impl.cc

@@ -615,6 +615,7 @@ VideoEncoder::EncoderInfo H264EncoderImpl::GetEncoderInfo() const {
   info.is_hardware_accelerated = false;
   info.has_internal_source = false;
   info.supports_simulcast = true;
+  info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420};
   return info;
 }
 

modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc

@@ -1244,6 +1244,8 @@ VideoEncoder::EncoderInfo LibvpxVp8Encoder::GetEncoderInfo() const {
     info.scaling_settings.min_pixels_per_frame =
         rate_control_settings_.LibvpxVp8MinPixels().value();
   }
+  info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420,
+                                  VideoFrameBuffer::Type::kNV12};
 
   if (inited_) {
     // |encoder_idx| is libvpx index where 0 is highest resolution.

modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc

@@ -604,6 +604,9 @@ TEST(LibvpxVp8EncoderTest, GetEncoderInfoReturnsStaticInformation) {
   EXPECT_TRUE(info.supports_simulcast);
   EXPECT_EQ(info.implementation_name, "libvpx");
   EXPECT_EQ(info.requested_resolution_alignment, 1);
+  EXPECT_THAT(info.preferred_pixel_formats,
+              testing::UnorderedElementsAre(VideoFrameBuffer::Type::kNV12,
+                                            VideoFrameBuffer::Type::kI420));
 }
 
 TEST(LibvpxVp8EncoderTest, RequestedResolutionAlignmentFromFieldTrial) {

modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc

@@ -1337,6 +1337,12 @@ TEST_F(TestVp9Impl, ScalabilityStructureIsAvailableInFlexibleMode) {
   EXPECT_TRUE(codec_specific_info.codecSpecific.VP9.ss_data_available);
 }
 
+TEST_F(TestVp9Impl, Profile0PreferredPixelFormats) {
+  EXPECT_THAT(encoder_->GetEncoderInfo().preferred_pixel_formats,
+              testing::UnorderedElementsAre(VideoFrameBuffer::Type::kNV12,
+                                            VideoFrameBuffer::Type::kI420));
+}
+
 TEST_F(TestVp9Impl, EncoderInfoFpsAllocation) {
   const uint8_t kNumSpatialLayers = 3;
   const uint8_t kNumTemporalLayers = 3;