Reland "Add scaling interface to VideoFrameBuffer"

(Reland with no changes after the fix to the downstream project)

This can be overriden for kNative frame types to perform scaling efficiently.

Default implementations for existing buffer types require actual
buffer implementation, thus this CL also merges "video_frame"
with "video_frame_I420" build targets.

Originally Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/186303

(Landing with TBR as it's unchaged reland of already approved CL)
[email protected],[email protected]

Bug: webrtc:11976, chromium:1132299
Change-Id: Ia23f7d3e474bd9cdc177104cc5c6d772f04b210f
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/187345
Commit-Queue: Ilya Nikolaevskiy <[email protected]>
Reviewed-by: Ilya Nikolaevskiy <[email protected]>
Reviewed-by: Mirko Bonadei <[email protected]>
Reviewed-by: Stefan Holmer <[email protected]>
Cr-Commit-Position: refs/heads/master@{#32362}

Author: Ilya Nikolaevskiy

api/video/BUILD.gn

@@ -42,6 +42,8 @@ rtc_library("video_rtp_headers") {
 rtc_library("video_frame") {
   visibility = [ "*" ]
   sources = [
+    "i420_buffer.cc",
+    "i420_buffer.h",
     "video_codec_type.h",
     "video_frame.cc",
     "video_frame.h",
@@ -59,7 +61,9 @@ rtc_library("video_frame") {
     "..:scoped_refptr",
     "../../rtc_base:checks",
     "../../rtc_base:rtc_base_approved",
+    "../../rtc_base/memory:aligned_malloc",
     "../../rtc_base/system:rtc_export",
+    "//third_party/libyuv",
   ]
   absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
 }
@@ -70,42 +74,11 @@ if (is_android) {
   }
 }
 
-rtc_source_set("recordable_encoded_frame") {
+# Deprecated empty target. Use "video_frame" instead.
+rtc_source_set("video_frame_i420") {
   visibility = [ "*" ]
-  sources = [ "recordable_encoded_frame.h" ]
-
-  deps = [
-    ":encoded_image",
-    ":video_frame",
-    ":video_rtp_headers",
-    "..:array_view",
-    "..:scoped_refptr",
-    "../../rtc_base:refcount",
-    "../units:timestamp",
-  ]
-}
-
-rtc_source_set("video_frame_type") {
-  visibility = [ "*" ]
-  sources = [ "video_frame_type.h" ]
-}
-
-rtc_library("video_frame_i420") {
-  visibility = [ "*" ]
-  sources = [
-    "i420_buffer.cc",
-    "i420_buffer.h",
-  ]
-  deps = [
-    ":video_frame",
-    ":video_rtp_headers",
-    "..:scoped_refptr",
-    "../../rtc_base",
-    "../../rtc_base:checks",
-    "../../rtc_base/memory:aligned_malloc",
-    "../../rtc_base/system:rtc_export",
-    "//third_party/libyuv",
-  ]
+  sources = []
+  deps = [ ":video_frame" ]
 }
 
 rtc_library("video_frame_i010") {
@@ -116,7 +89,6 @@ rtc_library("video_frame_i010") {
   ]
   deps = [
     ":video_frame",
-    ":video_frame_i420",
     ":video_rtp_headers",
     "..:scoped_refptr",
     "../../rtc_base",
@@ -134,7 +106,6 @@ rtc_library("video_frame_nv12") {
   ]
   deps = [
     ":video_frame",
-    ":video_frame_i420",
     "..:scoped_refptr",
     "../../rtc_base",
     "../../rtc_base:checks",
@@ -144,6 +115,26 @@ rtc_library("video_frame_nv12") {
   ]
 }
 
+rtc_source_set("recordable_encoded_frame") {
+  visibility = [ "*" ]
+  sources = [ "recordable_encoded_frame.h" ]
+
+  deps = [
+    ":encoded_image",
+    ":video_frame",
+    ":video_rtp_headers",
+    "..:array_view",
+    "..:scoped_refptr",
+    "../../rtc_base:refcount",
+    "../units:timestamp",
+  ]
+}
+
+rtc_source_set("video_frame_type") {
+  visibility = [ "*" ]
+  sources = [ "video_frame_type.h" ]
+}
+
 rtc_library("encoded_image") {
   visibility = [ "*" ]
   sources = [

api/video/nv12_buffer.cc

@@ -14,6 +14,7 @@
 #include "rtc_base/checks.h"
 #include "rtc_base/ref_counted_object.h"
 #include "third_party/libyuv/include/libyuv/convert.h"
+#include "third_party/libyuv/include/libyuv/scale.h"
 
 namespace webrtc {
 
@@ -122,4 +123,35 @@ void NV12Buffer::InitializeData() {
   memset(data_.get(), 0, NV12DataSize(height_, stride_y_, stride_uv_));
 }
 
+void NV12Buffer::CropAndScaleFrom(const NV12BufferInterface& src,
+                                  int offset_x,
+                                  int offset_y,
+                                  int crop_width,
+                                  int crop_height) {
+  RTC_CHECK_LE(crop_width, src.width());
+  RTC_CHECK_LE(crop_height, src.height());
+  RTC_CHECK_LE(crop_width + offset_x, src.width());
+  RTC_CHECK_LE(crop_height + offset_y, src.height());
+  RTC_CHECK_GE(offset_x, 0);
+  RTC_CHECK_GE(offset_y, 0);
+
+  // Make sure offset is even so that u/v plane becomes aligned.
+  const int uv_offset_x = offset_x / 2;
+  const int uv_offset_y = offset_y / 2;
+  offset_x = uv_offset_x * 2;
+  offset_y = uv_offset_y * 2;
+
+  const uint8_t* y_plane = src.DataY() + src.StrideY() * offset_y + offset_x;
+  const uint8_t* uv_plane =
+      src.DataUV() + src.StrideUV() * uv_offset_y + uv_offset_x * 2;
+
+  // kFilterBox is unsupported in libyuv, so using kFilterBilinear instead.
+  int res = libyuv::NV12Scale(y_plane, src.StrideY(), uv_plane, src.StrideUV(),
+                              crop_width, crop_height, MutableDataY(),
+                              StrideY(), MutableDataUV(), StrideUV(), width(),
+                              height(), libyuv::kFilterBilinear);
+
+  RTC_DCHECK_EQ(res, 0);
+}
+
 }  // namespace webrtc

api/video/nv12_buffer.h

@@ -56,6 +56,14 @@ class RTC_EXPORT NV12Buffer : public NV12BufferInterface {
   // are resolved in a better way. Or in the mean time, use SetBlack.
   void InitializeData();
 
+  // Scale the cropped area of |src| to the size of |this| buffer, and
+  // write the result into |this|.
+  void CropAndScaleFrom(const NV12BufferInterface& src,
+                        int offset_x,
+                        int offset_y,
+                        int crop_width,
+                        int crop_height);
+
  protected:
   NV12Buffer(int width, int height);
   NV12Buffer(int width, int height, int stride_y, int stride_uv);

api/video/test/BUILD.gn

@@ -20,7 +20,6 @@ rtc_library("rtc_api_video_unittests") {
     "..:video_adaptation",
     "..:video_bitrate_allocation",
     "..:video_frame",
-    "..:video_frame_i420",
     "..:video_frame_nv12",
     "..:video_rtp_headers",
     "../../../test:frame_utils",

api/video/video_frame_buffer.cc

@@ -10,10 +10,25 @@
 
 #include "api/video/video_frame_buffer.h"
 
+#include "api/video/i420_buffer.h"
 #include "rtc_base/checks.h"
 
 namespace webrtc {
 
+rtc::scoped_refptr<VideoFrameBuffer> VideoFrameBuffer::CropAndScale(
+    int offset_x,
+    int offset_y,
+    int crop_width,
+    int crop_height,
+    int scaled_width,
+    int scaled_height) {
+  rtc::scoped_refptr<I420Buffer> result =
+      I420Buffer::Create(scaled_width, scaled_height);
+  result->CropAndScaleFrom(*this->ToI420(), offset_x, offset_y, crop_width,
+                           crop_height);
+  return result;
+}
+
 const I420BufferInterface* VideoFrameBuffer::GetI420() const {
   // Overridden by subclasses that can return an I420 buffer without any
   // conversion, in particular, I420BufferInterface.
@@ -124,5 +139,4 @@ int NV12BufferInterface::ChromaWidth() const {
 int NV12BufferInterface::ChromaHeight() const {
   return (height() + 1) / 2;
 }
-
 }  // namespace webrtc

api/video/video_frame_buffer.h

@@ -79,6 +79,24 @@ class RTC_EXPORT VideoFrameBuffer : public rtc::RefCountInterface {
   // behave as the other GetXXX methods below.
   virtual const I420BufferInterface* GetI420() const;
 
+  // A format specific scale function. Default implementation works by
+  // converting to I420. But more efficient implementations may override it,
+  // especially for kNative.
+  // First, the image is cropped to |crop_width| and |crop_height| and then
+  // scaled to |scaled_width| and |scaled_height|.
+  virtual rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(int offset_x,
+                                                            int offset_y,
+                                                            int crop_width,
+                                                            int crop_height,
+                                                            int scaled_width,
+                                                            int scaled_height);
+
+  // Alias for common use case.
+  rtc::scoped_refptr<VideoFrameBuffer> Scale(int scaled_width,
+                                             int scaled_height) {
+    return CropAndScale(0, 0, width(), height(), scaled_width, scaled_height);
+  }
+
   // These functions should only be called if type() is of the correct type.
   // Calling with a different type will result in a crash.
   const I420ABufferInterface* GetI420A() const;

api/video_codecs/BUILD.gn

@@ -137,7 +137,7 @@ rtc_library("rtc_software_fallback_wrappers") {
   deps = [
     ":video_codecs_api",
     "..:fec_controller_api",
-    "../../api/video:video_frame_i420",
+    "../../api/video:video_frame",
     "../../media:rtc_h264_profile_id",
     "../../media:rtc_media_base",
     "../../modules/video_coding:video_codec_interface",

api/video_codecs/test/BUILD.gn

@@ -36,7 +36,6 @@ if (rtc_include_tests) {
       "../../video:encoded_image",
       "../../video:video_bitrate_allocation",
       "../../video:video_frame",
-      "../../video:video_frame_i420",
       "../../video:video_rtp_headers",
       "//testing/gtest",
     ]

common_video/BUILD.gn

@@ -49,7 +49,6 @@ rtc_library("common_video") {
     "../api/video:video_bitrate_allocation",
     "../api/video:video_bitrate_allocator",
     "../api/video:video_frame",
-    "../api/video:video_frame_i420",
     "../api/video:video_frame_nv12",
     "../api/video:video_rtp_headers",
     "../api/video_codecs:bitstream_parser_api",
@@ -105,7 +104,7 @@ if (rtc_include_tests) {
       "../api/units:time_delta",
       "../api/video:video_frame",
       "../api/video:video_frame_i010",
-      "../api/video:video_frame_i420",
+      "../api/video:video_frame_nv12",
       "../api/video:video_rtp_headers",
       "../media:rtc_h264_profile_id",
       "../rtc_base",

common_video/video_frame_unittest.cc

@@ -15,6 +15,7 @@
 
 #include "api/video/i010_buffer.h"
 #include "api/video/i420_buffer.h"
+#include "api/video/nv12_buffer.h"
 #include "rtc_base/bind.h"
 #include "rtc_base/time_utils.h"
 #include "test/fake_texture_frame.h"
@@ -157,6 +158,29 @@ rtc::scoped_refptr<PlanarYuvBuffer> CreateGradient(VideoFrameBuffer::Type type,
   return I010Buffer::Copy(*buffer);
 }
 
+rtc::scoped_refptr<NV12BufferInterface> CreateNV12Gradient(int width,
+                                                           int height) {
+  rtc::scoped_refptr<NV12Buffer> buffer(NV12Buffer::Create(width, height));
+  // Initialize with gradient, Y = 128(x/w + y/h), U = 256 x/w, V = 256 y/h
+  for (int x = 0; x < width; x++) {
+    for (int y = 0; y < height; y++) {
+      buffer->MutableDataY()[x + y * width] =
+          128 * (x * height + y * width) / (width * height);
+    }
+  }
+  int chroma_width = buffer->ChromaWidth();
+  int chroma_height = buffer->ChromaHeight();
+  for (int x = 0; x < chroma_width; x++) {
+    for (int y = 0; y < chroma_height; y++) {
+      buffer->MutableDataUV()[x * 2 + y * buffer->StrideUV()] =
+          255 * x / (chroma_width - 1);
+      buffer->MutableDataUV()[x * 2 + 1 + y * buffer->StrideUV()] =
+          255 * y / (chroma_height - 1);
+    }
+  }
+  return buffer;
+}
+
 // The offsets and sizes describe the rectangle extracted from the
 // original (gradient) frame, in relative coordinates where the
 // original frame correspond to the unit square, 0.0 <= x, y < 1.0.
@@ -495,6 +519,35 @@ INSTANTIATE_TEST_SUITE_P(All,
                          ::testing::Values(VideoFrameBuffer::Type::kI420,
                                            VideoFrameBuffer::Type::kI010));
 
+TEST(TestNV12Buffer, CropAndScale) {
+  const int kSourceWidth = 640;
+  const int kSourceHeight = 480;
+  const int kScaledWidth = 320;
+  const int kScaledHeight = 240;
+  const int kCropLeft = 40;
+  const int kCropTop = 30;
+  const int kCropRight = 0;
+  const int kCropBottom = 30;
+
+  rtc::scoped_refptr<VideoFrameBuffer> buf =
+      CreateNV12Gradient(kSourceWidth, kSourceHeight);
+
+  rtc::scoped_refptr<VideoFrameBuffer> scaled_buffer = buf->CropAndScale(
+      kCropLeft, kCropTop, kSourceWidth - kCropLeft - kCropRight,
+      kSourceHeight - kCropTop - kCropBottom, kScaledWidth, kScaledHeight);
+
+  // Parameters to CheckCrop indicate what part of the source frame is in the
+  // scaled frame.
+  const float kOffsetX = (kCropLeft + 0.0) / kSourceWidth;
+  const float kOffsetY = (kCropTop + 0.0) / kSourceHeight;
+  const float kRelativeWidth =
+      (kSourceWidth - kCropLeft - kCropRight + 0.0) / kSourceWidth;
+  const float kRelativeHeight =
+      (kSourceHeight - kCropTop - kCropBottom + 0.0) / kSourceHeight;
+  CheckCrop(*scaled_buffer->ToI420(), kOffsetX, kOffsetY, kRelativeWidth,
+            kRelativeHeight);
+}
+
 class TestPlanarYuvBufferRotate
     : public ::testing::TestWithParam<
           std::tuple<webrtc::VideoRotation, VideoFrameBuffer::Type>> {};