Support hardware accelerated FFMPEG decoder. (#711)

This change enables external video decoder factory for both MSDK decoder
and FFMPEG decoder.

Author: jianjunz

talk/owt/BUILD.gn

@@ -295,6 +295,17 @@ static_library("owt_sdk_base") {
       "sdk/base/win/videorendererd3d11.cc",
       "sdk/base/win/videorendererd3d11.h",
     ]
+    if (owt_msdk_header_root != "" || owt_ffmpeg_root != "") {
+      sources += [
+        "sdk/base/win/externalvideodecoderfactory.cc",
+        "sdk/base/win/externalvideodecoderfactory.h",
+        "sdk/base/win/externalvideoencoderfactory.cc",
+        "sdk/base/win/externalvideoencoderfactory.h",
+        "sdk/base/win/mediacapabilities.cc",
+        "sdk/base/win/mediacapabilities.h",
+        "sdk/base/win/vpedefs.h",
+      ]
+    }
     if (owt_msdk_header_root != "") {
       sources += [
         "sdk/base/win/base_allocator.cc",
@@ -303,23 +314,16 @@ static_library("owt_sdk_base") {
         "sdk/base/win/d3d11_allocator.h",
         "sdk/base/win/d3d_allocator.cc",
         "sdk/base/win/d3d_allocator.h",
-        "sdk/base/win/mediacapabilities.cc",
-        "sdk/base/win/mediacapabilities.h",
         "sdk/base/win/msdkvideobase.cc",
         "sdk/base/win/msdkvideobase.h",
         "sdk/base/win/msdkvideodecoder.cc",
         "sdk/base/win/msdkvideodecoder.h",
-        "sdk/base/win/msdkvideodecoderfactory.cc",
-        "sdk/base/win/msdkvideodecoderfactory.h",
         "sdk/base/win/msdkvideoencoder.cc",
         "sdk/base/win/msdkvideoencoder.h",
-        "sdk/base/win/msdkvideoencoderfactory.cc",
-        "sdk/base/win/msdkvideoencoderfactory.h",
         "sdk/base/win/sysmem_allocator.cc",
         "sdk/base/win/sysmem_allocator.h",
         "sdk/base/win/vp9ratecontrol.cc",
         "sdk/base/win/vp9ratecontrol.h",
-        "sdk/base/win/vpedefs.h",
       ]
       public_deps += [
         "//third_party/libvpx",

talk/owt/sdk/base/linux/msdkvideodecoderfactory.cc

@@ -15,18 +15,18 @@
 namespace owt {
 namespace base {
 
-MSDKVideoDecoderFactory::MSDKVideoDecoderFactory() {
+ExternalVideoDecoderFactory::ExternalVideoDecoderFactory() {
   // TODO: Add the other codecs support
   supported_codecs_.push_back(webrtc::kVideoCodecH264);
   supported_codecs_.push_back(webrtc::kVideoCodecH265);
 }
 
 
 
-MSDKVideoDecoderFactory::~MSDKVideoDecoderFactory() {}
+ExternalVideoDecoderFactory::~ExternalVideoDecoderFactory() {}
 
 
-std::unique_ptr<webrtc::VideoDecoder> MSDKVideoDecoderFactory::CreateVideoDecoder(
+std::unique_ptr<webrtc::VideoDecoder> ExternalVideoDecoderFactory::CreateVideoDecoder(
     const webrtc::SdpVideoFormat& format) {
 
   if (absl::EqualsIgnoreCase(format.name, cricket::kVp8CodecName) ||
@@ -42,7 +42,7 @@ std::unique_ptr<webrtc::VideoDecoder> MSDKVideoDecoderFactory::CreateVideoDecode
 
   return nullptr;
 }
-std::vector<webrtc::SdpVideoFormat> MSDKVideoDecoderFactory::GetSupportedFormats()
+std::vector<webrtc::SdpVideoFormat> ExternalVideoDecoderFactory::GetSupportedFormats()
     const {
 
   std::vector<webrtc::SdpVideoFormat> supported_codecs;

talk/owt/sdk/base/linux/msdkvideodecoderfactory.h

@@ -14,10 +14,10 @@ namespace owt {
 namespace base {
 
 // Declaration of MSDK based decoder factory.
-class MSDKVideoDecoderFactory : public webrtc::VideoDecoderFactory {
+class ExternalVideoDecoderFactory : public webrtc::VideoDecoderFactory {
  public:
-  MSDKVideoDecoderFactory();
-  virtual ~MSDKVideoDecoderFactory();
+  ExternalVideoDecoderFactory();
+  virtual ~ExternalVideoDecoderFactory();
 
   // WebRtcVideoDecoderFactory implementation.
   // VideoDecoderFactory implementation

talk/owt/sdk/base/peerconnectiondependencyfactory.cc

@@ -22,8 +22,8 @@
 #include "webrtc/rtc_base/physical_socket_server.h"
 #if defined(WEBRTC_WIN)
 #include <d3d11.h>
-#include "talk/owt/sdk/base/win/msdkvideodecoderfactory.h"
-#include "talk/owt/sdk/base/win/msdkvideoencoderfactory.h"
+#include "talk/owt/sdk/base/win/externalvideodecoderfactory.h"
+#include "talk/owt/sdk/base/win/externalvideoencoderfactory.h"
 #elif defined(WEBRTC_LINUX)
 #include "talk/owt/sdk/base/linux/msdkvideodecoderfactory.h"
 #elif defined(WEBRTC_IOS)
@@ -190,8 +190,8 @@ void PeerConnectionDependencyFactory::
   if (encoded_frame_) {
     encoder_factory.reset(new EncodedVideoEncoderFactory());
   } else if (render_hardware_acceleration_enabled_) {
-#ifdef OWT_USE_MSDK
-    encoder_factory.reset(new MSDKVideoEncoderFactory());
+#if defined(OWT_USE_MSDK) || defined(OWT_USE_FFMPEG)
+    encoder_factory.reset(new ExternalVideoEncoderFactory());
 #else
     // For Linux HW encoder pending verification.
     encoder_factory = webrtc::CreateBuiltinVideoEncoderFactory();
@@ -204,8 +204,8 @@ void PeerConnectionDependencyFactory::
     decoder_factory.reset(new CustomizedVideoDecoderFactory(
         GlobalConfiguration::GetCustomizedVideoDecoder()));
   } else if (render_hardware_acceleration_enabled_) {
-#ifdef OWT_USE_MSDK
-    decoder_factory.reset(new MSDKVideoDecoderFactory(nullptr));
+#if defined(OWT_USE_MSDK) || defined(OWT_USE_FFMPEG)
+    decoder_factory.reset(new ExternalVideoDecoderFactory(nullptr));
 #else
     decoder_factory = webrtc::CreateBuiltinVideoDecoderFactory();
 #endif

talk/owt/sdk/base/win/d3d11va_h264_decoder.cc

@@ -97,7 +97,7 @@ int H264DXVADecoderImpl::InitHwContext(AVCodecContext* ctx,
   device_hwctx = (AVD3D11VADeviceContext*)device_ctx->hwctx;
 
   hr = D3D11CreateDevice(
-      nullptr, D3D_DRIVER_TYPE_UNKNOWN, nullptr, creation_flags, feature_levels,
+      nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, creation_flags, feature_levels,
       sizeof(feature_levels) / sizeof(feature_levels[0]), D3D11_SDK_VERSION,
       &device_hwctx->device, &feature_levels_out, &d3d11_device_context_);
   if (FAILED(hr)) {
@@ -317,8 +317,8 @@ int32_t H264DXVADecoderImpl::Decode(const webrtc::EncodedImage& input_image,
     // We get one frame from the decoder.
     if (frame != nullptr && frame->format == hw_pix_fmt) {
       ID3D11Texture2D* texture = (ID3D11Texture2D*)frame->data[0];
-      int width = frame->crop_right - frame->crop_left;
-      int height = frame->crop_bottom - frame->crop_top;
+      int width = frame->width;
+      int height = frame->height;
       int index = (intptr_t)frame->data[1];
       D3D11_TEXTURE2D_DESC texture_desc;
 

talk/owt/sdk/base/win/msdkvideodecoderfactory.cc renamed to talk/owt/sdk/base/win/externalvideodecoderfactory.cc

@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-#include "talk/owt/sdk/base/win/msdkvideodecoderfactory.h"
+#include "talk/owt/sdk/base/win/externalvideodecoderfactory.h"
 #include <vector>
 #include "absl/strings/match.h"
 #include "media/base/codec.h"
@@ -11,16 +11,19 @@
 #include "modules/video_coding/codecs/vp8/include/vp8.h"
 #include "modules/video_coding/codecs/vp9/include/vp9.h"
 #include "talk/owt/sdk/base/codecutils.h"
-#include "talk/owt/sdk/base/win/msdkvideodecoder.h"
-#include "webrtc/rtc_base/checks.h"
 #ifdef OWT_USE_FFMPEG
 #include "talk/owt/sdk/base/win/d3d11va_h264_decoder.h"
 #endif
+#ifdef OWT_USE_MSDK
+#include "talk/owt/sdk/base/win/msdkvideodecoder.h"
+#endif
+#include "webrtc/rtc_base/checks.h"
+#include "webrtc/rtc_base/logging.h"
 
 namespace owt {
 namespace base {
 
-MSDKVideoDecoderFactory::MSDKVideoDecoderFactory(ID3D11Device* d3d11_device_external) {
+ExternalVideoDecoderFactory::ExternalVideoDecoderFactory(ID3D11Device* d3d11_device_external) {
   supported_codec_types_.clear();
 
   bool is_vp8_hw_supported = false, is_vp9_hw_supported = false;
@@ -66,10 +69,10 @@ MSDKVideoDecoderFactory::MSDKVideoDecoderFactory(ID3D11Device* d3d11_device_exte
   }
 }
 
-MSDKVideoDecoderFactory::~MSDKVideoDecoderFactory() {}
+ExternalVideoDecoderFactory::~ExternalVideoDecoderFactory() {}
 
 std::unique_ptr<webrtc::VideoDecoder>
-MSDKVideoDecoderFactory::CreateVideoDecoder(
+ExternalVideoDecoderFactory::CreateVideoDecoder(
     const webrtc::SdpVideoFormat& format) {
   bool vp9_hw = false, vp8_hw = false, av1_hw = false, h264_hw = false;
 #ifdef WEBRTC_USE_H265
@@ -121,10 +124,10 @@ MSDKVideoDecoderFactory::CreateVideoDecoder(
   }
 #endif
 
-  return MSDKVideoDecoder::Create(cricket::VideoCodec(format));
+  RTC_CHECK_NOTREACHED();
 }
 
- std::vector<webrtc::SdpVideoFormat> MSDKVideoDecoderFactory::GetSupportedFormats()
+ std::vector<webrtc::SdpVideoFormat> ExternalVideoDecoderFactory::GetSupportedFormats()
     const {
   std::vector<webrtc::SdpVideoFormat> supported_codecs;
   supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kVp8CodecName));

talk/owt/sdk/base/win/msdkvideodecoderfactory.h renamed to talk/owt/sdk/base/win/externalvideodecoderfactory.h

@@ -20,11 +20,11 @@
 
 namespace owt {
 namespace base {
-// Declaration of MSDK based decoder factory.
-class MSDKVideoDecoderFactory : public webrtc::VideoDecoderFactory {
+// Decoder factory for MSDK decoder and FFMPEG decoder.
+class ExternalVideoDecoderFactory : public webrtc::VideoDecoderFactory {
  public:
-  MSDKVideoDecoderFactory(ID3D11Device* d3d11_device_external);
-  virtual ~MSDKVideoDecoderFactory();
+  ExternalVideoDecoderFactory(ID3D11Device* d3d11_device_external);
+  virtual ~ExternalVideoDecoderFactory();
 
   // VideoDecoderFactory implementation
   std::vector<webrtc::SdpVideoFormat> GetSupportedFormats() const override;

talk/owt/sdk/base/win/msdkvideoencoderfactory.cc renamed to talk/owt/sdk/base/win/externalvideoencoderfactory.cc

@@ -2,21 +2,23 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-#include "talk/owt/sdk/base/win/msdkvideoencoderfactory.h"
+#include "talk/owt/sdk/base/win/externalvideoencoderfactory.h"
 #include <string>
 #include "absl/strings/match.h"
 #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
 #include "modules/video_coding/codecs/h264/include/h264.h"
 #include "modules/video_coding/codecs/vp8/include/vp8.h"
 #include "modules/video_coding/codecs/vp9/include/vp9.h"
 #include "talk/owt/sdk/base/codecutils.h"
+#ifdef OWT_USE_MSDK
 #include "talk/owt/sdk/base/win/msdkvideoencoder.h"
+#endif
 #include "webrtc/api/video_codecs/vp9_profile.h"
 
 namespace owt {
 namespace base {
 
-MSDKVideoEncoderFactory::MSDKVideoEncoderFactory() {
+ExternalVideoEncoderFactory::ExternalVideoEncoderFactory() {
   supported_codec_types_.clear();
   MediaCapabilities* media_capability = MediaCapabilities::Get();
   std::vector<owt::base::VideoCodec> codecs_to_check;
@@ -39,7 +41,7 @@ MSDKVideoEncoderFactory::MSDKVideoEncoderFactory() {
   supported_codec_types_.push_back(webrtc::kVideoCodecAV1);
 }
 
-std::unique_ptr<webrtc::VideoEncoder> MSDKVideoEncoderFactory::CreateVideoEncoder(
+std::unique_ptr<webrtc::VideoEncoder> ExternalVideoEncoderFactory::CreateVideoEncoder(
     const webrtc::SdpVideoFormat& format) {
   bool vp9_hw = false, vp8_hw = false, av1_hw = false, h264_hw = false;
 #ifdef WEBRTC_USE_H265
@@ -73,11 +75,11 @@ std::unique_ptr<webrtc::VideoEncoder> MSDKVideoEncoderFactory::CreateVideoEncode
            !h265_hw) {
   }
 #endif
-  return MSDKVideoEncoder::Create(cricket::VideoCodec(format));
+  RTC_CHECK_NOTREACHED();
 }
 
 std::vector<webrtc::SdpVideoFormat>
-MSDKVideoEncoderFactory::GetSupportedFormats() const {
+ExternalVideoEncoderFactory::GetSupportedFormats() const {
   std::vector<webrtc::SdpVideoFormat> supported_codecs;
   supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kVp8CodecName));
   supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kAv1CodecName));

talk/owt/sdk/base/win/msdkvideoencoderfactory.h renamed to talk/owt/sdk/base/win/externalvideoencoderfactory.h

@@ -14,10 +14,11 @@
 
 namespace owt {
 namespace base {
-class MSDKVideoEncoderFactory : public webrtc::VideoEncoderFactory {
+// Encoder factory for MSDK encoder and FFMPEG encoder.
+class ExternalVideoEncoderFactory : public webrtc::VideoEncoderFactory {
  public:
-  MSDKVideoEncoderFactory();
-  ~MSDKVideoEncoderFactory() {}
+  ExternalVideoEncoderFactory();
+  ~ExternalVideoEncoderFactory() {}
   std::unique_ptr<webrtc::VideoEncoder> CreateVideoEncoder(
       const webrtc::SdpVideoFormat& format) override;