Integrate libvpx rate controller with MSDK for VP9

Author: taste1981

talk/owt/sdk/base/mediautils.cc

@@ -161,6 +161,31 @@ absl::optional<AV1Profile> StringToAV1Profile(const std::string& str) {
   return absl::nullopt;
 }
 
+absl::optional<H265ProfileId> StringToH265Profile(const std::string& str) {
+  const absl::optional<int> i = rtc::StringToNumber<int>(str);
+  if (!i.has_value())
+    return absl::nullopt;
+  // See ISO/IEC-23008-2 section A.3.5. we use the general_profile_idc
+  // as the profile-id per RFC 7798.
+  switch (i.value()) {
+    case 1:
+      return H265ProfileId::kMain;
+    case 2:
+      return H265ProfileId::kMain10;
+    case 3:
+      return H265ProfileId::kMainStillPicture;
+    case 4:
+      return H265ProfileId::kMainRExt;
+#if (MFX_VERSION >= MFX_VERSION_NEXT)
+    case 5:
+      return H265ProfileId::kScc;
+#endif
+    default:
+      return absl::nullopt;
+  }
+  return absl::nullopt;
+}
+
 absl::optional<AV1Profile> MediaUtils::ParseSdpForAV1Profile(
     const webrtc::SdpVideoFormat::Parameters& params) {
   const auto profile_it = params.find(kAV1FmtpProfileId);
@@ -170,5 +195,14 @@ absl::optional<AV1Profile> MediaUtils::ParseSdpForAV1Profile(
   return StringToAV1Profile(profile_str);
 }
 
+absl::optional<H265ProfileId> MediaUtils::ParseSdpForH265Profile(
+    const webrtc::SdpVideoFormat::Parameters& params) {
+  const auto profile_it = params.find(kHEVCFmtpProfileId);
+  if (profile_it == params.end())
+    return H265ProfileId::kMain;
+  const std::string& profile_str = profile_it->second;
+  return StringToH265Profile(profile_str);
+}
+
 }  // namespace base
 }  // namespace owt

talk/owt/sdk/base/mediautils.h

@@ -180,6 +180,8 @@ class MediaUtils {
                                   bool& is_idr);
   static absl::optional<AV1Profile> ParseSdpForAV1Profile(
       const webrtc::SdpVideoFormat::Parameters& params);
+  static absl::optional<H265ProfileId> ParseSdpForH265Profile(
+      const webrtc::SdpVideoFormat::Parameters& params);
 };
 }
 }

talk/owt/sdk/base/stream.cc

@@ -17,7 +17,7 @@
 
 #if defined(WEBRTC_WIN)
 #include "talk/owt/sdk/base/desktopcapturer.h"
-#include "talk/owt/sdk/base/win/videorendererwin.h"
+#include "talk/owt/sdk/base/win/videorendererd3d11.h"
 #include "webrtc/modules/desktop_capture/desktop_capture_options.h"
 #endif
 #if defined(WEBRTC_IOS)
@@ -82,6 +82,7 @@ Stream::Stream()
       renderer_impl_(nullptr),
       audio_renderer_impl_(nullptr),
       d3d9_renderer_impl_(nullptr),
+      d3d11_renderer_impl_(nullptr),
 #ifdef OWT_ENABLE_QUIC
       source_(AudioSourceInfo::kUnknown,
               VideoSourceInfo::kUnknown,
@@ -96,6 +97,7 @@ Stream::Stream(const std::string& id)
       renderer_impl_(nullptr),
       audio_renderer_impl_(nullptr),
       d3d9_renderer_impl_(nullptr),
+      d3d11_renderer_impl_(nullptr),
 #ifdef OWT_ENABLE_QUIC
       source_(AudioSourceInfo::kUnknown,
               VideoSourceInfo::kUnknown,
@@ -271,11 +273,11 @@ void Stream::AttachVideoRenderer(VideoRenderWindow& render_window) {
         << "There are more than one video tracks, the first one "
            "will be attachecd to renderer.";
   }
-  WebrtcVideoRendererD3D9Impl* old_renderer =
-      d3d9_renderer_impl_ ? d3d9_renderer_impl_ : nullptr;
-  d3d9_renderer_impl_ =
-      new WebrtcVideoRendererD3D9Impl(render_window.GetWindowHandle());
-  video_tracks[0]->AddOrUpdateSink(d3d9_renderer_impl_, rtc::VideoSinkWants());
+  WebrtcVideoRendererD3D11Impl* old_renderer =
+      d3d11_renderer_impl_ ? d3d11_renderer_impl_ : nullptr;
+  d3d11_renderer_impl_ =
+      new WebrtcVideoRendererD3D11Impl(render_window.GetWindowHandle());
+  video_tracks[0]->AddOrUpdateSink(d3d11_renderer_impl_, rtc::VideoSinkWants());
   if (old_renderer)
     delete old_renderer;
   RTC_LOG(LS_INFO) << "Attached the stream to a renderer.";
@@ -285,7 +287,7 @@ void Stream::AttachVideoRenderer(VideoRenderWindow& render_window) {
 void Stream::DetachVideoRenderer() {
 #if defined(WEBRTC_WIN)
   if (media_stream_ == nullptr ||
-      (renderer_impl_ == nullptr && d3d9_renderer_impl_ == nullptr))
+      (renderer_impl_ == nullptr && d3d11_renderer_impl_ == nullptr))
     return;
 #elif defined(WEBRTC_LINUX)
   if (media_stream_ == nullptr ||
@@ -305,10 +307,10 @@ void Stream::DetachVideoRenderer() {
     renderer_impl_ = nullptr;
   }
 #if defined(WEBRTC_WIN)
-  if (d3d9_renderer_impl_ != nullptr) {
-    video_tracks[0]->RemoveSink(d3d9_renderer_impl_);
-    delete d3d9_renderer_impl_;
-    d3d9_renderer_impl_ = nullptr;
+  if (d3d11_renderer_impl_ != nullptr) {
+    video_tracks[0]->RemoveSink(d3d11_renderer_impl_);
+    delete d3d11_renderer_impl_;
+    d3d11_renderer_impl_ = nullptr;
   }
 #endif
 #if defined(WEBRTC_LINUX)

talk/owt/sdk/base/win/msdkvideobase.cc

@@ -139,6 +139,13 @@ bool MSDKFactory::LoadDecoderPlugin(uint32_t codec_id,
       }
       *plugin_id = MFX_PLUGINID_VP8D_HW;
       break;
+    case MFX_CODEC_VP9:
+      sts = MFXVideoUSER_Load(*session, &MFX_PLUGINID_VP9D_HW, 1);
+      if (sts != MFX_ERR_NONE) {
+        return false;
+      }
+      *plugin_id = MFX_PLUGINID_VP9D_HW;
+      break;
     default:
       break;
   }

talk/owt/sdk/base/win/msdkvideodecoder.cc

@@ -47,6 +47,7 @@ MSDKVideoDecoder::MSDKVideoDecoder()
   m_pInputSurfaces = nullptr;
   m_video_param_extracted = false;
   m_decBsOffset = 0;
+  inited = false;
   surface_handle.reset(new D3D11ImageHandle());
 }
 
@@ -147,7 +148,8 @@ int32_t MSDKVideoDecoder::InitDecodeOnCodecThread() {
   uint32_t codec_id = MFX_CODEC_AVC;
 
   if (inited) {
-    m_pmfxDEC->Close();
+    if (m_pmfxDEC)
+      m_pmfxDEC->Close();
     MSDK_SAFE_DELETE_ARRAY(m_pInputSurfaces);
 
     if (m_pMFXAllocator) {
@@ -171,6 +173,10 @@ int32_t MSDKVideoDecoder::InitDecodeOnCodecThread() {
       codec_id = MFX_CODEC_AV1;
     }
 
+    if (!factory->LoadDecoderPlugin(codec_id, m_mfxSession, &m_pluginID)) {
+      return WEBRTC_VIDEO_CODEC_ERROR;
+    }
+
     if (!CreateD3D11Device()) {
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
@@ -194,15 +200,18 @@ int32_t MSDKVideoDecoder::InitDecodeOnCodecThread() {
     MSDK_ZERO_MEMORY(m_mfxBS);
     m_mfxBS.Data = new mfxU8[MSDK_BS_INIT_SIZE];
     m_mfxBS.MaxLength = MSDK_BS_INIT_SIZE;
+    RTC_LOG(LS_ERROR) << "Creating underlying MSDK decoder.";
     m_pmfxDEC.reset(new MFXVideoDECODE(*m_mfxSession));
     if (m_pmfxDEC == nullptr) {
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
   }
 
   m_mfxVideoParams.mfx.CodecId = codec_id;
-
+  if (codec_id == MFX_CODEC_VP9)
+    m_mfxVideoParams.mfx.EnableReallocRequest = MFX_CODINGOPTION_ON;
   inited = true;
+  RTC_LOG(LS_ERROR) << "InitDecodeOnCodecThread --";
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
@@ -212,16 +221,19 @@ int32_t MSDKVideoDecoder::Decode(
     int64_t renderTimeMs) {
 
   mfxStatus sts = MFX_ERR_NONE;
-  HRESULT hr;
-  bool device_opened = false;
   mfxFrameSurface1 *pOutputSurface = nullptr;
+
   m_mfxVideoParams.IOPattern =
       MFX_IOPATTERN_OUT_VIDEO_MEMORY;
-  m_mfxVideoParams.AsyncDepth = 1;
+  m_mfxVideoParams.AsyncDepth = 4;
+
   ReadFromInputStream(&m_mfxBS, inputImage.data(), inputImage.size());
 
 dec_header:
   if (inited && !m_video_param_extracted) {
+    if (!m_pmfxDEC.get()) {
+      RTC_LOG(LS_ERROR) << "MSDK decoder not created.";
+    }
     sts = m_pmfxDEC->DecodeHeader(&m_mfxBS, &m_mfxVideoParams);
     if (MFX_ERR_NONE == sts || MFX_WRN_PARTIAL_ACCELERATION == sts) {
       mfxU16 nSurfNum = 0;

talk/owt/sdk/base/win/msdkvideodecoderfactory.cc

@@ -108,7 +108,7 @@ std::unique_ptr<webrtc::VideoDecoder> MSDKVideoDecoderFactory::CreateVideoDecode
  std::vector<webrtc::SdpVideoFormat> MSDKVideoDecoderFactory::GetSupportedFormats()
     const {
   std::vector<webrtc::SdpVideoFormat> supported_codecs;
-  supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kVp8CodecName));
+  //supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kVp8CodecName));
   for (const webrtc::SdpVideoFormat& format : webrtc::SupportedVP9Codecs())
     supported_codecs.push_back(format);
   for (const webrtc::SdpVideoFormat& format : owt::base::CodecUtils::SupportedH264Codecs())