HW video acceleration using Media SDK refactoring.

Author: taste1981

talk/owt/BUILD.gn

@@ -179,6 +179,7 @@ static_library("owt_sdk_base") {
 
   if (owt_msdk_header_root != "") {
     include_dirs += [ owt_msdk_header_root ]
+    defines += [ "OWT_USE_MSDK" ]
   }
 
   if (rtc_include_internal_audio_device) {
@@ -187,6 +188,7 @@ static_library("owt_sdk_base") {
   if (rtc_use_x11) {
     defines += [ "WEBRTC_USE_X11" ]
   }
+
   if (!rtc_use_h265) {
     defines += [ "DISABLE_H265" ]
   } else {
@@ -214,7 +216,11 @@ static_library("owt_sdk_base") {
       "sdk/base/win/base_allocator.h",
       "sdk/base/win/d3d_allocator.cc",
       "sdk/base/win/d3d_allocator.h",
+      "sdk/base/win/d3d11_allocator.cc",
+      "sdk/base/win/d3d11_allocator.h",
       "sdk/base/win/d3dnativeframe.h",
+      "sdk/base/win/mediacapabilities.cc",
+      "sdk/base/win/mediacapabilities.h",
       "sdk/base/win/msdkvideobase.cc",
       "sdk/base/win/msdkvideobase.h",
       "sdk/base/win/msdkvideodecoder.cc",
@@ -229,10 +235,12 @@ static_library("owt_sdk_base") {
       "sdk/base/win/sysmem_allocator.h",
       "sdk/base/win/videorendererwin.cc",
       "sdk/base/win/videorendererwin.h",
+      "sdk/base/win/videorendererd3d11.cc",
+      "sdk/base/win/videorendererd3d11.h",
       "sdk/base/win/vp9ratecontrol.cc",
       "sdk/base/win/vp9ratecontrol.h",
-      #"sdk/base/win/vp9temporallayers.cc",
-      #"sdk/base/win/vp9temporallayers.h",
+      "sdk/base/win/vp9temporallayers.cc",
+      "sdk/base/win/vp9temporallayers.h",
       "sdk/base/windowcapturer.cc",
     ]
     public_deps += [ "//third_party/webrtc/modules/audio_device:audio_device_module_from_input_and_output" ]

talk/owt/sdk/base/mediautils.cc

@@ -9,28 +9,42 @@
 #include "webrtc/common_video/h264/prefix_parser.h"
 #include "webrtc/rtc_base/checks.h"
 #include "webrtc/rtc_base/bit_buffer.h"
+#include "webrtc/rtc_base/string_to_number.h"
 #include "system_wrappers/include/field_trial.h"
 #include "talk/owt/sdk/base/mediautils.h"
+#include "talk/owt/sdk/base/win/mediacapabilities.h"
 
 namespace owt {
 namespace base {
+
+const char kAV1FmtpProfileId[] = "profile";
+const char kAV1FmtpLevelId[] = "level-idx";
+
+const char kHEVCFmtpProfileId[] = "profile-id";
+const char kHEVCFmtpLevelId[] = "level-id";
+
 static const std::map<const std::string, const Resolution> resolution_name_map = {
     {"cif", Resolution(352, 288)},
     {"vga", Resolution(640, 480)},
     {"hd720p", Resolution(1280, 720)},
-    {"hd1080p", Resolution(1920, 1080)}};
+    {"hd1080p", Resolution(1920, 1080)},
+    {"4k", Resolution(3840, 2160)},
+    {"8k", Resolution(7680, 4320)}};
+
 static const std::map<const std::string, const AudioCodec>
     audio_codec_names = {
         {"opus", AudioCodec::kOpus}, {"isac", AudioCodec::kIsac},
         {"g722", AudioCodec::kG722}, {"pcmu", AudioCodec::kPcmu},
         {"pcma", AudioCodec::kPcma}, {"ilbc", AudioCodec::kIlbc},
         {"aac", AudioCodec::kAac},   {"ac3", AudioCodec::kAc3},
         {"asao", AudioCodec::kAsao}, {"unknown", AudioCodec::kUnknown}};
+
 static const std::map<const std::string, const VideoCodec>
     video_codec_names = {{"vp8", VideoCodec::kVp8},
                          {"vp9", VideoCodec::kVp9},
                          {"h264", VideoCodec::kH264},
                          {"h265", VideoCodec::kH265}};
+
 std::string MediaUtils::GetResolutionName(const Resolution& resolution) {
   for (auto it = resolution_name_map.begin(); it != resolution_name_map.end();
        ++it) {
@@ -40,6 +54,7 @@ std::string MediaUtils::GetResolutionName(const Resolution& resolution) {
   }
   return "";
 }
+
 AudioCodec MediaUtils::GetAudioCodecFromString(const std::string& codec_name) {
   auto it = audio_codec_names.find(codec_name);
   if (it != audio_codec_names.end()) {
@@ -127,5 +142,33 @@ bool MediaUtils::GetH264TemporalInfo(uint8_t* buffer, size_t buffer_length,
   }
   return prefix_nal_found;
 }
+
+absl::optional<AV1Profile> StringToAV1Profile(const std::string& str) {
+  const absl::optional<int> i = rtc::StringToNumber<int>(str);
+  if (!i.has_value())
+    return absl::nullopt;
+
+  switch (i.value()) {
+    case 0:
+      return AV1Profile::kMain;
+    case 1:
+      return AV1Profile::kHigh;
+    case 2:
+      return AV1Profile::kProfessional;
+    default:
+      return absl::nullopt;
+  }
+  return absl::nullopt;
+}
+
+absl::optional<AV1Profile> MediaUtils::ParseSdpForAV1Profile(
+    const webrtc::SdpVideoFormat::Parameters& params) {
+  const auto profile_it = params.find(kAV1FmtpProfileId);
+  if (profile_it == params.end())
+    return AV1Profile::kMain;
+  const std::string& profile_str = profile_it->second;
+  return StringToAV1Profile(profile_str);
+}
+
 }  // namespace base
 }  // namespace owt

talk/owt/sdk/base/mediautils.h

@@ -1,13 +1,20 @@
 // Copyright (C) <2018> Intel Corporation
 //
 // SPDX-License-Identifier: Apache-2.0
+
 #ifndef OWT_BASE_MEDIAUTILS_H_
 #define OWT_BASE_MEDIAUTILS_H_
+
 #include "absl/types/optional.h"
+#include "api/video_codecs/sdp_video_format.h"
+#include <string>
+#include "talk/owt/sdk/base/win/mediacapabilities.h"
 #include "talk/owt/sdk/include/cpp/owt/base/commontypes.h"
 
 namespace owt {
 namespace base {
+
+
 class MediaUtils {
  public:
   static std::string GetResolutionName(const Resolution& resolution);
@@ -21,6 +28,8 @@ class MediaUtils {
                                   int& temporal_id,
                                   int& priority_id,
                                   bool& is_idr);
+  static absl::optional<AV1Profile> ParseSdpForAV1Profile(
+      const webrtc::SdpVideoFormat::Parameters& params);
 };
 }
 }

talk/owt/sdk/base/sysinfo.cc

@@ -5,6 +5,10 @@
 #include <sys/utsname.h>
 #endif  // WEBRTC_LINUX
 #include "webrtc/rtc_base/checks.h"
+#if defined(WEBRTC_WIN)
+#include <memory>
+#include "webrtc/rtc_base/win/windows_version.h"
+#endif
 #include "talk/owt/sdk/base/sysinfo.h"
 namespace owt {
 namespace base {
@@ -20,7 +24,40 @@ std::string SysInfo::SdkType() {
 #endif
 }
 #if defined(WEBRTC_WIN)
+using namespace rtc::rtc_win;
 std::string SysInfo::OsName() {
+  Version version = GetVersion();
+  switch (version) {
+    case VERSION_PRE_XP:
+      return "Pre-Windows XP";
+    case VERSION_XP:
+      return "Windows XP";
+    case VERSION_SERVER_2003:
+      return "Windows Server 2003";
+    case VERSION_VISTA:
+      return "Windows Vista";
+    case VERSION_WIN7:
+      return "Windows 7";
+    case VERSION_WIN8:
+      return "Windows 8";
+    case VERSION_WIN8_1:
+      return "Windows 8.1";  // May also be Windows Server 2012 R1.
+    // Be noted if the application is not manifest for Win8.1 or Win10, it will aways
+    // Win8 for version higher than that.
+    case VERSION_WIN10:      // TH 10586
+    case VERSION_WIN10_TH2:  // TH 14393
+      return "Windows 10 Threshold";
+    case VERSION_WIN10_RS1:
+      return "Windows 10 RS1";  // Redstone 1
+    case VERSION_WIN10_RS2:
+      return "Windows 10 RS2";  // Redstone 2
+    case VERSION_WIN10_RS3:
+      return "Windows 10 RS3";  // Redstone 3, Build 16299
+    case VERSION_WIN10_RS4:
+      return "Windows 10 RS4";  // Redstone 4, Version 1803.
+    default:
+      return kUnknown;
+  }
   return "Windows NT";
 }
 std::string SysInfo::OsVersion() {

talk/owt/sdk/base/win/d3dnativeframe.h

@@ -17,6 +17,6 @@ struct NativeD3DSurfaceHandle {
   int width_;   // width of the frame passing from decoder
   int height_;  // height of the frame passing from decoder
 };
-}
+
 }
 #endif  // OWT_BASE_WIN_D3DNATIVEFRAME_H