VP9/AV1 hardware acceleration

Author: taste1981

talk/owt/BUILD.gn

@@ -239,8 +239,8 @@ static_library("owt_sdk_base") {
       "sdk/base/win/videorendererd3d11.h",
       "sdk/base/win/vp9ratecontrol.cc",
       "sdk/base/win/vp9ratecontrol.h",
-      "sdk/base/win/vp9temporallayers.cc",
-      "sdk/base/win/vp9temporallayers.h",
+      #"sdk/base/win/vp9temporallayers.cc",
+      #"sdk/base/win/vp9temporallayers.h",
       "sdk/base/windowcapturer.cc",
     ]
     public_deps += [ "//third_party/webrtc/modules/audio_device:audio_device_module_from_input_and_output" ]

talk/owt/sdk/base/codecutils.cc

@@ -38,6 +38,11 @@ std::vector<webrtc::SdpVideoFormat> CodecUtils::GetSupportedH265Codecs() {
                                  {{cricket::kH265FmtpProfileSpace, "0"},
                                   {cricket::kH265FmtpProfileId, "1"},
                                   {cricket::kH265FmtpTierFlag, "0"},
+                                  {cricket::kH265FmtpLevelId, "120"}}),
+          webrtc::SdpVideoFormat(cricket::kH265CodecName,
+                                 {{cricket::kH265FmtpProfileSpace, "0"},
+                                  {cricket::kH265FmtpProfileId, "2"},
+                                  {cricket::kH265FmtpTierFlag, "0"},
                                   {cricket::kH265FmtpLevelId, "120"}})};
 }
 #endif

talk/owt/sdk/base/mediautils.h

@@ -8,12 +8,162 @@
 #include "absl/types/optional.h"
 #include "api/video_codecs/sdp_video_format.h"
 #include <string>
-#include "talk/owt/sdk/base/win/mediacapabilities.h"
 #include "talk/owt/sdk/include/cpp/owt/base/commontypes.h"
 
 namespace owt {
 namespace base {
 
+/// AV1 profiles as negotiated in SDP profile field.
+enum class AV1Profile : int {
+  kMain = 0,     ///< 4:0:0(monochrome) or 4:2:0, 8/10-bit
+  kHigh,         ///< 4:0:0/4:2:0/4:4:4, 8/10-bit
+  kProfessional  ///< 4:0:0/4:2:0/4:2:2/4:4:4, 8/10/12-bit
+};
+
+/// AV1 levels as negotiated in SDP level-idx field.
+/// Calculated by level(x.y)-> level-idx(z): z = (x - 2) * 4 + y;
+enum class AV1Level : int {
+  kLevel2 =
+      0,  ///< 426x240@30fps. All 2.x profiles: max 8 tiles, 4 tile columns.
+  kLevel2_1 = 1,  ///< 640x360@30fps
+  kLevel3 =
+      4,  ///< 854x480@30fps. All 3.x profiles: max 16 tiles, 6 tile columns.
+  kLevel3_1 = 5,  ///< 720p@30fps. Max 16 tiles, 6 tile columns.
+  kLevel4 = 8,  ///< 1080p@30fps. All 4.x profiles: max 32 tiles, 8 tile columns
+  kLevel4_1 = 9,  ///< 1080p@60fps
+  kLevel5 = 12,   ///< 4k@30fps. All 5.x profiles: max 64 tiles, 8 tile columns.
+  kLevel5_1 = 13,  ///< 4k@60fps
+  kLevel5_2 = 14,  ///< 4k@120fps
+  kLevel5_3 = 15,  ///< 4k@120fps
+  kLevel6 = 16,  ///< 8k@30fps. All 6.x profiles: max 128 tiles, 16 tile columns
+  kLevel6_1 = 17,  ///< 8k@60fps
+  kLevel6_2 = 18,  ///< 8k@120fps
+  kLevel6_3 = 19,  ///< 8k@120fps
+  kLevelReserved   ///< Levels not defined yet.
+};
+
+/// AV1 tiers as negotiated in SDP tier field.
+/// This equals to the seq_tier accrording to AV1 spec. Value 1
+/// is only allowed for level above 4.0(inclusive).
+enum class AV1Tier : int { kTier0 = 0, kTier1 };
+
+/// AVC profiles
+enum class H264Profile : int {
+  kConstrainedBaseline = 0,
+  kBaseline,
+  kMain,
+  kHigh,
+  kUnknown
+};
+
+/// VP9 profiles.
+enum class VP9Profile : int {
+  kProfile0 = 0,  ///< 4:2:0, 8-bit
+  kProfile1,      ///< 4:2:2 or 4:4:4, 8-bit
+  kProfile2,      ///< 4:2:0, 10-bit or 12-bit
+  kProfile3       ///< 4:2:2 or 4:4:4, 10-bit or 12-bit
+};
+
+/// Profile space as negotiated in SDP profile-space field.
+/// Default to 0. (Range from 0 to 3).
+enum class H265ProfileSpace : int {
+  KDefault = 0,
+  kReserved1,
+  kReserved2,
+  kReserved3
+};
+
+/// Profile space as negotiated in SDP profile-id field.
+/// Not all possible profiles are listed here.
+enum class H265ProfileId : int {
+  kUnknown = 0,
+  kMain,
+  kMain10,
+  kMainStillPicture,
+  kMainRExt  // Range extension
+#if (MFX_VERSION >= 1032)
+  ,
+  KScc  // Screen content extension.
+#endif
+};
+
+/// Level ID as negotiated in SDP level-id field.
+/// Calculated by: level(x.y) * 30
+enum class H265Level : int {
+  kUnknown = 0,
+  kLevel1 = 30,
+  kLevel2 = 60,
+  kLevle2_1 = 63,
+  kLevel3 = 90,
+  kLevel3_1 = 93,
+  kLevel4 = 120,
+  kLevel4_1 = 123,
+  kLevel5 = 150,
+  kLevel5_1 = 153,
+  kLevel5_2 = 156,
+  kLevel6 = 180,
+  kLevel6_1 = 183,
+  kLevel6_2 = 186,
+  kLevel8_5 = 255
+};
+
+/// Tier as negotiated in SDP tier-flag field.
+enum class H265Tier : int {
+  kMain = 0,
+  kHigh,
+};
+
+enum class SamplingMode : int {
+  kNv12 = 0,
+  kARGB,
+  kP010,  // Planar, 4:2:0 10-bit.
+  kY410,  // Packed, 4:4:4 10-bit.
+  kP016,  // Planar, 4:2:0 16-bit.
+  kY416,  // Packed, 4:4:4 16-bit.
+};
+
+/// Encoder BRC mode.
+enum class BRCMode : int {
+  kUnknown = 0,
+  kCBR = 1,
+  kCQP = 2,
+  kVBR = 3,
+  kICQ = 4,
+  kAVBR = 5,
+  kVCM = 6,
+  kExternal = 7
+};
+
+struct VP9CodecCapability {
+  VP9Profile profile;
+};
+
+struct H264CodecCapability {
+  H264Profile profile;
+};
+
+struct VP8CodecCapability {
+};
+
+struct AV1CodecCapability {
+  AV1Profile profile;
+};
+
+struct H265CodecCapability {
+  H265ProfileId profile;
+};
+
+union CodecSpecificInfoUnion {
+  VP9CodecCapability VP9;
+#ifndef DISABLE_H265
+  H265CodecCapability H265;
+#endif
+  H264CodecCapability H264;
+  AV1CodecCapability AV1;
+  VP8CodecCapability VP8;
+  ~CodecSpecificInfoUnion() {}
+};
+
 
 class MediaUtils {
  public:

talk/owt/sdk/base/win/d3d11_allocator.cc

@@ -1,5 +1,10 @@
+// Copyright (C) <2020> Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+
 /******************************************************************************\
-Copyright (c) 2005-2018, Intel Corporation
+Copyright (c) 2005-2019, Intel Corporation
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
@@ -17,13 +22,6 @@ The original version of this sample may be obtained from https://software.intel.
 or https://software.intel.com/en-us/media-client-solutions-support.
 \**********************************************************************************/
 
-// Copyright (C) <2018> Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-
-
-#if defined(WEBRTC_WIN)
-
 #include "msdkvideobase.h"
 
 #include <objbase.h>
@@ -62,7 +60,7 @@ struct sequence<mfxHDL> {
 
 D3D11FrameAllocator::D3D11FrameAllocator()
 {
-    m_pDeviceContext = NULL;
+    m_pDeviceContext = nullptr;
 }
 
 D3D11FrameAllocator::~D3D11FrameAllocator()
@@ -88,7 +86,7 @@ D3D11FrameAllocator::TextureSubResource  D3D11FrameAllocator::GetResourceFromMid
 mfxStatus D3D11FrameAllocator::Init(mfxAllocatorParams *pParams)
 {
     D3D11AllocatorParams *pd3d11Params = 0;
-    pd3d11Params = dynamic_cast<D3D11AllocatorParams *>(pParams);
+    pd3d11Params = static_cast<D3D11AllocatorParams *>(pParams);
 
     if (NULL == pd3d11Params ||
         NULL == pd3d11Params->pDevice)
@@ -152,7 +150,16 @@ mfxStatus D3D11FrameAllocator::LockFrame(mfxMemId mid, mfxFrameData *ptr)
                 DXGI_FORMAT_R16G16B16A16_UNORM != desc.Format &&
                 DXGI_FORMAT_P010 != desc.Format &&
                 DXGI_FORMAT_AYUV != desc.Format
-)
+#if (MFX_VERSION >= 1027)
+                && DXGI_FORMAT_Y210 != desc.Format &&
+                DXGI_FORMAT_Y410 != desc.Format
+#endif
+#if (MFX_VERSION >= 1031)
+                && DXGI_FORMAT_P016 != desc.Format &&
+                DXGI_FORMAT_Y216 != desc.Format &&
+                DXGI_FORMAT_Y416 != desc.Format
+#endif
+            )
             {
                 return MFX_ERR_LOCK_MEMORY;
             }
@@ -185,6 +192,9 @@ mfxStatus D3D11FrameAllocator::LockFrame(mfxMemId mid, mfxFrameData *ptr)
     switch (desc.Format)
     {
         case DXGI_FORMAT_P010:
+#if (MFX_VERSION >= 1031)
+        case DXGI_FORMAT_P016:
+#endif
         case DXGI_FORMAT_NV12:
             ptr->Pitch = (mfxU16)lockedRect.RowPitch;
             ptr->Y = (mfxU8 *)lockedRect.pData;
@@ -247,8 +257,35 @@ mfxStatus D3D11FrameAllocator::LockFrame(mfxMemId mid, mfxFrameData *ptr)
             ptr->Y16 = (mfxU16 *)lockedRect.pData;
             ptr->U16 = 0;
             ptr->V16 = 0;
-
             break;
+#if (MFX_VERSION >= 1031)
+        case DXGI_FORMAT_Y416:
+          ptr->PitchHigh = (mfxU16)(lockedRect.RowPitch / (1 << 16));
+          ptr->PitchLow = (mfxU16)(lockedRect.RowPitch % (1 << 16));
+          ptr->U16 = (mfxU16*)lockedRect.pData;
+          ptr->Y16 = ptr->U16 + 1;
+          ptr->V16 = ptr->Y16 + 1;
+          ptr->A = (mfxU8*)(ptr->V16 + 1);
+          break;
+        case DXGI_FORMAT_Y216:
+#endif
+#if (MFX_VERSION >= 1027)
+        case DXGI_FORMAT_Y210:
+          ptr->PitchHigh = (mfxU16)(lockedRect.RowPitch / (1 << 16));
+          ptr->PitchLow = (mfxU16)(lockedRect.RowPitch % (1 << 16));
+          ptr->Y16 = (mfxU16*)lockedRect.pData;
+          ptr->U16 = ptr->Y16 + 1;
+          ptr->V16 = ptr->Y16 + 3;
+          break;
+        case DXGI_FORMAT_Y410:
+          ptr->PitchHigh = (mfxU16)(lockedRect.RowPitch / (1 << 16));
+          ptr->PitchLow = (mfxU16)(lockedRect.RowPitch % (1 << 16));
+          ptr->Y410 = (mfxY410*)lockedRect.pData;
+          ptr->Y = 0;
+          ptr->V = 0;
+          ptr->A = 0;
+          break;
+#endif
         default:
 
             return MFX_ERR_LOCK_MEMORY;
@@ -518,10 +555,23 @@ DXGI_FORMAT D3D11FrameAllocator::ConverColortFormat(mfxU32 fourcc)
         case MFX_FOURCC_AYUV:
             return DXGI_FORMAT_AYUV;
 
+#if (MFX_VERSION >= 1027)
+        case MFX_FOURCC_Y210:
+          return DXGI_FORMAT_Y210;
+        case MFX_FOURCC_Y410:
+          return DXGI_FORMAT_Y410;
+#endif
+#if (MFX_VERSION >= 1031)
+        case MFX_FOURCC_P016:
+          return DXGI_FORMAT_P016;
+        case MFX_FOURCC_Y216:
+          return DXGI_FORMAT_Y216;
+        case MFX_FOURCC_Y416:
+          return DXGI_FORMAT_Y416;
+#endif
         default:
             return DXGI_FORMAT_UNKNOWN;
     }
 }
 }  // namespace base
 }  // namespace owt
-#endif // #if defined(WEBRTC_WIN)

talk/owt/sdk/base/win/d3d11_allocator.h

@@ -1,3 +1,7 @@
+// Copyright (C) <2020> Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
 /******************************************************************************\
 Copyright (c) 2005-2018, Intel Corporation
 All rights reserved.
@@ -36,16 +40,20 @@ The original version of this sample may be obtained from
 or https://software.intel.com/en-us/media-client-solutions-support.
 \**********************************************************************************/
 
-// Copyright (C) <2018> Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-
 // This file is borrowed from MSDK sample with some minor modification.
 #ifndef OWT_BASE_WIN_D3D11ALLOCATOR_H__
 #define OWT_BASE_WIN_D3D11ALLOCATOR_H__
 
-#include "base_allocator.h"
+#include <d3d11.h>
 #include <limits>
+#include <map>
+#include <vector>
+#include <windows.h>
+#include "base_allocator.h"
+
+
+struct ID3D11VideoDevice;
+struct ID3D11VideoContext;
 
 namespace owt {
 namespace base {
@@ -134,15 +142,6 @@ class MFXReadWriteMid
     mfxMemId   m_mid_to_report;
 };
 
-#if defined(WEBRTC_WIN)
-
-#include <d3d11.h>
-#include <vector>
-#include <map>
-
-struct ID3D11VideoDevice;
-struct ID3D11VideoContext;
-
 struct D3D11AllocatorParams : mfxAllocatorParams
 {
     ID3D11Device *pDevice;
@@ -281,5 +280,5 @@ class D3D11FrameAllocator: public BaseFrameAllocator
 };
 }  // namespace base
 }  // namespace owt
-#endif // #if defined(WEBRTC_WIN)
+
 #endif // OWT_BASE_WIN_D3D11ALLOCATOR_H__

talk/owt/sdk/base/win/d3dnativeframe.h

@@ -18,5 +18,8 @@ struct NativeD3DSurfaceHandle {
   int height_;  // height of the frame passing from decoder
 };
 
-}
+// We keep D3D11ImageHandle definition in videorendererinterface,
+// as we may allow external renderer to accept the same handle.
+}  // namespace base
+}  // namespace owt
 #endif  // OWT_BASE_WIN_D3DNATIVEFRAME_H