Add super resolution support for renderer. (#518)

Author: taste1981

talk/owt/BUILD.gn

@@ -202,6 +202,7 @@ static_library("owt_sdk_base") {
       "sdk/base/win/msdkvideoencoderfactory.h",
       "sdk/base/win/sysmem_allocator.cc",
       "sdk/base/win/sysmem_allocator.h",
+      "sdk/base/win/vpedefs.h",
     ]
   }
 

talk/owt/sdk/base/globalconfiguration.cc

@@ -28,5 +28,6 @@ IcePortRanges GlobalConfiguration::ice_port_ranges_ = {
     {0, 0}};
 bool GlobalConfiguration::pre_decode_dump_enabled_ = false;
 bool GlobalConfiguration::post_encode_dump_enabled_ = false;
+bool GlobalConfiguration::video_super_resolution_enabled_ = false;
 }  // namespace base
 }

talk/owt/sdk/base/win/videorendererd3d11.cc

@@ -9,6 +9,7 @@
 #include <system_error>
 #include "talk/owt/sdk/base/nativehandlebuffer.h"
 #include "talk/owt/sdk/base/win/d3dnativeframe.h"
+#include "talk/owt/sdk/include/cpp/owt/base/globalconfiguration.h"
 #include "talk/owt/sdk/include/cpp/owt/base/videorendererinterface.h"
 #include "third_party/libyuv/include/libyuv/convert.h"
 #include "webrtc/api/video/i420_buffer.h"
@@ -19,9 +20,23 @@ using namespace rtc;
 namespace owt {
 namespace base {
 
+// Driver specific VPE interface for SR/FRC.
+static const GUID GUID_VPE_INTERFACE = {
+    0xedd1d4b9,
+    0x8659,
+    0x4cbc,
+    {0xa4, 0xd6, 0x98, 0x31, 0xa2, 0x16, 0x3a, 0xc3}};
+
+#define VPE_FN_SCALING_MODE_PARAM 0x37
+#define VPE_FN_MODE_PARAM 0x20
+#define VPE_FN_SET_VERSION_PARAM 0x01
+#define VPE_FN_SR_SET_PARAM 0x401
+#define VPE_FN_SET_CPU_GPU_COPY_PARAM 0x2B
+
 WebrtcVideoRendererD3D11Impl::WebrtcVideoRendererD3D11Impl(HWND wnd)
     : wnd_(wnd), clock_(Clock::GetRealTimeClock()) {
   CreateDXGIFactory(__uuidof(IDXGIFactory2), (void**)(&dxgi_factory_));
+  sr_enabled_ = SupportSuperResolution();
 }
 
 // The swapchain needs to use window height/width of even number.
@@ -35,7 +50,7 @@ bool WebrtcVideoRendererD3D11Impl::GetWindowSizeForSwapChain(int& width, int& he
   height = rect.bottom - rect.top;
 
   if (width % 2) {
-    window_width_ += 1;
+    width += 1;
   }
   if (height % 2) {
     height += 1;
@@ -556,6 +571,61 @@ void WebrtcVideoRendererD3D11Impl::RenderD3D11Texture(int width, int height) {
                                                           true, &rect);
   d3d11_video_context_->VideoProcessorSetStreamFrameFormat(
       video_processor_, 0, D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE);
+
+  // Setup VPE for SR.
+  if (sr_enabled_) {
+    VPE_FUNCTION function_params;
+    VPE_VERSION vpe_version = {};
+    VPE_MODE vpe_mode = {};
+    SR_SCALING_MODE sr_scaling_params = {};
+    VPE_SR_PARAMS sr_params = {};
+    void* p_ext_data = nullptr;
+    UINT data_size = 0;
+    const GUID* p_ext_guid = nullptr;
+
+    // Set VPE version
+    ZeroMemory(&function_params, sizeof(function_params));
+    vpe_version.Version = (UINT)VPE_VERSION_3_0;
+    function_params.Function = VPE_FN_SET_VERSION_PARAM;
+    function_params.pVPEVersion = &vpe_version;
+    p_ext_data = &function_params;
+    data_size = sizeof(function_params);
+    p_ext_guid = &GUID_VPE_INTERFACE;
+
+    hr = d3d11_video_context_->VideoProcessorSetOutputExtension(
+        video_processor_, p_ext_guid, data_size, p_ext_data);
+    if (FAILED(hr))
+      goto sr_fail;
+
+    // Clear mode
+    ZeroMemory(&function_params, sizeof(function_params));
+    vpe_mode.Mode = VPE_MODE_NONE;
+    function_params.Function = VPE_FN_MODE_PARAM;
+    function_params.pVPEMode = &vpe_mode;
+    p_ext_data = &function_params;
+    data_size = sizeof(function_params);
+    p_ext_guid = &GUID_VPE_INTERFACE;
+    hr = d3d11_video_context_->VideoProcessorSetOutputExtension(
+        video_processor_, p_ext_guid, data_size, p_ext_data);
+    if (FAILED(hr))
+      goto sr_fail;
+
+    // Set SR parameters
+    ZeroMemory(&function_params, sizeof(function_params));
+    sr_params.bEnable = true;
+    sr_params.SRMode = DEFAULT_SCENARIO_MODE;
+    function_params.Function = VPE_FN_SR_SET_PARAM;
+    function_params.pSRParams = &sr_params;
+    p_ext_data = &function_params;
+    data_size = sizeof(function_params);
+    p_ext_guid = &GUID_VPE_INTERFACE;
+    hr = d3d11_video_context_->VideoProcessorSetOutputExtension(
+        video_processor_, p_ext_guid, data_size, p_ext_data);
+    if (FAILED(hr))
+      goto sr_fail;
+  }
+
+sr_fail:
   hr = d3d11_video_context_->VideoProcessorBlt(
       video_processor_, output_view.Get(), 0, 1, &stream_data);
   if (FAILED(hr)) {
@@ -695,5 +765,10 @@ bool WebrtcVideoRendererD3D11Impl::CreateStagingTexture(int width, int height) {
   return true;
 }
 
+// Checks support for super resolution.
+bool WebrtcVideoRendererD3D11Impl::SupportSuperResolution() {
+  return GlobalConfiguration::GetVideoSuperResolutionEnabled();
+}
+
 }  // namespace base
 }  // namespace owt

talk/owt/sdk/base/win/videorendererd3d11.h

@@ -26,6 +26,7 @@
 #include <wrl.h>
 #include <vector>
 
+#include "talk/owt/sdk/base/win/vpedefs.h"
 #include "webrtc/api/video/video_frame.h"
 #include "webrtc/api/video/video_sink_interface.h"
 #include "webrtc/rtc_base/thread.h"
@@ -53,6 +54,7 @@ class WebrtcVideoRendererD3D11Impl
   bool InitSwapChain(int widht, int height, bool reset);
   bool CreateStagingTexture(int width, int height);
   bool GetWindowSizeForSwapChain(int& width, int& height);
+  bool SupportSuperResolution();
 
   // Render window objects
   HWND wnd_ = nullptr;
@@ -91,6 +93,8 @@ class WebrtcVideoRendererD3D11Impl
   bool d3d11_raw_inited_ = false;
   // Remote view is using MPO swapchain.
   bool d3d11_mpo_inited_ = false;
+  // Support super resolution
+  bool sr_enabled_ = false;
   webrtc::Clock* clock_;
 };
 

talk/owt/sdk/base/win/vpedefs.h

@@ -0,0 +1,77 @@
+// Copyright (C) <2021> Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef OWT_BASE_WIN_VPEDEFS_H
+#define OWT_BASE_WIN_VPEDEFS_H
+
+enum ScalingMode : int {
+  SCALING_MODE_DEFAULT = 0,                 // Default
+  SCALING_MODE_QUALITY,                     // LowerPower
+  SCALING_MODE_SUPERRESOLUTION              // SuperREsolution
+};
+
+enum VPEMode : int {
+  VPE_MODE_NONE = 0x0,
+  VPE_MODE_PREPROC = 0x1,
+  VPE_MODE_CPU_GPU_COPY = 0x3,
+};
+
+enum VPE_VERSION_ENUM : int {
+  VPE_VERSION_1_0 = 0x0001,
+  VPE_VERSION_2_0 = 0x0002,
+  VPE_VERSION_3_0 = 0x0003,
+  VPE_VERSION_UNKNOWN = 0xffff,
+};
+
+enum VPE_SUPER_RESOLUTION_MODE : int {
+  DEFAULT_SCENARIO_MODE = 0,
+  CAMERA_SCENARIO_MODE = 1,
+};
+
+enum VPE_CPU_GPU_COPY_DIRECTION : int {
+    VPE_CPU_GPU_COPY_DIRECTION_CPU_TO_GPU,
+    VPE_CPU_GPU_COPY_DIRECTION_GPU_TO_CPU,
+    VPE_CPU_GPU_COPY_DIRECTION_MMC_IN,
+    VPE_CPU_GPU_COPY_DIRECTION_NOW_MMC_IN,
+    VPE_CPU_GPU_COPY_DIRECTION_MMC_OUT,
+    VPE_CPU_GPU_COPY_DIRECTION_NOW_MMC_OUT,
+};
+
+typedef struct _SR_SCALING_MODE {
+  UINT Fastscaling;
+}SR_SCALING_MODE, *PSR_SCALING_MODE;
+
+typedef struct _VPE_VERSION {
+  UINT Version;
+}VPE_VERSION, *PVPE_VERSION;
+
+typedef struct _VPE_MODE {
+  UINT Mode;
+}VPE_MODE, *PVPE_MODE;
+
+typedef struct _VPE_FUNCTION {
+  UINT Function;               
+  union {
+    void* pSrCalingmode;
+    void* pVPEMode;
+    void* pVPEVersion;
+    void* pSRParams;
+    void* pCpuGpuCopyParam;
+  };
+} VPE_FUNCTION, *PVPE_FUNCTION;
+
+typedef struct _VPE_SR_PARAMS {
+  UINT bEnable : 1;  // [in], Enable SR
+  UINT ReservedBits : 31;
+  VPE_SUPER_RESOLUTION_MODE SRMode;
+  UINT Reserved[4];
+}VPE_SR_PARAMS, *PVPE_SR_PARAMS;
+
+typedef struct _VPE_CPU_GPU_COPY_PARAM {
+  VPE_CPU_GPU_COPY_DIRECTION Direction;
+  UINT MemSize;
+  void* pSystemMem;
+}VPE_CPU_GPU_COPY_PARAM, *PVPE_CPU_GPU_COPY_PARAM;
+
+#endif

talk/owt/sdk/include/cpp/owt/base/globalconfiguration.h

@@ -88,6 +88,32 @@ class GlobalConfiguration {
     hardware_acceleration_enabled_ = enabled;
   }
 #endif
+
+#if defined(WEBRTC_WIN)
+  /**
+   @brief Enable driver-based super resolution(SR) for video rendering if underlying
+   platform supports it. This can only be enabled on Windows platform and is by default
+   turned off. Requires 11th Generation Intel(R) Core(TM) processors or above, or
+   Intel(R) discrete graphics for SR to function properly.
+   Be noted turning this on does not neccessarily enable SR on all hardwares.
+   If SR is not supported by driver, render will silently fall back to normal scaling mode
+   and no error will be prompted. It is expected when SR is effectively on, 3D
+   usage will increase, so application needs to balance accordingly.
+   @param enabled If true, D3D11 video processor in the SDK will turn on SR when approriate.
+   */
+  static void SetVideoSuperResolutionEnabled(bool enabled) {
+    video_super_resolution_enabled_ = enabled;
+  }
+
+  /**
+   @brief Get SDK setting on whether super resolution is allowed. This does not
+   neccessarily indicate if super resolution is effective on or not in render pipeline.
+   @return True if SDK allows turnning on super resolution when appropriate. False otherwise.
+  */
+  static bool GetVideoSuperResolutionEnabled() {
+    return video_super_resolution_enabled_;
+  }
+#endif
   /** @cond */
   /**
    @brief This function sets the capturing frame type to be encoded video frame.
@@ -339,6 +365,8 @@ class GlobalConfiguration {
   static std::unique_ptr<VideoDecoderInterface> video_decoder_;
 
   static AudioProcessingSettings audio_processing_settings_;
+
+  static bool video_super_resolution_enabled_;
 };
 }
 }