New APIs in Device utils

Author: taste1981

talk/owt/sdk/base/deviceutils.cc

@@ -62,7 +62,8 @@ std::vector<Resolution> DeviceUtils::VideoCapturerSupportedResolutions(
   } else {
     for (int32_t i = 0; i < info->NumberOfCapabilities(id.c_str()); i++) {
       if (info->GetCapability(id.c_str(), i, capability) == 0) {
-        resolutions.push_back(Resolution(capability.width, capability.height));
+        resolutions.push_back(
+            Resolution(capability.width, capability.height));
       } else {
         RTC_LOG(LS_WARNING) << "Failed to get capability.";
       }
@@ -89,8 +90,8 @@ std::vector<Resolution> DeviceUtils::VideoCapturerSupportedResolutions(
       if (found) {
         for (int32_t i = 0; i < info->NumberOfCapabilities(vcm_id); i++) {
           if (info->GetCapability(vcm_id, i, capability) == 0) {
-            resolutions.push_back(
-                Resolution(capability.width, capability.height));
+            resolutions.push_back(Resolution(
+                capability.width, capability.height));
           } else {
             RTC_LOG(LS_WARNING) << "Failed to get capability.";
           }
@@ -100,5 +101,68 @@ std::vector<Resolution> DeviceUtils::VideoCapturerSupportedResolutions(
   }
   return resolutions;
 }
+
+std::vector<CameraCapability> DeviceUtils::VideoCapturerSupportedCapabilities(
+    const std::string& id) {
+  std::vector<CameraCapability> resolutions;
+  webrtc::VideoCaptureCapability capability;
+  std::unique_ptr<webrtc::VideoCaptureModule::DeviceInfo> info(
+      webrtc::VideoCaptureFactory::CreateDeviceInfo());
+  if (!info) {
+    RTC_LOG(LS_ERROR) << "CreateDeviceInfo failed";
+  } else {
+    for (int32_t i = 0; i < info->NumberOfCapabilities(id.c_str()); i++) {
+      if (info->GetCapability(id.c_str(), i, capability) == 0) {
+        resolutions.push_back(CameraCapability(
+            capability.width, capability.height, capability.maxFPS));
+      } else {
+        RTC_LOG(LS_WARNING) << "Failed to get capability.";
+      }
+    }
+    // Try to get capabilities by device name if getting capabilities by ID is
+    // failed.
+    // TODO(jianjun): Remove this when creating stream by device name is no
+    // longer supported.
+    if (resolutions.size() == 0) {
+      // Get device ID by name.
+      int num_cams = info->NumberOfDevices();
+      char vcm_id[256] = "";
+      bool found = false;
+      for (int index = 0; index < num_cams; ++index) {
+        char vcm_name[256] = "";
+        if (info->GetDeviceName(index, vcm_name, arraysize(vcm_name), vcm_id,
+                                arraysize(vcm_id)) != -1) {
+          if (id == reinterpret_cast<char*>(vcm_name)) {
+            found = true;
+            break;
+          }
+        }
+      }
+      if (found) {
+        for (int32_t i = 0; i < info->NumberOfCapabilities(vcm_id); i++) {
+          if (info->GetCapability(vcm_id, i, capability) == 0) {
+            resolutions.push_back(CameraCapability(
+                capability.width, capability.height, capability.maxFPS));
+          } else {
+            RTC_LOG(LS_WARNING) << "Failed to get capability.";
+          }
+        }
+      }
+    }
+  }
+  return resolutions;
+}
+
+std::string DeviceUtils::GetDeviceNameByIndex(int index) {
+  std::unique_ptr<webrtc::VideoCaptureModule::DeviceInfo> info(
+      webrtc::VideoCaptureFactory::CreateDeviceInfo());
+  char device_name[256];
+  char unique_name[256];
+  info->GetDeviceName(static_cast<uint32_t>(index), device_name,
+                      sizeof(device_name), unique_name, sizeof(unique_name));
+  std::string name(device_name);
+  return name;
+}
+
 }
 }

talk/owt/sdk/base/peerconnectionchannel.cc

@@ -29,6 +29,7 @@ bool PeerConnectionChannel::InitializePeerConnection() {
   video_transceiver_direction_ = webrtc::RtpTransceiverDirection::kSendRecv;
   configuration_.enable_dtls_srtp = true;
   configuration_.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan;
+  configuration_.media_config.enable_dscp = true;
   peer_connection_ =
       (factory_->CreatePeerConnection(configuration_, this)).get();
   if (!peer_connection_.get()) {

talk/owt/sdk/base/win/msdkvideoencoderfactory.cc

@@ -38,7 +38,7 @@ MSDKVideoEncoderFactory::MSDKVideoEncoderFactory() {
   codecs_to_check.push_back(owt::base::VideoCodec::kVp8);
   std::vector<VideoEncoderCapability> capabilities =
       media_capability->SupportedCapabilitiesForVideoEncoder(codecs_to_check);
-
+#if 0
   for (auto& capability : capabilities) {
     if (capability.codec_type == owt::base::VideoCodec::kH264 &&
         !is_h264_hw_supported) {
@@ -65,6 +65,12 @@ MSDKVideoEncoderFactory::MSDKVideoEncoderFactory() {
     }
 #endif
   }
+#endif
+  supported_codec_types.push_back(webrtc::kVideoCodecH264);
+  supported_codec_types.push_back(webrtc::kVideoCodecVP9);
+  supported_codec_types.push_back(webrtc::kVideoCodecVP8);
+  supported_codec_types.push_back(webrtc::kVideoCodecH265);
+  supported_codec_types.push_back(webrtc::kVideoCodecAV1);
 }
 
 std::unique_ptr<webrtc::VideoEncoder> MSDKVideoEncoderFactory::CreateVideoEncoder(
@@ -100,8 +106,9 @@ std::unique_ptr<webrtc::VideoEncoder> MSDKVideoEncoderFactory::CreateVideoEncode
     return webrtc::CreateLibaomAv1Encoder();
 #ifndef DISABLE_H265
   else if (absl::EqualsIgnoreCase(format.name, cricket::kH265CodecName) &&
-           !h265_hw)
-    return nullptr;
+           !h265_hw) {
+  }
+   // return nullptr;
 #endif
   return MSDKVideoEncoder::Create(cricket::VideoCodec(format));
 }

talk/owt/sdk/base/win/videorendererd3d11.cc

@@ -79,13 +79,24 @@ void WebrtcVideoRendererD3D11Impl::OnFrame(
     else
       return;
 
-    width_ = width;
-    height_ = height;
+    if (width_ != width || height_ != height) {
+      width_ = width;
+      height_ = height;
+    }
+
+    RECT rect;
+    GetClientRect(wnd_, &rect);
 
+    if (window_width != rect.right - rect.left ||
+        window_height != rect.bottom - rect.top) {
+      need_swapchain_recreate_ = true;
+      window_width = rect.right - rect.left;
+      window_height = rect.bottom - rect.top;
+    }
 
     if (render_device != d3d11_device_) {
-      d3d11_device_ = render_device;
       need_swapchain_recreate_ = true;
+      d3d11_device_ = render_device;
     }
 
     if (need_swapchain_recreate_) {
@@ -113,7 +124,23 @@ void WebrtcVideoRendererD3D11Impl::OnFrame(
       D3DPRESENT_PARAMETERS d3d_params = {};
 
       d3d_params.Windowed = true;
-      d3d_params.SwapEffect = D3DSWAPEFFECT_COPY;
+      d3d_params.SwapEffect = D3DSWAPEFFECT_DISCARD;
+
+      // Check anti-alias support
+      static D3DMULTISAMPLE_TYPE multisample_types[] = {
+          D3DMULTISAMPLE_5_SAMPLES, D3DMULTISAMPLE_4_SAMPLES,
+          D3DMULTISAMPLE_2_SAMPLES, D3DMULTISAMPLE_NONE};
+      DWORD multisample_quality = 0;
+      for (int i = 0; i < 4; i++) {
+        HRESULT hr = m_d3d_->CheckDeviceMultiSampleType(
+            D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, D3DFMT_A8R8G8B8, true,
+            multisample_types[i], &multisample_quality);
+        if (SUCCEEDED(hr)) {
+          d3d_params.MultiSampleType = multisample_types[i];
+          d3d_params.MultiSampleQuality = multisample_quality - 1;
+          break;
+        }
+      }
 
       IDirect3DDevice9* d3d_device;
       if (m_d3d_->CreateDevice(D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, wnd_,
@@ -138,6 +165,7 @@ void WebrtcVideoRendererD3D11Impl::OnFrame(
 
       m_d3d_device_->SetRenderState(D3DRS_CULLMODE, D3DCULL_NONE);
       m_d3d_device_->SetRenderState(D3DRS_LIGHTING, FALSE);
+      m_d3d_device_->SetRenderState(D3DRS_MULTISAMPLEANTIALIAS, TRUE);
       ResizeD3D9RenderPipeline(video_frame.width(), video_frame.height());
       d3d9_inited_for_raw_ = true;
     } else {
@@ -186,6 +214,7 @@ void WebrtcVideoRendererD3D11Impl::ResizeD3D9RenderPipeline(size_t width, size_t
   width_ = width;
   height_ = height;
   IDirect3DTexture9* texture;
+  // Texture should alwasy be created with the size of frame.
   m_d3d_device_->CreateTexture(static_cast<UINT>(width_),
                                static_cast<UINT>(height_), 1, 0, D3DFMT_A8R8G8B8,
                                D3DPOOL_MANAGED, &texture, nullptr);
@@ -208,6 +237,8 @@ void WebrtcVideoRendererD3D11Impl::ResizeD3D9RenderPipeline(size_t width, size_t
   m_vertex_buffer_->Unlock();
 }
 
+// TODO: Use D3D11 for rendering of I420 view as well.
+#if 0
 HRESULT WebrtcVideoRendererD3D11Impl::CreateD3D11Device() {
   HRESULT hr = S_OK;
 
@@ -228,23 +259,21 @@ HRESULT WebrtcVideoRendererD3D11Impl::CreateD3D11Device() {
   // Create the surface to copy NV12 image to.
   D3D11_TEXTURE2D_DESC texture_desc;
   memset(&texture_desc, 0, sizeof(texture_desc));
-  texture_desc.Format = DXGI_FORMAT_NV12;
-  texture_desc.Width = width_;
-  texture_desc.Height = height_;
+  texture_desc.Format = DXGI_FORMAT_R8_UNORM;
+  texture_desc.Width = width_;  // same as video width
+  texture_desc.Height = height_; // same as video height
   texture_desc.ArraySize = 1;
   texture_desc.MipLevels = 1;
-  texture_desc.Usage = D3D11_USAGE_DYNAMIC;
-  texture_desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
-  texture_desc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE;
+  texture_desc.Usage = D3D11_USAGE_DEFAULT; // We use UpdateSubresource to update the texture,
+                                            // so not setting to D3D11_USAGE_DEFAULT;
 
   hr = d3d11_device_->CreateTexture2D(&texture_desc, nullptr,
-                                      &sw_shared_texture_);
-  if (FAILED(hr)) {
-    RTC_LOG(LS_ERROR)
-        << "Failed to create shared texture to copy nv12 frame to.";
-    return hr;
-  }
+                                      &texture_planes_[0]);
+  
+  texture_desc.Width = width_ / 2;
+  texture_desc.Height = height_ / 2;
 }
+#endif
 
 // Method to create the necessary 3D pipeline objects to render a textured 3D
 // QUAD
@@ -471,12 +500,15 @@ void WebrtcVideoRendererD3D11Impl::RenderToBackbuffer(int array_slice) {
   FLOAT clear_color[4] = {0.0f, 0.0f, 0.0f, 0.0f};
   d3d11_device_context_->ClearRenderTargetView(render_target_view_, clear_color);
 
+  RECT rect;
+  GetClientRect(wnd_, &rect);
+
   // Set viewport
   D3D11_VIEWPORT viewport = {0};
-  viewport.TopLeftX = x_offset_;
-  viewport.TopLeftY = y_offset_;
-  viewport.Width = width_;
-  viewport.Height = height_;
+  viewport.TopLeftX = 0;
+  viewport.TopLeftY = 0;
+  viewport.Width = rect.right - rect.left;
+  viewport.Height = rect.bottom - rect.top;
   d3d11_device_context_->RSSetViewports(1, &viewport);
 
   // Bind the textures

talk/owt/sdk/base/win/videorendererd3d11.h

@@ -60,8 +60,10 @@ class WebrtcVideoRendererD3D11Impl
   void Destroy();
   void FillSwapChainDesc(DXGI_SWAP_CHAIN_DESC1& scd);
   void ResetTextureViews();
+#if 0
   HRESULT CreateD3D11Device();
   void WriteNV12ToTexture();
+#endif
   HRESULT CreateRenderPipeline();
   HRESULT ResizeRenderPipeline();
   void ResizeD3D9RenderPipeline(size_t width, size_t height);
@@ -81,6 +83,8 @@ class WebrtcVideoRendererD3D11Impl
   uint16_t y_offset_ = 0;
   uint16_t width_ = 0;
   uint16_t height_ = 0;
+  int window_width = 0;
+  int window_height = 0;
   bool need_swapchain_recreate_ = true;
 
   // Owner of the d3d11 device/context is decoder.
@@ -101,7 +105,6 @@ class WebrtcVideoRendererD3D11Impl
   ID3D11PixelShader* pixel_shader_ = nullptr;
   ID3D11InputLayout* input_layout_ = nullptr;
   ID3D11Buffer* vertex_buffer_ = nullptr;
-  ID3D11Texture2D* sw_shared_texture_ = nullptr;
 
   // Using D3D9 for rendering SW frames.
   bool d3d9_inited_for_raw_ = false;
@@ -110,6 +113,7 @@ class WebrtcVideoRendererD3D11Impl
   rtc::scoped_refptr<IDirect3DTexture9> m_texture_;
   rtc::scoped_refptr<IDirect3DVertexBuffer9> m_vertex_buffer_;
   UINT views_count = 0;
+
 };
 }  // namespace base
 }  // namespace owt

talk/owt/sdk/conference/conferencepeerconnectionchannel.cc

@@ -1007,6 +1007,24 @@ void ConferencePeerConnectionChannel::SendPublishMessage(
                   encoding.num_temporal_layers <= 4) {
                 param.num_temporal_layers = encoding.num_temporal_layers;
               }
+              if (encoding.priority != owt::base::NetworkPriority::kDefault) {
+                switch (encoding.priority) {
+                  case owt::base::NetworkPriority::kVeryLow:
+                    param.network_priority = webrtc::Priority::kVeryLow;
+                    break;
+                  case owt::base::NetworkPriority::kLow:
+                    param.network_priority = webrtc::Priority::kLow;
+                    break;
+                  case owt::base::NetworkPriority::kMedium:
+                    param.network_priority = webrtc::Priority::kMedium;
+                    break;
+                  case owt::base::NetworkPriority::kHigh:
+                    param.network_priority = webrtc::Priority::kHigh;
+                    break;
+                  default:
+                    break;
+                }
+              }
               param.active = encoding.active;
               transceiver_init.send_encodings.push_back(param);
             }

talk/owt/sdk/include/cpp/owt/base/commontypes.h

@@ -31,12 +31,20 @@ enum class VideoCodec : int {
   kUnknown
 };
 /// Track kind
-enum class TrackKind : int{
+enum class TrackKind : int {
   kAudio = 1,
   kVideo,
   kAudioAndVideo,
   kUnknown
 };
+/// Network priority
+enum class NetworkPriority : int {
+  kVeryLow,
+  kLow,
+  kMedium,
+  kHigh,
+  kDefault
+};
 /// This class represents a resolution value.
 struct Resolution {
   /// Construct an instance with width and height equal to 0.
@@ -49,6 +57,23 @@ struct Resolution {
   unsigned long width;
   unsigned long height;
 };
+
+/// This class represents a camera capability.
+struct CameraCapability {
+  /// Construct an instance with width and height equal to 0.
+  explicit CameraCapability() : width(0), height(0), fps(0) {}
+  /// Construct an instance with specify width and height.
+  CameraCapability(unsigned long w, unsigned long h, int fps)
+      : width(w), height(h), fps(fps) {}
+  bool operator==(const CameraCapability& rhs) const {
+    return this->width == rhs.width && this->height == rhs.height &&
+           this->fps == rhs.fps ;
+  }
+  unsigned long width;
+  unsigned long height;
+  int32_t fps;
+};
+
 /// Audio codec parameters for an audio track.
 struct AudioCodecParameters {
   /// Construct an instance of AudioCodecParameters with default param.
@@ -93,6 +118,8 @@ struct RtpEncodingParameters {
   // Value to use for RID RTP header extension.
   // Called "encodingId" in ORTC.
   std::string rid = "";
+
+  NetworkPriority priority = NetworkPriority::kDefault;
 };
 
 /// Audio encoding parameters.

talk/owt/sdk/include/cpp/owt/base/deviceutils.h

@@ -22,6 +22,10 @@ class DeviceUtils {
       const std::string& id);
   /// Get the camera device index by its device id.
   static int GetVideoCaptureDeviceIndex(const std::string& id);
+  /// Get camera device's user friendly name by index.
+  static std::string GetDeviceNameByIndex(int index);
+  static std::vector<CameraCapability> VideoCapturerSupportedCapabilities(
+      const std::string& id);
 };
 }
 }