Enable IC background blur feature

build_overrides/owt.gni

@@ -0,0 +1,5 @@
+declare_args() {
+  owt_msdk_lib_root = ""
+  owt_msdk_header_root = ""
+  owt_build_ic = false
+}

scripts/build-win.py

@@ -35,7 +35,7 @@
     ]
 
 
-def gngen(arch, ssl_root, msdk_root, quic_root, scheme, tests):
+def gngen(arch, ssl_root, msdk_root, quic_root, openvino_root, scheme, tests):
     gn_args = list(GN_ARGS)
     gn_args.append('target_cpu="%s"' % arch)
     using_llvm = False
@@ -76,6 +76,9 @@ def gngen(arch, ssl_root, msdk_root, quic_root, scheme, tests):
         else:
             return False
         gn_args.append('owt_use_quic=true')
+    if openvino_root:
+        gn_args.append(f'owt_build_ic=true')
+        gn_args.append(f'owt_openvino_root="{openvino_root}"')
     if tests:
         gn_args.append('rtc_include_tests=true')
         gn_args.append('owt_include_tests=true')
@@ -183,6 +186,7 @@ def main():
     parser.add_argument('--ssl_root', help='Path for OpenSSL.')
     parser.add_argument('--msdk_root', help='Path for MSDK.')
     parser.add_argument('--quic_root', help='Path to QUIC library')
+    parser.add_argument('--openvino_root', default="", help='Path for OpenVINO.')
     parser.add_argument('--scheme', default='debug', choices=('debug', 'release'),
                         help='Schemes for building. Supported value: debug, release')
     parser.add_argument('--gn_gen', default=False, action='store_true',
@@ -195,19 +199,16 @@ def main():
                         help='To generate the API document.')
     parser.add_argument('--output_path', help='Path to copy sdk.')
     opts = parser.parse_args()
-    if opts.ssl_root and not os.path.exists(os.path.expanduser(opts.ssl_root)):
-        print('Invalid ssl_root.')
-        return 1
-    if opts.msdk_root and not os.path.exists(os.path.expanduser(opts.msdk_root)):
-        print('Invalid msdk_root')
-        return 1
-    if opts.quic_root and not os.path.exists(os.path.expanduser(opts.quic_root)):
-        print('Invalid quic_root')
-        return 1
+    for name in ['ssl', 'msdk', 'quic', 'openvino']:
+        attr = name + '_root'
+        path = getattr(opts, attr)
+        if path is not None and not os.path.exists(os.path.expanduser(path)):
+            print('Invalid {}.'.format(attr))
+            return 1
     print(opts)
     if opts.gn_gen:
         if not gngen(opts.arch, opts.ssl_root, opts.msdk_root, opts.quic_root,
-                     opts.scheme, opts.tests):
+                     opts.openvino_root, opts.scheme, opts.tests):
             return 1
     if opts.sdk:
         if not ninjabuild(opts.arch, opts.scheme):

talk/owt/BUILD.gn

@@ -3,12 +3,11 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import("//build_overrides/webrtc.gni")
+import("//build_overrides/owt.gni")
 import("//testing/test.gni")
 
 declare_args() {
   include_internal_audio_device = true
-  owt_msdk_lib_root = ""
-  owt_msdk_header_root = ""
 }
 
 # Introduced for using libvpx config files. We only enable libvpx rate
@@ -83,6 +82,11 @@ if (!is_ios) {
       "//third_party/webrtc:webrtc",
       "//third_party/webrtc/api:libjingle_peerconnection_api",
     ]
+    if (owt_build_ic) {
+      data_deps = [
+        "sdk/ic:owt_ic",
+      ]
+    }
     complete_static_lib = true
   }
 }
@@ -99,6 +103,8 @@ static_library("owt_sdk_base") {
     "sdk/base/functionalobserver.cc",
     "sdk/base/functionalobserver.h",
     "sdk/base/globalconfiguration.cc",
+    "sdk/base/icmanager.cc",
+    "sdk/base/icmanager.h",
     "sdk/base/localcamerastreamparameters.cc",
     "sdk/base/logging.cc",
     "sdk/base/logsinks.cc",
@@ -118,6 +124,7 @@ static_library("owt_sdk_base") {
     "sdk/base/sysinfo.h",
     "sdk/base/vcmcapturer.cc",
     "sdk/base/vcmcapturer.h",
+    "sdk/base/videoframepostprocessing.h",
     "sdk/base/webrtcaudiorendererimpl.cc",
     "sdk/base/webrtcaudiorendererimpl.h",
     "sdk/include/cpp/owt/base/audioplayerinterface.h",

talk/owt/sdk/base/cameravideocapturer.cc

@@ -36,18 +36,28 @@ void CameraVideoCapturer::OnFrame(const webrtc::VideoFrame& frame) {
     return;
   }
 
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer = frame.video_frame_buffer();
+
   if (out_height != frame.height() || out_width != frame.width()) {
     // Video adapter has requested a down-scale. Allocate a new buffer and
     // return scaled version.
     rtc::scoped_refptr<webrtc::I420Buffer> scaled_buffer =
         webrtc::I420Buffer::Create(out_width, out_height);
-    scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420());
+    scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420());  
+    buffer = scaled_buffer;
+  }
+
+  for (auto &pp : video_frame_post_processings_) {
+    buffer = pp->Process(buffer);
+  }
+
+  if (buffer != frame.video_frame_buffer()) {
     broadcaster_.OnFrame(webrtc::VideoFrame::Builder()
-                             .set_video_frame_buffer(scaled_buffer)
+                             .set_video_frame_buffer(buffer)
                              .set_rotation(webrtc::kVideoRotation_0)
                              .set_timestamp_us(frame.timestamp_us())
                              .set_id(frame.id())
-                             .build());
+                             .build());  
   } else {
     // No adaptations needed, just return the frame as is.
     broadcaster_.OnFrame(frame);
@@ -71,9 +81,14 @@ void CameraVideoCapturer::RemoveSink(
   UpdateVideoAdapter();
 }
 
+void CameraVideoCapturer::AddVideoFramePostProcessing(
+    std::shared_ptr<VideoFramePostProcessing> post_processing) {
+  video_frame_post_processings_.emplace_back(post_processing);
+}
+
 void CameraVideoCapturer::UpdateVideoAdapter() {
   video_adapter_.OnSinkWants(broadcaster_.wants());
 }
 
-}  // namespace test
-}  // namespace webrtc
+}  // namespace base
+}  // namespace owt

talk/owt/sdk/base/cameravideocapturer.h

@@ -13,12 +13,15 @@
 #include <stddef.h>
 
 #include <memory>
+#include <vector>
 
 #include "api/video/video_frame.h"
 #include "api/video/video_source_interface.h"
 #include "media/base/video_adapter.h"
 #include "media/base/video_broadcaster.h"
 
+#include "videoframepostprocessing.h"
+
 // This file is borrowed from webrtc project
 namespace owt {
 namespace base {
@@ -32,6 +35,9 @@ class CameraVideoCapturer : public rtc::VideoSourceInterface<webrtc::VideoFrame>
                        const rtc::VideoSinkWants& wants) override;
   void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override;
 
+  void AddVideoFramePostProcessing(
+      const std::shared_ptr<VideoFramePostProcessing> post_processing);
+
  protected:
   void OnFrame(const webrtc::VideoFrame& frame);
   rtc::VideoSinkWants GetSinkWants();
@@ -41,6 +47,8 @@ class CameraVideoCapturer : public rtc::VideoSourceInterface<webrtc::VideoFrame>
 
   rtc::VideoBroadcaster broadcaster_;
   cricket::VideoAdapter video_adapter_;
+  std::vector<std::shared_ptr<owt::base::VideoFramePostProcessing>>
+      video_frame_post_processings_;
 };
 }  // namespace base
 }  // namespace owt

talk/owt/sdk/base/icmanager.cc

@@ -0,0 +1,39 @@
+#include "icmanager.h"
+
+#include "webrtc/rtc_base/logging.h"
+
+namespace owt {
+namespace base {
+
+ICManager* ICManager::GetInstance() {
+  static ICManager instance;
+  return &instance;
+}
+
+std::shared_ptr<VideoFramePostProcessing> ICManager::CreatePostProcessing(
+    const char* name) {
+  return create_post_processing_
+             ? std::shared_ptr<VideoFramePostProcessing>(
+                   create_post_processing_(name),
+                   [](VideoFramePostProcessing* ptr) { ptr->Release(); })
+             : nullptr;
+}
+
+ICManager::ICManager() {
+  if (owt_ic_dll_ = LoadLibrary(L"owt_ic.dll")) {
+    RTC_LOG(INFO) << "owt_ic.dll is loaded.";
+    create_post_processing_ =
+        (CREATE_POST_PROCESSING)GetProcAddress(owt_ic_dll_, "CreatePostProcessing");
+  } else {
+    RTC_LOG(WARNING) << "owt_ic.dll is not loaded.";
+  }
+}
+
+ICManager::~ICManager() {
+  if (owt_ic_dll_) {
+    FreeLibrary(owt_ic_dll_);
+  }
+}
+
+}  // namespace base
+}  // namespace owt

talk/owt/sdk/base/icmanager.h

@@ -0,0 +1,36 @@
+#ifndef OWT_BASE_ICMANAGER_H_
+#define OWT_BASE_ICMANAGER_H_
+
+#include <windows.h>
+#include <memory>
+
+#include "api/scoped_refptr.h"
+#include "base/macros.h"
+#include "videoframepostprocessing.h"
+
+namespace owt {
+namespace base {
+
+class ICManager {
+ public:
+  static ICManager* GetInstance();
+  std::shared_ptr<VideoFramePostProcessing> CreatePostProcessing(
+      const char* name);
+
+ private:
+  ICManager();
+  ~ICManager();
+
+  typedef owt::base::VideoFramePostProcessing* (*CREATE_POST_PROCESSING)(
+      const char* name);
+
+  HMODULE owt_ic_dll_ = nullptr;
+  CREATE_POST_PROCESSING create_post_processing_ = nullptr;
+
+  DISALLOW_COPY_AND_ASSIGN(ICManager);
+};
+
+}  // namespace base
+}  // namespace owt
+
+#endif  // OWT_BASE_ICMANAGER_H_

talk/owt/sdk/base/localcamerastreamparameters.cc

@@ -19,6 +19,9 @@ LocalCameraStreamParameters::LocalCameraStreamParameters(bool audio_enabled,
 void LocalCameraStreamParameters::Fps(int fps) {
   fps_ = fps;
 }
+void LocalCameraStreamParameters::BackgroundBlur(bool enable) {
+  background_blur_ = enable;
+}
 void LocalCameraStreamParameters::CameraId(const std::string& camera_id) {
   camera_id_ = camera_id;
 }

talk/owt/sdk/base/peerconnectiondependencyfactory.cc

@@ -47,6 +47,7 @@
 #pragma comment(lib, "wmcodecdspuuid.lib")
 #pragma comment(lib, "amstrmid.lib")
 #pragma comment(lib, "strmiids.lib")
+#pragma comment(lib, "dcomp.lib")
 #ifdef OWT_USE_MSDK
 #pragma comment(lib, "mf.lib")
 #pragma comment(lib, "mfplat.lib")

talk/owt/sdk/base/stream.cc

@@ -11,8 +11,10 @@
 #include "webrtc/sdk/media_constraints.h"
 
 #include "talk/owt/sdk/base/customizedframescapturer.h"
+#include "talk/owt/sdk/base/icmanager.h"
 #include "talk/owt/sdk/base/webrtcaudiorendererimpl.h"
 #include "talk/owt/sdk/base/webrtcvideorendererimpl.h"
+#include "talk/owt/sdk/ic/backgroundblur.h"
 #include "talk/owt/sdk/include/cpp/owt/base/framegeneratorinterface.h"
 
 #if defined(WEBRTC_WIN)
@@ -45,7 +47,8 @@ class CapturerTrackSource : public webrtc::VideoTrackSource {
       const size_t width,
       const size_t height,
       const size_t fps,
-      int capture_device_idx) {
+      int capture_device_idx,
+      bool background_blur_enabled = false) {
     std::unique_ptr<owt::base::VcmCapturer> capturer;
     std::unique_ptr<webrtc::VideoCaptureModule::DeviceInfo> info(
         webrtc::VideoCaptureFactory::CreateDeviceInfo());
@@ -56,6 +59,9 @@ class CapturerTrackSource : public webrtc::VideoTrackSource {
     for (int i = 0; i < num_devices; ++i) {
       capturer = absl::WrapUnique(owt::base::VcmCapturer::Create(
           width, height, fps, capture_device_idx));
+      if (background_blur_enabled) {
+        capturer->AddVideoFramePostProcessing(ICManager::GetInstance()->CreatePostProcessing("background_blur"));
+      }
       if (capturer) {
         return new rtc::RefCountedObject<CapturerTrackSource>(
             std::move(capturer));
@@ -513,7 +519,8 @@ LocalStream::LocalStream(const LocalCameraStreamParameters& parameters,
         CapturerTrackSource::Create(
             parameters.ResolutionWidth(), parameters.ResolutionHeight(),
             parameters.Fps(),
-            DeviceUtils::GetVideoCaptureDeviceIndex(parameters.CameraId()));
+            DeviceUtils::GetVideoCaptureDeviceIndex(parameters.CameraId()),
+            parameters.BackgroundBlur());
 #else
     capturer_ = ObjcVideoCapturerFactory::Create(parameters);
     if (!capturer_) {

talk/owt/sdk/base/videoframepostprocessing.h

@@ -0,0 +1,23 @@
+#ifndef OWT_BASE_VIDEOFRAMEPOSTPROCESSING_H_
+#define OWT_BASE_VIDEOFRAMEPOSTPROCESSING_H_
+
+#include "api/scoped_refptr.h"
+#include "api/video/video_frame_buffer.h"
+
+namespace owt {
+namespace base {
+
+class VideoFramePostProcessing {
+ public:
+  virtual ~VideoFramePostProcessing() = default;
+
+  virtual rtc::scoped_refptr<webrtc::VideoFrameBuffer> Process(
+      const rtc::scoped_refptr<webrtc::VideoFrameBuffer>& buffer) = 0;
+
+  virtual void Release() noexcept { delete this; }
+};
+
+}  // namespace base
+}  // namespace owt
+
+#endif  // OWT_IC_VIDEOFRAMEPOSTPROCESSING_H_