Enable IC background blur feature

README.md

@@ -27,6 +27,9 @@ The following dependencies are for Windows only:
 - [Visual Studio](https://chromium.googlesource.com/chromium/src/+/refs/heads/main/docs/windows_build_instructions.md#visual-studio).
 - [Intel Media SDK for Windows, version 2020 R1 or higher](https://software.intel.com/en-us/media-sdk/choose-download/client).
 
+If you want intelligent collaboration video post-processor features, you need:
+- [OpenVINO Toolkit](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit-download.html).
+
 ### Get the code
 - Make sure you clone the source code to a directory named `src`.
 - Create a file named .gclient in the directory above the `src` dir, with these contents:
@@ -54,6 +57,7 @@ target_os = []
   - The optional `msdk_root` should be set to the directory of your Intel MediaSDK for Windows, version 2020 R1 or higher. This is typically
   `C:\Program Files (x86)\IntelSWTools\Intel(R) Media SDK 2020 R1\Software Development Kit`. If specified, will enable hardware accelerated video codecs for most of the video codecs.
   - The optional `--sdk` is to inform the build script to use `lib.exe` that is part of Visual Studio toolchain for merging owt libraries with external openssl libraries.
+  - To enable intelligent collaboration video post-processors (background blur, etc.), set the `openvino_root` option, which would typically be `--openvino_root "C:\Program Files (x86)\Intel\openvino_2021"`.
 
 
 #### Linux

build_overrides/owt.gni

@@ -0,0 +1,10 @@
+# Copyright (C) <2021> Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+declare_args() {
+  owt_msdk_lib_root = ""
+  owt_msdk_header_root = ""
+  owt_build_ic = false
+  owt_msvcrt = false
+}

jenkins/windows.jenkinsfile

@@ -35,7 +35,7 @@ pipeline {
                     }
                     steps {
                         echo "$GIT_COMMIT"
-                        bat "%windowsCIPathBranch%_%CHANGE_TARGET%/buildSdk.bat %GIT_COMMIT%"
+                        bat "%windowsCIPathBranch%_%CHANGE_TARGET%/buildSdk.bat %GIT_COMMIT% --with-openvino"
                     }
                 }
                 stage("startMcu") {
@@ -73,18 +73,18 @@ pipeline {
             }
             steps {
                 echo "Buliding.."
-                bat "%windowsConferenceCasePath%/build.bat"
+                bat "%windowsConferenceCasePath%/build.bat --configuration Release-MD"
             }
         }
-        stage("BulidTestP2P") {
+        stage("BulidP2P") {
             agent{
                 node {
                     label "windows"
                 }
             }
             steps {
                 echo "Buliding.."
-                bat "%windowsP2PCasePath%/build.bat"
+                bat "%windowsP2PCasePath%/build.bat --configuration Release-MD"
             }
         }
         stage("TestConference") {
@@ -95,7 +95,7 @@ pipeline {
             }
             steps {
                 echo "Testing.."
-                bat "%windowsConferenceCasePath%/runCase.bat"
+                bat "%windowsConferenceCasePath%/runCase.bat --configuration Release-MD"
             }
         }
         stage("TestP2P") {
@@ -106,7 +106,7 @@ pipeline {
             }
             steps {
                 echo "Testing.."
-                bat "%windowsP2PCasePath%/runCase.bat"
+                bat "%windowsP2PCasePath%/runCase.bat --configuration Release-MD"
             }
         }
     }

scripts/build-win.py

@@ -11,6 +11,7 @@
 '''
 
 import os
+import re
 import sys
 import subprocess
 import argparse
@@ -35,7 +36,7 @@
     ]
 
 
-def gngen(arch, ssl_root, msdk_root, quic_root, scheme, tests):
+def gngen(arch, ssl_root, msdk_root, quic_root, openvino_root, scheme, tests):
     gn_args = list(GN_ARGS)
     gn_args.append('target_cpu="%s"' % arch)
     using_llvm = False
@@ -76,6 +77,20 @@ def gngen(arch, ssl_root, msdk_root, quic_root, scheme, tests):
         else:
             return False
         gn_args.append('owt_use_quic=true')
+    if openvino_root:
+        gn_args.append('owt_build_ic=true')
+        gn_args.append('owt_msvcrt=true')
+        gn_args.append('owt_openvino_root="{}"'.format(openvino_root))
+        opencv_root = os.path.join(openvino_root, 'opencv')
+        opencv_version_bin = os.path.join(opencv_root, 'bin', 'opencv_version.exe')
+        try:
+            output = subprocess.check_output([opencv_version_bin])
+            opencv_version = ''.join(re.findall('\d', output.decode()))
+            gn_args.append('owt_opencv_root="{}"'.format(opencv_root))
+            gn_args.append('owt_opencv_version={}'.format(opencv_version))
+        except FileNotFoundError as e:
+            print('File not found: {}'.format(opencv_version_bin))
+            return False
     if tests:
         gn_args.append('rtc_include_tests=true')
         gn_args.append('owt_include_tests=true')
@@ -183,6 +198,7 @@ def main():
     parser.add_argument('--ssl_root', help='Path for OpenSSL.')
     parser.add_argument('--msdk_root', help='Path for MSDK.')
     parser.add_argument('--quic_root', help='Path to QUIC library')
+    parser.add_argument('--openvino_root', help='Path for OpenVINO.')
     parser.add_argument('--scheme', default='debug', choices=('debug', 'release'),
                         help='Schemes for building. Supported value: debug, release')
     parser.add_argument('--gn_gen', default=False, action='store_true',
@@ -195,19 +211,16 @@ def main():
                         help='To generate the API document.')
     parser.add_argument('--output_path', help='Path to copy sdk.')
     opts = parser.parse_args()
-    if opts.ssl_root and not os.path.exists(os.path.expanduser(opts.ssl_root)):
-        print('Invalid ssl_root.')
-        return 1
-    if opts.msdk_root and not os.path.exists(os.path.expanduser(opts.msdk_root)):
-        print('Invalid msdk_root')
-        return 1
-    if opts.quic_root and not os.path.exists(os.path.expanduser(opts.quic_root)):
-        print('Invalid quic_root')
-        return 1
+    for name in ['ssl', 'msdk', 'quic', 'openvino']:
+        attr = name + '_root'
+        path = getattr(opts, attr)
+        if path is not None and not os.path.exists(os.path.expanduser(path)):
+            print('Invalid {}.'.format(attr))
+            return 1
     print(opts)
     if opts.gn_gen:
         if not gngen(opts.arch, opts.ssl_root, opts.msdk_root, opts.quic_root,
-                     opts.scheme, opts.tests):
+                     opts.openvino_root, opts.scheme, opts.tests):
             return 1
     if opts.sdk:
         if not ninjabuild(opts.arch, opts.scheme):

scripts/prepare_dev.py

@@ -45,7 +45,8 @@
     ('0014-Fix-missing-ffmpeg-configure-item-for-msvc-build.patch', FFMPEG_PATH),
     ('0015-Remove-custom-d8-dependency.patch', BUILD_PATH),
     ('0016-Remove-deprecated-create_srcjar-property.patch', THIRD_PARTY_PATH),
-    ('0017-Build-libvpx-with-RTC-rate-control-impl-included.patch', THIRD_PARTY_PATH) 
+    ('0017-Build-libvpx-with-RTC-rate-control-impl-included.patch', THIRD_PARTY_PATH),
+    ('0018-Enable-msvcrt-switch.patch', BUILD_PATH),
 ]
 
 def _patch(ignoreFailures=False):

talk/owt/BUILD.gn

@@ -3,12 +3,11 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import("//build_overrides/webrtc.gni")
+import("//build_overrides/owt.gni")
 import("//testing/test.gni")
 
 declare_args() {
   include_internal_audio_device = true
-  owt_msdk_lib_root = ""
-  owt_msdk_header_root = ""
 }
 
 # Introduced for using libvpx config files. We only enable libvpx rate
@@ -109,6 +108,7 @@ static_library("owt_sdk_base") {
     "sdk/base/peerconnectionchannel.h",
     "sdk/base/peerconnectiondependencyfactory.cc",
     "sdk/base/peerconnectiondependencyfactory.h",
+    "sdk/base/pluginmanager.cc",
     "sdk/base/sdputils.cc",
     "sdk/base/sdputils.h",
     "sdk/base/stream.cc",
@@ -128,7 +128,9 @@ static_library("owt_sdk_base") {
     "sdk/include/cpp/owt/base/framegeneratorinterface.h",
     "sdk/include/cpp/owt/base/localcamerastreamparameters.h",
     "sdk/include/cpp/owt/base/logging.h",
+    "sdk/include/cpp/owt/base/pluginmanager.h",
     "sdk/include/cpp/owt/base/stream.h",
+    "sdk/include/cpp/owt/base/videoframepostprocessor.h",
     "sdk/include/cpp/owt/base/videorendererinterface.h",
   ]
   if (is_win || is_linux) {
@@ -141,10 +143,13 @@ static_library("owt_sdk_base") {
       "sdk/base/customizedvideosource.h",
       "sdk/base/encodedvideoencoderfactory.cc",
       "sdk/base/encodedvideoencoderfactory.h",
+      "sdk/base/sharedobjectloader.h",
+      "sdk/base/sharedobjectpointer.h",
       "sdk/base/webrtcvideorendererimpl.cc",
       "sdk/base/webrtcvideorendererimpl.h",
       "sdk/base/windowcapturer.cc",
       "sdk/include/cpp/owt/base/videodecoderinterface.h",
+      "sdk/include/cpp/owt/ic/icmanagerinterface.h",
     ]
   }
   public_deps = [
@@ -261,6 +266,7 @@ static_library("owt_sdk_base") {
       "sdk/base/win/device_info_mf.cc",
       "sdk/base/win/video_capture_mf.h",
       "sdk/base/win/video_capture_mf.cc",
+      "sdk/base/win/sharedobjectloader.cc",
     ]
     public_deps += [ "//third_party/webrtc/modules/audio_device:audio_device_module_from_input_and_output" ]
 
@@ -286,6 +292,7 @@ static_library("owt_sdk_base") {
       ]
     }
     sources += [
+        "sdk/base/linux/sharedobjectloader.cc",
         "sdk/base/linux/xwindownativeframe.h",
         "sdk/base/linux/videorenderlinux.cc",
         "sdk/base/linux/videorenderlinux.h",
@@ -316,6 +323,10 @@ static_library("owt_sdk_base") {
       "-Wno-reorder",
     ]
   }
+  if (owt_build_ic) {
+    defines += [ "OWT_BUILD_IC" ]
+    data_deps = [ "sdk/ic:owt_ic" ]
+  }
 }
 static_library("owt_sdk_p2p") {
   deps = [
@@ -498,7 +509,10 @@ if (owt_include_tests) {
 
     # Only the root target should depend on this.
     visibility = [ "//:default" ]
-    deps = [ ":owt_unittests" ]
+    deps = [
+      ":owt_unittests",
+      "//talk/owt/sdk/sample",
+    ]
   }
   test("owt_unittests") {
     testonly = true

talk/owt/docs/features/plugins/ic/ic-developer-guide.md

@@ -0,0 +1,31 @@
+# IC Developer Guide
+
+This guide is to introduce how to add a post processor into IC plugin.
+
+The IC plugin is to support Intelligent Collaboration features in the OWT, which
+are background blur, face detection, face framing, eye contact correction and 
+etc. video frame post processing algorithms using deep learning neural networks.
+In this plugin, OpenVINO inference engine is used to do the neural network
+inference.
+
+The IC plugin is compiled to an individual shared library, whose entry is defined
+in `talk/owt/sdk/ic/icmanager.h`. The ICManager will be initialized as a global
+singleton instance and will be used to create the post processor instances.
+
+To introduce a new post processor, you need to create a class in the
+`talk/owt/sdk/ic` directory, inheriting the `owt::ic::VideoFramePostProcessor`
+interface and implement its virtual methods. The post processor's main logic
+should be in the `Process` function, which returns the processed frame buffer.
+You may use inference engine C++ API for neural network inferencing, but be sure
+to catch and handle all exceptions. Exceptions are not expected in the main OWT.
+
+To make it possible for creating your post processor by `CreatePostProcessor`
+method., change the code in `talk/owt/sdk/ic/icmanager.cc`. You may get the
+inference engine core instance from the `ICManager`. You also need to add new
+enumeration item of `owt::ic::ICPostProcessor` in
+`talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h`.
+
+Finally, update the `ic-user-guide.md` to introduce the parameters' usage to the
+user. Also edit the `.gn` files to make the new codes compiled.
+
+See the background blur's code for more detail.

talk/owt/docs/features/plugins/ic/ic-user-guide.md

@@ -0,0 +1,77 @@
+# IC User Guide
+
+The IC plugin is to support Intelligent Collaboration features in the OWT, which
+are background blur, face detection, face framing, eye contact correction and 
+etc. video frame post processing algorithms using deep learning neural networks.
+
+To use the IC plugin, first initialize the global plugin instance in plugin 
+manager.
+```cpp
+owt::ic::ICManagerInterface* ic_plugin = owt::base::PluginManager::ICPlugin();
+```
+If the `owt::ic::ICManagerInterface* ic_plugin` is not `nullptr`, the ic plugin 
+is succesfully initialized. Otherwise, check whether `owt_ic.dll` is in 
+your path. The pointer is managed by `PluginManager` so you do not delete it.
+
+You may get a `std::shared_ptr<owt::base::VideoFramePostProcessor>` by
+using `ic_plugin->CreatePostProcessor(owt::ic::ICPostProcessor)`. Then configure
+the post-processor, and finally add it into the
+`LocalCameraStreamParameters.PostProcessors()`, which is used for creating
+`LocalStream`.
+
+The post-processors will be applied to the produced frame in the order you add
+them.
+
+```cpp
+std::shared_ptr<owt::base::VideoFramePostProcessor> post_processor =
+    ic_plugin->CreatePostProcessor(owt::ic::ICPostProcessor::BACKGROUND_BLUR);
+// do some preparation, see below
+owt::base::LocalCameraStreamParameters param(true, true);
+param.PostProcessors().push_back(post_processor);
+```
+
+See below sections for each post processor's usage.
+
+## Background Blur
+The background blur post processor uses a selfie segmentation neural network 
+model to detect the area of human in frames. The background part will be applied
+with a Gaussian filter to make it blurred. You can customize its blur radius to
+change the blurring degree.
+Use `ic_plugin->CreatePostProcessor(owt::ic::ICPostProcessor::BACKGROUND_BLUR)`
+to get a background blur post processor instance.
+
+### Model
+This post-processor use a neural network model, which can be found at
+[owt-selfie-segmentation-144x256.xml](https://github.com/open-webrtc-toolkit/owt-model-zoo/raw/main/selfie-segmentation/144x256/owt-selfie-segmentation-144x256.xml)
+and 
+[owt-selfie-segmentation-144x256.bin](https://github.com/open-webrtc-toolkit/owt-model-zoo/raw/main/selfie-segmentation/144x256/owt-selfie-segmentation-144x256.bin).
+Download and save them to your project, then call `ReadModel` and `LoadModel` to
+prepare the model. When reading model, you only need to specify the path to
+`.xml` file, and the `.bin` file should be in the same directory, which will be
+automatically loaded by the framework.
+
+Note that the loading process will taken place in the function, so this may be
+time-consuming depends on the model's size. Make sure you call this in a proper
+time. 
+
+The function will return false if there is any error, and the error 
+message will be printed to the log output.
+
+```cpp
+background_blur->ReadModel("/path/to/model.xml");
+background_blur->LoadModel("CPU");
+```
+
+### Parameters
+**blur_radius**: A positive integer representing the blur radius used in
+Gaussian blur processing to the background. The larger the blur radius is, the
+more blurred will the frame be, and vise versa. Zero or negative blur_radius
+will lead to the Gaussian blur taking no effect. Default: 55.
+
+### Sample
+```cpp
+background_blur->SetParameter("blur_radius", 55);
+```
+
+There is a sample program about using background blur, which can be found at
+`talk/owt/sdk/sample/win/sample_background_blur`.