diff --git a/README.md b/README.md index 4778e30e4..84cae5b54 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,9 @@ The following dependencies are for Windows only: - [Visual Studio](https://chromium.googlesource.com/chromium/src/+/refs/heads/main/docs/windows_build_instructions.md#visual-studio). - [Intel Media SDK for Windows, version 2020 R1 or higher](https://software.intel.com/en-us/media-sdk/choose-download/client). +If you want intelligent collaboration video post-processor features, you need: +- [OpenVINO Toolkit](https://www.intel.com/content/www/us/en/developer/tools/openvino-toolkit-download.html). + ### Get the code - Make sure you clone the source code to a directory named `src`. - Create a file named .gclient in the directory above the `src` dir, with these contents: @@ -54,6 +57,7 @@ target_os = [] - The optional `msdk_root` should be set to the directory of your Intel MediaSDK for Windows, version 2020 R1 or higher. This is typically `C:\Program Files (x86)\IntelSWTools\Intel(R) Media SDK 2020 R1\Software Development Kit`. If specified, will enable hardware accelerated video codecs for most of the video codecs. - The optional `--sdk` is to inform the build script to use `lib.exe` that is part of Visual Studio toolchain for merging owt libraries with external openssl libraries. + - To enable intelligent collaboration video post-processors (background blur, etc.), set the `openvino_root` option, which would typically be `--openvino_root "C:\Program Files (x86)\Intel\openvino_2021"`. #### Linux diff --git a/build_overrides/owt.gni b/build_overrides/owt.gni new file mode 100644 index 000000000..d43743147 --- /dev/null +++ b/build_overrides/owt.gni @@ -0,0 +1,10 @@ +# Copyright (C) <2021> Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +declare_args() { + owt_msdk_lib_root = "" + owt_msdk_header_root = "" + owt_build_ic = false + owt_msvcrt = false +} diff --git a/jenkins/windows.jenkinsfile b/jenkins/windows.jenkinsfile index eef26e373..d103c7b40 100644 --- a/jenkins/windows.jenkinsfile +++ b/jenkins/windows.jenkinsfile @@ -35,7 +35,7 @@ pipeline { } steps { echo "$GIT_COMMIT" - bat "%windowsCIPathBranch%_%CHANGE_TARGET%/buildSdk.bat %GIT_COMMIT%" + bat "%windowsCIPathBranch%_%CHANGE_TARGET%/buildSdk.bat %GIT_COMMIT% --with-openvino" } } stage("startMcu") { @@ -73,10 +73,10 @@ pipeline { } steps { echo "Buliding.." - bat "%windowsConferenceCasePath%/build.bat" + bat "%windowsConferenceCasePath%/build.bat --configuration Release-MD" } } - stage("BulidTestP2P") { + stage("BulidP2P") { agent{ node { label "windows" @@ -84,7 +84,7 @@ pipeline { } steps { echo "Buliding.." - bat "%windowsP2PCasePath%/build.bat" + bat "%windowsP2PCasePath%/build.bat --configuration Release-MD" } } stage("TestConference") { @@ -95,7 +95,7 @@ pipeline { } steps { echo "Testing.." - bat "%windowsConferenceCasePath%/runCase.bat" + bat "%windowsConferenceCasePath%/runCase.bat --configuration Release-MD" } } stage("TestP2P") { @@ -106,7 +106,7 @@ pipeline { } steps { echo "Testing.." - bat "%windowsP2PCasePath%/runCase.bat" + bat "%windowsP2PCasePath%/runCase.bat --configuration Release-MD" } } } diff --git a/scripts/build-win.py b/scripts/build-win.py index 4b6d77eb1..86351c054 100644 --- a/scripts/build-win.py +++ b/scripts/build-win.py @@ -11,6 +11,7 @@ ''' import os +import re import sys import subprocess import argparse @@ -35,7 +36,7 @@ ] -def gngen(arch, ssl_root, msdk_root, quic_root, scheme, tests): +def gngen(arch, ssl_root, msdk_root, quic_root, openvino_root, scheme, tests): gn_args = list(GN_ARGS) gn_args.append('target_cpu="%s"' % arch) using_llvm = False @@ -76,6 +77,20 @@ def gngen(arch, ssl_root, msdk_root, quic_root, scheme, tests): else: return False gn_args.append('owt_use_quic=true') + if openvino_root: + gn_args.append('owt_build_ic=true') + gn_args.append('owt_msvcrt=true') + gn_args.append('owt_openvino_root="{}"'.format(openvino_root)) + opencv_root = os.path.join(openvino_root, 'opencv') + opencv_version_bin = os.path.join(opencv_root, 'bin', 'opencv_version.exe') + try: + output = subprocess.check_output([opencv_version_bin]) + opencv_version = ''.join(re.findall('\d', output.decode())) + gn_args.append('owt_opencv_root="{}"'.format(opencv_root)) + gn_args.append('owt_opencv_version={}'.format(opencv_version)) + except FileNotFoundError as e: + print('File not found: {}'.format(opencv_version_bin)) + return False if tests: gn_args.append('rtc_include_tests=true') gn_args.append('owt_include_tests=true') @@ -183,6 +198,7 @@ def main(): parser.add_argument('--ssl_root', help='Path for OpenSSL.') parser.add_argument('--msdk_root', help='Path for MSDK.') parser.add_argument('--quic_root', help='Path to QUIC library') + parser.add_argument('--openvino_root', help='Path for OpenVINO.') parser.add_argument('--scheme', default='debug', choices=('debug', 'release'), help='Schemes for building. Supported value: debug, release') parser.add_argument('--gn_gen', default=False, action='store_true', @@ -195,19 +211,16 @@ def main(): help='To generate the API document.') parser.add_argument('--output_path', help='Path to copy sdk.') opts = parser.parse_args() - if opts.ssl_root and not os.path.exists(os.path.expanduser(opts.ssl_root)): - print('Invalid ssl_root.') - return 1 - if opts.msdk_root and not os.path.exists(os.path.expanduser(opts.msdk_root)): - print('Invalid msdk_root') - return 1 - if opts.quic_root and not os.path.exists(os.path.expanduser(opts.quic_root)): - print('Invalid quic_root') - return 1 + for name in ['ssl', 'msdk', 'quic', 'openvino']: + attr = name + '_root' + path = getattr(opts, attr) + if path is not None and not os.path.exists(os.path.expanduser(path)): + print('Invalid {}.'.format(attr)) + return 1 print(opts) if opts.gn_gen: if not gngen(opts.arch, opts.ssl_root, opts.msdk_root, opts.quic_root, - opts.scheme, opts.tests): + opts.openvino_root, opts.scheme, opts.tests): return 1 if opts.sdk: if not ninjabuild(opts.arch, opts.scheme): diff --git a/scripts/prepare_dev.py b/scripts/prepare_dev.py index d6ed6a8fe..df791cd6e 100644 --- a/scripts/prepare_dev.py +++ b/scripts/prepare_dev.py @@ -45,7 +45,8 @@ ('0014-Fix-missing-ffmpeg-configure-item-for-msvc-build.patch', FFMPEG_PATH), ('0015-Remove-custom-d8-dependency.patch', BUILD_PATH), ('0016-Remove-deprecated-create_srcjar-property.patch', THIRD_PARTY_PATH), - ('0017-Build-libvpx-with-RTC-rate-control-impl-included.patch', THIRD_PARTY_PATH) + ('0017-Build-libvpx-with-RTC-rate-control-impl-included.patch', THIRD_PARTY_PATH), + ('0018-Enable-msvcrt-switch.patch', BUILD_PATH), ] def _patch(ignoreFailures=False): diff --git a/talk/owt/BUILD.gn b/talk/owt/BUILD.gn index 792b2dd88..4c594f30c 100644 --- a/talk/owt/BUILD.gn +++ b/talk/owt/BUILD.gn @@ -3,12 +3,11 @@ # SPDX-License-Identifier: Apache-2.0 import("//build_overrides/webrtc.gni") +import("//build_overrides/owt.gni") import("//testing/test.gni") declare_args() { include_internal_audio_device = true - owt_msdk_lib_root = "" - owt_msdk_header_root = "" } # Introduced for using libvpx config files. We only enable libvpx rate @@ -109,6 +108,7 @@ static_library("owt_sdk_base") { "sdk/base/peerconnectionchannel.h", "sdk/base/peerconnectiondependencyfactory.cc", "sdk/base/peerconnectiondependencyfactory.h", + "sdk/base/pluginmanager.cc", "sdk/base/sdputils.cc", "sdk/base/sdputils.h", "sdk/base/stream.cc", @@ -128,7 +128,9 @@ static_library("owt_sdk_base") { "sdk/include/cpp/owt/base/framegeneratorinterface.h", "sdk/include/cpp/owt/base/localcamerastreamparameters.h", "sdk/include/cpp/owt/base/logging.h", + "sdk/include/cpp/owt/base/pluginmanager.h", "sdk/include/cpp/owt/base/stream.h", + "sdk/include/cpp/owt/base/videoframepostprocessor.h", "sdk/include/cpp/owt/base/videorendererinterface.h", ] if (is_win || is_linux) { @@ -141,10 +143,13 @@ static_library("owt_sdk_base") { "sdk/base/customizedvideosource.h", "sdk/base/encodedvideoencoderfactory.cc", "sdk/base/encodedvideoencoderfactory.h", + "sdk/base/sharedobjectloader.h", + "sdk/base/sharedobjectpointer.h", "sdk/base/webrtcvideorendererimpl.cc", "sdk/base/webrtcvideorendererimpl.h", "sdk/base/windowcapturer.cc", "sdk/include/cpp/owt/base/videodecoderinterface.h", + "sdk/include/cpp/owt/ic/icmanagerinterface.h", ] } public_deps = [ @@ -261,6 +266,7 @@ static_library("owt_sdk_base") { "sdk/base/win/device_info_mf.cc", "sdk/base/win/video_capture_mf.h", "sdk/base/win/video_capture_mf.cc", + "sdk/base/win/sharedobjectloader.cc", ] public_deps += [ "//third_party/webrtc/modules/audio_device:audio_device_module_from_input_and_output" ] @@ -286,6 +292,7 @@ static_library("owt_sdk_base") { ] } sources += [ + "sdk/base/linux/sharedobjectloader.cc", "sdk/base/linux/xwindownativeframe.h", "sdk/base/linux/videorenderlinux.cc", "sdk/base/linux/videorenderlinux.h", @@ -316,6 +323,10 @@ static_library("owt_sdk_base") { "-Wno-reorder", ] } + if (owt_build_ic) { + defines += [ "OWT_BUILD_IC" ] + data_deps = [ "sdk/ic:owt_ic" ] + } } static_library("owt_sdk_p2p") { deps = [ @@ -498,7 +509,10 @@ if (owt_include_tests) { # Only the root target should depend on this. visibility = [ "//:default" ] - deps = [ ":owt_unittests" ] + deps = [ + ":owt_unittests", + "//talk/owt/sdk/sample", + ] } test("owt_unittests") { testonly = true diff --git a/talk/owt/docs/features/plugins/ic/ic-developer-guide.md b/talk/owt/docs/features/plugins/ic/ic-developer-guide.md new file mode 100644 index 000000000..6236a77fa --- /dev/null +++ b/talk/owt/docs/features/plugins/ic/ic-developer-guide.md @@ -0,0 +1,31 @@ +# IC Developer Guide + +This guide is to introduce how to add a post processor into IC plugin. + +The IC plugin is to support Intelligent Collaboration features in the OWT, which +are background blur, face detection, face framing, eye contact correction and +etc. video frame post processing algorithms using deep learning neural networks. +In this plugin, OpenVINO inference engine is used to do the neural network +inference. + +The IC plugin is compiled to an individual shared library, whose entry is defined +in `talk/owt/sdk/ic/icmanager.h`. The ICManager will be initialized as a global +singleton instance and will be used to create the post processor instances. + +To introduce a new post processor, you need to create a class in the +`talk/owt/sdk/ic` directory, inheriting the `owt::ic::VideoFramePostProcessor` +interface and implement its virtual methods. The post processor's main logic +should be in the `Process` function, which returns the processed frame buffer. +You may use inference engine C++ API for neural network inferencing, but be sure +to catch and handle all exceptions. Exceptions are not expected in the main OWT. + +To make it possible for creating your post processor by `CreatePostProcessor` +method., change the code in `talk/owt/sdk/ic/icmanager.cc`. You may get the +inference engine core instance from the `ICManager`. You also need to add new +enumeration item of `owt::ic::ICPostProcessor` in +`talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h`. + +Finally, update the `ic-user-guide.md` to introduce the parameters' usage to the +user. Also edit the `.gn` files to make the new codes compiled. + +See the background blur's code for more detail. diff --git a/talk/owt/docs/features/plugins/ic/ic-user-guide.md b/talk/owt/docs/features/plugins/ic/ic-user-guide.md new file mode 100644 index 000000000..c3e407041 --- /dev/null +++ b/talk/owt/docs/features/plugins/ic/ic-user-guide.md @@ -0,0 +1,77 @@ +# IC User Guide + +The IC plugin is to support Intelligent Collaboration features in the OWT, which +are background blur, face detection, face framing, eye contact correction and +etc. video frame post processing algorithms using deep learning neural networks. + +To use the IC plugin, first initialize the global plugin instance in plugin +manager. +```cpp +owt::ic::ICManagerInterface* ic_plugin = owt::base::PluginManager::ICPlugin(); +``` +If the `owt::ic::ICManagerInterface* ic_plugin` is not `nullptr`, the ic plugin +is succesfully initialized. Otherwise, check whether `owt_ic.dll` is in +your path. The pointer is managed by `PluginManager` so you do not delete it. + +You may get a `std::shared_ptr` by +using `ic_plugin->CreatePostProcessor(owt::ic::ICPostProcessor)`. Then configure +the post-processor, and finally add it into the +`LocalCameraStreamParameters.PostProcessors()`, which is used for creating +`LocalStream`. + +The post-processors will be applied to the produced frame in the order you add +them. + +```cpp +std::shared_ptr post_processor = + ic_plugin->CreatePostProcessor(owt::ic::ICPostProcessor::BACKGROUND_BLUR); +// do some preparation, see below +owt::base::LocalCameraStreamParameters param(true, true); +param.PostProcessors().push_back(post_processor); +``` + +See below sections for each post processor's usage. + +## Background Blur +The background blur post processor uses a selfie segmentation neural network +model to detect the area of human in frames. The background part will be applied +with a Gaussian filter to make it blurred. You can customize its blur radius to +change the blurring degree. +Use `ic_plugin->CreatePostProcessor(owt::ic::ICPostProcessor::BACKGROUND_BLUR)` +to get a background blur post processor instance. + +### Model +This post-processor use a neural network model, which can be found at +[owt-selfie-segmentation-144x256.xml](https://github.com/open-webrtc-toolkit/owt-model-zoo/raw/main/selfie-segmentation/144x256/owt-selfie-segmentation-144x256.xml) +and +[owt-selfie-segmentation-144x256.bin](https://github.com/open-webrtc-toolkit/owt-model-zoo/raw/main/selfie-segmentation/144x256/owt-selfie-segmentation-144x256.bin). +Download and save them to your project, then call `ReadModel` and `LoadModel` to +prepare the model. When reading model, you only need to specify the path to +`.xml` file, and the `.bin` file should be in the same directory, which will be +automatically loaded by the framework. + +Note that the loading process will taken place in the function, so this may be +time-consuming depends on the model's size. Make sure you call this in a proper +time. + +The function will return false if there is any error, and the error +message will be printed to the log output. + +```cpp +background_blur->ReadModel("/path/to/model.xml"); +background_blur->LoadModel("CPU"); +``` + +### Parameters +**blur_radius**: A positive integer representing the blur radius used in +Gaussian blur processing to the background. The larger the blur radius is, the +more blurred will the frame be, and vise versa. Zero or negative blur_radius +will lead to the Gaussian blur taking no effect. Default: 55. + +### Sample +```cpp +background_blur->SetParameter("blur_radius", 55); +``` + +There is a sample program about using background blur, which can be found at +`talk/owt/sdk/sample/win/sample_background_blur`. diff --git a/talk/owt/docs/features/plugins/plugin-developer-guide.md b/talk/owt/docs/features/plugins/plugin-developer-guide.md new file mode 100644 index 000000000..f0cf424d3 --- /dev/null +++ b/talk/owt/docs/features/plugins/plugin-developer-guide.md @@ -0,0 +1,94 @@ +# Plugin Developer Guide + +There is an `owt::base::PluginManager` managing all DLL plugins. For new dynamic +linked library, you may use `PluginManager` to manage its loading and unloading +process, and get accessed from the `PluginManager` singleton instance. + +## OWT side +To avoid resource release issues, the DLL is expected to be loaded before the +first time it is used, and to be unloaded when the program exits. The lifecycle +of the DLL will be managed by `owt::base::SharedObjectPointer`. + +In `pluginmanager.h`, add one method to access the singleton instance of your +plugin +```cpp +static owt::your_namespace::YourPluginInterface* YourPlugin(); +``` +In `pluginmanager.cc`, implement a template specialization of `SOTrait` to +assign the suffix of the name of C function exported in the DLL. Then implement +a singleton getter. +```cpp +template <> +struct SOTrait { + static constexpr auto name = "YourPlugin"; +}; + +owt::your_namespace::YourPluginInterface* PluginManager::YourPlugin() { + static owt::base::SharedObjectPointer + your_plugin("dll_name" DLL_SUFFIX); + return your_plugin.Get(); +} +``` + +## DLL side +There should be a `YourPlugin` entrance class inheriting the +`YourPluginInterface`. As is designed in the `PluginManager` the `YourPlugin` +will have only one instance. Do the necessary resource acquisition in the +constructor and release them in the destructor. Using static members or +variables is not recommended. + +Create your class instance as a shared pointer, making it easy to be destructed +using the destructor defined in the DLL. + +Finally, create a creator and destroyer of your plugin entrance class, naming +them the name you set in the `SOTrait` template specialization with a `Create` +or `Destroy` prefix. + +In `yourplugin.h`: +```cpp +#include + +namespace owt { +namespace your_namespace { + +class YourPlugin : public YourPluginInterface { +public: + YourPlugin() = default; + ~YourPlugin() override = default; + + std::shared_ptr CreateYourClass() override; +}; + +} // namespace your_namespace +} // namespace owt + +YourPluginInterface* CreateYourPlugin(); + +void DestroyYourPlugin(YourPluginInterface* ptr); +``` + +In `yourplugin.cc`: +```cpp +#include "path/to/yourplugin.h" + +namespace owt { +namespace your_namespace { + +std::shared_ptr YourPlugin::CreateYourClass() override { + return std::make_shared(); +} + +} // namespace your_namespace +} // namespace owt + +YourPluginInterface* CreateYourPlugin() { + return new YourPlugin; +} + +void DestroyYourPlugin(YourPluginInterface* ptr) { + delete ptr; +} +``` + +Add your files into a `shared_library` target named `dll_name` (the name you +used in the `pluginmanager.cc`) in the `BUILD.gn` file. diff --git a/talk/owt/patches/0018-Enable-msvcrt-switch.patch b/talk/owt/patches/0018-Enable-msvcrt-switch.patch new file mode 100644 index 000000000..69d0c1d62 --- /dev/null +++ b/talk/owt/patches/0018-Enable-msvcrt-switch.patch @@ -0,0 +1,33 @@ +From 0105cb7e2fe398e7707d19089d063d2d87418018 Mon Sep 17 00:00:00 2001 +From: Zhibo Wang +Date: Fri, 19 Nov 2021 15:04:19 +0800 +Subject: [PATCH] Enable msvcrt switch + +--- + config/win/BUILD.gn | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/config/win/BUILD.gn b/config/win/BUILD.gn +index 51437c66c..1d02e0600 100644 +--- a/config/win/BUILD.gn ++++ b/config/win/BUILD.gn +@@ -11,6 +11,7 @@ import("//build/config/win/visual_studio_version.gni") + import("//build/timestamp.gni") + import("//build/toolchain/goma.gni") + import("//build/toolchain/toolchain.gni") ++import("//build_overrides/owt.gni") + + assert(is_win) + +@@ -473,7 +474,7 @@ config("delayloads_not_for_child_dll") { + # See https://msdn.microsoft.com/en-us/library/2kzt1wy3.aspx for a reference of + # what each value does. + config("default_crt") { +- if (is_component_build) { ++ if (is_component_build || owt_msvcrt) { + # Component mode: dynamic CRT. Since the library is shared, it requires + # exceptions or will give errors about things not matching, so keep + # exceptions on. +-- +2.33.0 + diff --git a/talk/owt/sdk/base/cameravideocapturer.cc b/talk/owt/sdk/base/cameravideocapturer.cc index 62f270666..4f333df27 100644 --- a/talk/owt/sdk/base/cameravideocapturer.cc +++ b/talk/owt/sdk/base/cameravideocapturer.cc @@ -36,14 +36,25 @@ void CameraVideoCapturer::OnFrame(const webrtc::VideoFrame& frame) { return; } + rtc::scoped_refptr buffer = + frame.video_frame_buffer(); + if (out_height != frame.height() || out_width != frame.width()) { // Video adapter has requested a down-scale. Allocate a new buffer and // return scaled version. rtc::scoped_refptr scaled_buffer = webrtc::I420Buffer::Create(out_width, out_height); scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420()); + buffer = scaled_buffer; + } + + for (auto& post_processor : video_frame_post_processors_) { + buffer = post_processor->Process(buffer); + } + + if (buffer != frame.video_frame_buffer()) { broadcaster_.OnFrame(webrtc::VideoFrame::Builder() - .set_video_frame_buffer(scaled_buffer) + .set_video_frame_buffer(buffer) .set_rotation(webrtc::kVideoRotation_0) .set_timestamp_us(frame.timestamp_us()) .set_id(frame.id()) @@ -71,6 +82,11 @@ void CameraVideoCapturer::RemoveSink( UpdateVideoAdapter(); } +void CameraVideoCapturer::AddVideoFramePostProcessor( + std::shared_ptr post_processor) { + video_frame_post_processors_.emplace_back(post_processor); +} + void CameraVideoCapturer::UpdateVideoAdapter() { video_adapter_.OnSinkWants(broadcaster_.wants()); } diff --git a/talk/owt/sdk/base/cameravideocapturer.h b/talk/owt/sdk/base/cameravideocapturer.h index 0941190fe..649eccfe5 100644 --- a/talk/owt/sdk/base/cameravideocapturer.h +++ b/talk/owt/sdk/base/cameravideocapturer.h @@ -13,12 +13,15 @@ #include #include +#include #include "api/video/video_frame.h" #include "api/video/video_source_interface.h" #include "media/base/video_adapter.h" #include "media/base/video_broadcaster.h" +#include "talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h" + // This file is borrowed from webrtc project namespace owt { namespace base { @@ -32,6 +35,9 @@ class CameraVideoCapturer : public rtc::VideoSourceInterface const rtc::VideoSinkWants& wants) override; void RemoveSink(rtc::VideoSinkInterface* sink) override; + void AddVideoFramePostProcessor( + const std::shared_ptr post_processor); + protected: void OnFrame(const webrtc::VideoFrame& frame); rtc::VideoSinkWants GetSinkWants(); @@ -41,6 +47,8 @@ class CameraVideoCapturer : public rtc::VideoSourceInterface rtc::VideoBroadcaster broadcaster_; cricket::VideoAdapter video_adapter_; + std::vector> + video_frame_post_processors_; }; } // namespace base } // namespace owt diff --git a/talk/owt/sdk/base/linux/sharedobjectloader.cc b/talk/owt/sdk/base/linux/sharedobjectloader.cc new file mode 100644 index 000000000..024fad359 --- /dev/null +++ b/talk/owt/sdk/base/linux/sharedobjectloader.cc @@ -0,0 +1,26 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#include "talk/owt/sdk/base/sharedobjectloader.h" + +#include + +namespace owt { +namespace base { + +SharedObjectLoader::SharedObjectLoader(const char* path) + : shared_object(dlopen(path, RTLD_NOW), dlclose) {} + +SharedObjectLoader::~SharedObjectLoader() {} + +bool SharedObjectLoader::IsLoaded() const { + return shared_object_.get(); +} + +void* SharedObjectLoader::get_symbol(const char* name) const { + return shared_object_ ? dlsym(shared_object.get(), name) : nullptr; +} + +} // namespace base +} // namespace owt diff --git a/talk/owt/sdk/base/localcamerastreamparameters.cc b/talk/owt/sdk/base/localcamerastreamparameters.cc index ed40131c6..41be59a21 100644 --- a/talk/owt/sdk/base/localcamerastreamparameters.cc +++ b/talk/owt/sdk/base/localcamerastreamparameters.cc @@ -29,6 +29,10 @@ void LocalCameraStreamParameters::Resolution(int width, int height) { void LocalCameraStreamParameters::StreamName(const std::string& stream_name){ stream_name_ = stream_name; } +std::vector>& +LocalCameraStreamParameters::PostProcessors() { + return post_processors_; +} LocalDesktopStreamParameters::LocalDesktopStreamParameters( bool audio_enabled, bool video_enabled) diff --git a/talk/owt/sdk/base/peerconnectiondependencyfactory.cc b/talk/owt/sdk/base/peerconnectiondependencyfactory.cc index cf9e3210a..b9cd097c7 100644 --- a/talk/owt/sdk/base/peerconnectiondependencyfactory.cc +++ b/talk/owt/sdk/base/peerconnectiondependencyfactory.cc @@ -47,6 +47,7 @@ #pragma comment(lib, "wmcodecdspuuid.lib") #pragma comment(lib, "amstrmid.lib") #pragma comment(lib, "strmiids.lib") +#pragma comment(lib, "dcomp.lib") #ifdef OWT_USE_MSDK #pragma comment(lib, "mf.lib") #pragma comment(lib, "mfplat.lib") diff --git a/talk/owt/sdk/base/pluginmanager.cc b/talk/owt/sdk/base/pluginmanager.cc new file mode 100644 index 000000000..51ef9f1b5 --- /dev/null +++ b/talk/owt/sdk/base/pluginmanager.cc @@ -0,0 +1,32 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#include "talk/owt/sdk/include/cpp/owt/base/pluginmanager.h" + +#include "talk/owt/sdk/base/sharedobjectpointer.h" + +#if defined(WEBRTC_WIN) +#define DLL_SUFFIX ".dll" +#elif defined(WEBRTC_LINUX) +#define DLL_SUFFIX ".so" +#endif + +namespace owt { +namespace base { + +#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) +template <> +struct SOTrait { + static constexpr auto name = "ICManager"; +}; + +owt::ic::ICManagerInterface* PluginManager::ICPlugin() { + static owt::base::SharedObjectPointer ic_plugin( + "owt_ic" DLL_SUFFIX); + return ic_plugin.Get(); +} +#endif + +} // namespace base +} // namespace owt diff --git a/talk/owt/sdk/base/sharedobjectloader.h b/talk/owt/sdk/base/sharedobjectloader.h new file mode 100644 index 000000000..5452bde1b --- /dev/null +++ b/talk/owt/sdk/base/sharedobjectloader.h @@ -0,0 +1,31 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_BASE_SHAREDOBJECTLOADER_H_ +#define OWT_BASE_SHAREDOBJECTLOADER_H_ + +#include +#include + +namespace owt { +namespace base { + +// This class is derived from openvino inference engine +class SharedObjectLoader { + public: + explicit SharedObjectLoader(const char* path); + ~SharedObjectLoader(); + + bool IsLoaded() const; + void* GetSymbol(const char* name) const; + void* GetSymbol(const std::string& name) const; + + private: + std::shared_ptr shared_object_; +}; + +} // namespace base +} // namespace owt + +#endif // OWT_BASE_SHAREDOBJECTLOADER_H_ diff --git a/talk/owt/sdk/base/sharedobjectpointer.h b/talk/owt/sdk/base/sharedobjectpointer.h new file mode 100644 index 000000000..75b4ae236 --- /dev/null +++ b/talk/owt/sdk/base/sharedobjectpointer.h @@ -0,0 +1,61 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_BASE_SHAREDOBJECTPOINTER_H_ +#define OWT_BASE_SHAREDOBJECTPOINTER_H_ + +#include "talk/owt/sdk/base/sharedobjectloader.h" + +#include +#include + +#include "third_party/webrtc/rtc_base/logging.h" + +namespace owt { +namespace base { + +// This class is derived from openvino inference engine +template +struct SOTrait {}; + +// This class is derived from openvino inference engine +template +class SharedObjectPointer { + public: + explicit SharedObjectPointer(const char* path) : so_(path) { Load(); } + + bool IsLoaded() const { return so_.IsLoaded(); } + + T* Get() { return ptr_.get(); } + + protected: + void Load() { + if (!so_.IsLoaded()) { + return; + } + + using Creator = T*(); + using Destroyer = void(T*); + Creator* creator = reinterpret_cast( + so_.GetSymbol(std::string("Create") + SOTrait::name)); + Destroyer* destroyer = reinterpret_cast( + so_.GetSymbol(std::string("Destroy") + SOTrait::name)); + + if (creator && destroyer) { + ptr_ = std::shared_ptr(creator(), destroyer); + } else { + RTC_LOG(LS_WARNING) << "Create" << SOTrait::name // + << " or Destroy" << SOTrait::name + << " is missing in the shared object."; + } + } + + SharedObjectLoader so_; + std::shared_ptr ptr_; +}; + +} // namespace base +} // namespace owt + +#endif // OWT_BASE_SHAREDOBJECTPOINTER_H_ diff --git a/talk/owt/sdk/base/stream.cc b/talk/owt/sdk/base/stream.cc index 4f9a7872e..f795a7fd2 100644 --- a/talk/owt/sdk/base/stream.cc +++ b/talk/owt/sdk/base/stream.cc @@ -45,7 +45,9 @@ class CapturerTrackSource : public webrtc::VideoTrackSource { const size_t width, const size_t height, const size_t fps, - int capture_device_idx) { + int capture_device_idx, + const std::vector>& + post_processors) { std::unique_ptr capturer; std::unique_ptr info( webrtc::VideoCaptureFactory::CreateDeviceInfo()); @@ -56,6 +58,9 @@ class CapturerTrackSource : public webrtc::VideoTrackSource { for (int i = 0; i < num_devices; ++i) { capturer = absl::WrapUnique(owt::base::VcmCapturer::Create( width, height, fps, capture_device_idx)); + for (auto& post_processor : post_processors) { + capturer->AddVideoFramePostProcessor(post_processor); + } if (capturer) { return new rtc::RefCountedObject( std::move(capturer)); @@ -513,7 +518,8 @@ LocalStream::LocalStream(const LocalCameraStreamParameters& parameters, CapturerTrackSource::Create( parameters.ResolutionWidth(), parameters.ResolutionHeight(), parameters.Fps(), - DeviceUtils::GetVideoCaptureDeviceIndex(parameters.CameraId())); + DeviceUtils::GetVideoCaptureDeviceIndex(parameters.CameraId()), + parameters.PostProcessors()); #else capturer_ = ObjcVideoCapturerFactory::Create(parameters); if (!capturer_) { diff --git a/talk/owt/sdk/base/win/sharedobjectloader.cc b/talk/owt/sdk/base/win/sharedobjectloader.cc new file mode 100644 index 000000000..7143d11a2 --- /dev/null +++ b/talk/owt/sdk/base/win/sharedobjectloader.cc @@ -0,0 +1,58 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#include "talk/owt/sdk/base/sharedobjectloader.h" + +#include +#include + +#include "third_party/webrtc/rtc_base/logging.h" + +namespace owt { +namespace base { +namespace { + +std::string GetErrorAsString(DWORD message_id) { + LPSTR buffer = NULL; + size_t size = FormatMessageA( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, + message_id, MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US), (LPSTR)&buffer, + 0, NULL); + std::string message(buffer, size); + LocalFree(buffer); + return message; +} + +} // namespace + +SharedObjectLoader::SharedObjectLoader(const char* path) + : shared_object_(LoadLibraryA(path), FreeLibrary) { + if (!shared_object_) { + DWORD message_id = GetLastError(); + RTC_LOG(LS_WARNING) << "Load shared library " << path << " failed. " + << "Error code " << message_id << ": " + << GetErrorAsString(message_id); + } +} + +SharedObjectLoader::~SharedObjectLoader() {} + +bool SharedObjectLoader::IsLoaded() const { + return shared_object_.get(); +} + +void* SharedObjectLoader::GetSymbol(const char* name) const { + if (shared_object_) { + return GetProcAddress(reinterpret_cast(shared_object_.get()), + name); + } + return nullptr; +} + +void* SharedObjectLoader::GetSymbol(const std::string& name) const { + return GetSymbol(name.c_str()); +} + +} // namespace base +} // namespace owt diff --git a/talk/owt/sdk/ic/BUILD.gn b/talk/owt/sdk/ic/BUILD.gn new file mode 100644 index 000000000..b73baecd8 --- /dev/null +++ b/talk/owt/sdk/ic/BUILD.gn @@ -0,0 +1,146 @@ +# Copyright (C) <2021> Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +import("//build_overrides/owt.gni") +import("//build_overrides/webrtc.gni") +import("//testing/test.gni") + +declare_args() { + owt_openvino_root = "" + owt_openvino_source_code = false + owt_opencv_root = "" + owt_opencv_version = 0 +} + +# Currently OpenVINO is used only in IC plugin, so check args in file scope. +assert(owt_openvino_root != "", "owt_openvino_root is not set") +assert(owt_opencv_root != "" && owt_opencv_version != 0, + "owt_opencv_root or owt_opencv_version is not set") + +if (is_debug) { + capitalized_scheme = "Debug" + lib_suffix = "d.lib" + dll_suffix = "d.dll" +} else { + capitalized_scheme = "Release" + lib_suffix = ".lib" + dll_suffix = ".dll" +} + +if (owt_openvino_source_code) { + inference_engine_root = owt_openvino_root + "/inference-engine" + bin_dir = owt_openvino_root + "/bin/intel64/" + capitalized_scheme + lib_dir = bin_dir + ngraph_bin_dir = bin_dir + tbb_bin_dir = inference_engine_root + "/temp/tbb/bin" + openvino_include_dirs = [ + inference_engine_root + "/src/inference_engine/include/ie", + owt_openvino_root + "/ngraph/core/include", + ] +} else { + inference_engine_root = owt_openvino_root + "/inference_engine" + bin_dir = inference_engine_root + "/bin/intel64/" + capitalized_scheme + lib_dir = inference_engine_root + "/lib/intel64/" + capitalized_scheme + ngraph_bin_dir = owt_openvino_root + "/deployment_tools/ngraph/lib" + tbb_bin_dir = inference_engine_root + "/external/tbb/bin" + openvino_include_dirs = [ inference_engine_root + "/include" ] +} + +openvino_modules = [ + "ir_reader", + "lp_transformations", + "preproc", + "transformations", +] + +opencv_modules = [ + "core", + "highgui", + "imgcodecs", + "imgproc", + "videoio", +] + +config("openvino") { + defines = [ "OWT_USE_OPENVINO" ] + if (is_debug) { + defines += [ "_ITERATOR_DEBUG_LEVEL=2" ] + } + include_dirs = openvino_include_dirs + lib_dirs = [ lib_dir ] +} + +config("openvino_opencv") { + defines = [ "OWT_USE_OPENCV" ] + include_dirs = [ owt_opencv_root + "/include" ] + lib_dirs = [ owt_opencv_root + "/lib" ] +} + +shared_library("owt_ic") { + sources = [ + "backgroundblur.cc", + "backgroundblur.h", + "icmanager.cc", + "icmanager.h", + "selfiesegmentationmodel.cc", + "selfiesegmentationmodel.h", + ] + defines = [ "OWT_IC_IMPL" ] + configs -= [ + "//build/config/compiler:no_exceptions", + "//build/config/compiler:no_rtti", + ] + configs += [ + ":openvino", + ":openvino_opencv", + ] + include_dirs = [ "//talk/owt/sdk/include/cpp" ] + libs = [ + "inference_engine" + lib_suffix, + "inference_engine_transformations" + lib_suffix, + ] + foreach(module, opencv_modules) { + libs += [ "opencv_" + module + owt_opencv_version + lib_suffix ] + } + deps = [ + "//third_party/libyuv:libyuv", + "//third_party/webrtc/api/video:video_frame", + ] + data_deps = [ + ":openvino_library", + ":openvino_opencv_library", + ] +} + +copy("openvino_library") { + sources = [ ngraph_bin_dir + "/ngraph" + dll_suffix ] + if (is_debug) { + sources += [ tbb_bin_dir + "/tbb_debug.dll" ] + } else { + sources += [ tbb_bin_dir + "/tbb.dll" ] + } + sources += [ + bin_dir + "/inference_engine" + dll_suffix, + bin_dir + "/MKLDNNPlugin" + dll_suffix, + ] + foreach(module, openvino_modules) { + sources += [ bin_dir + "/inference_engine_" + module + dll_suffix ] + } + if (owt_openvino_source_code && is_debug) { + foreach(dll_file, sources) { + sources += [ string_replace(dll_file, ".dll", ".pdb") ] + } + } + sources += [ bin_dir + "/plugins.xml" ] + outputs = [ "$root_build_dir/{{source_file_part}}" ] +} + +copy("openvino_opencv_library") { + sources = [] + foreach(module, opencv_modules) { + sources += [ owt_opencv_root + "/bin/opencv_" + module + + owt_opencv_version + dll_suffix ] + } + outputs = [ "$root_build_dir/{{source_file_part}}" ] +} diff --git a/talk/owt/sdk/ic/backgroundblur.cc b/talk/owt/sdk/ic/backgroundblur.cc new file mode 100644 index 000000000..1e995a0dd --- /dev/null +++ b/talk/owt/sdk/ic/backgroundblur.cc @@ -0,0 +1,113 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#include "backgroundblur.h" + +#include +#include + +#include "libyuv/convert.h" +#include "opencv2/imgproc.hpp" +#include "third_party/webrtc/api/video/i420_buffer.h" +#include "third_party/webrtc/rtc_base/logging.h" + +#include "selfiesegmentationmodel.h" + +namespace owt { +namespace ic { + +BackgroundBlur::BackgroundBlur(InferenceEngine::Core& core) : model_(core) {} + +bool BackgroundBlur::ReadModel(const std::string& modelXmlPath) { + try { + model_.ReadModel(modelXmlPath); + } catch (const std::exception& e) { + RTC_LOG(LS_ERROR) << "Read model failed: " << e.what(); + return false; + } + return true; +} + +bool BackgroundBlur::LoadModel(const std::string& device) { + try { + model_.LoadModel(device); + } catch (const std::exception& e) { + RTC_LOG(LS_ERROR) << "Load model failed: " << e.what(); + return false; + } + return true; +} + +bool BackgroundBlur::SetParameter(const std::string& key, int value) { + if (key == "blur_radius") { + blur_radius_ = value; + return true; + } + return owt::base::VideoFramePostProcessor::SetParameter(key, value); +} + +rtc::scoped_refptr BackgroundBlur::Process( + const rtc::scoped_refptr& buffer) { + if (!model_.IsLoaded()) { + RTC_LOG(LS_WARNING) << "Background blur model is not loaded."; + return buffer; + } + + if (buffer->type() == webrtc::VideoFrameBuffer::Type::kNative) { + RTC_LOG(LS_WARNING) << "Native video frame buffer is not supported."; + return buffer; + } + + auto i420 = buffer->GetI420(); + std::vector rgb(3ll * i420->width() * i420->height()); + libyuv::I420ToRAW(i420->DataY(), i420->StrideY(), i420->DataU(), + i420->StrideU(), i420->DataV(), i420->StrideV(), rgb.data(), + i420->width() * 3, i420->width(), i420->height()); + cv::Mat frame(buffer->height(), buffer->width(), CV_8UC3, rgb.data()); + + cv::Mat input; + frame.convertTo(input, CV_32FC3, 1. / UCHAR_MAX); + cv::Mat mask; + try { + mask = model_.Predict(input); // mask is of 8UC1 + } catch (const std::exception& e) { + RTC_LOG(LS_ERROR) << e.what(); + return buffer; + } + + cv::resize(mask, mask, {buffer->width(), buffer->height()}); + cv::Mat foreground_mask; + cv::merge(std::vector{mask, mask, mask}, foreground_mask); + cv::Mat background_mask; + cv::merge(std::vector{UCHAR_MAX - mask, UCHAR_MAX - mask, + UCHAR_MAX - mask}, + background_mask); + + // Apply a masked blur on background + cv::Mat masked_frame; + cv::multiply(frame, background_mask, masked_frame, 1. / UCHAR_MAX); + cv::Mat background; + cv::GaussianBlur(masked_frame, background, {blur_radius_, blur_radius_}, 0); + cv::Mat blurred_mask; + cv::GaussianBlur(background_mask, blurred_mask, {blur_radius_, blur_radius_}, + 0); + cv::divide(background, blurred_mask, background, UCHAR_MAX); + cv::multiply(background, background_mask, background, 1. / UCHAR_MAX); + + cv::Mat foreground; + cv::multiply(frame, foreground_mask, foreground, 1. / UCHAR_MAX); + frame = foreground + background; + + auto new_buffer = + webrtc::I420Buffer::Create(buffer->width(), buffer->height()); + libyuv::RAWToI420(frame.data, frame.cols * 3, new_buffer->MutableDataY(), + new_buffer->StrideY(), new_buffer->MutableDataU(), + new_buffer->StrideU(), new_buffer->MutableDataV(), + new_buffer->StrideV(), buffer->width(), buffer->height()); + + return new_buffer; +} + +} // namespace ic +} // namespace owt diff --git a/talk/owt/sdk/ic/backgroundblur.h b/talk/owt/sdk/ic/backgroundblur.h new file mode 100644 index 000000000..7d0e688e5 --- /dev/null +++ b/talk/owt/sdk/ic/backgroundblur.h @@ -0,0 +1,36 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_IC_BACKGROUNDBLUR_H_ +#define OWT_IC_BACKGROUNDBLUR_H_ + +#include + +#include "talk/owt/sdk/ic/selfiesegmentationmodel.h" +#include "talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h" + +namespace owt { +namespace ic { + +class BackgroundBlur final : public owt::base::VideoFramePostProcessor { + public: + BackgroundBlur(InferenceEngine::Core& core); + ~BackgroundBlur() override = default; + + bool ReadModel(const std::string& modelXmlPath) override; + bool LoadModel(const std::string& device) override; + bool SetParameter(const std::string& key, int value) override; + + rtc::scoped_refptr Process( + const rtc::scoped_refptr& buffer) override; + + private: + SelfieSegmentationModel model_; + int blur_radius_ = 55; +}; + +} // namespace ic +} // namespace owt + +#endif // OWT_IC_BACKGROUNDBLUR_H_ diff --git a/talk/owt/sdk/ic/icmanager.cc b/talk/owt/sdk/ic/icmanager.cc new file mode 100644 index 000000000..2be20cd81 --- /dev/null +++ b/talk/owt/sdk/ic/icmanager.cc @@ -0,0 +1,59 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#include "talk/owt/sdk/ic/icmanager.h" + +#include "inference_engine.hpp" +#include "talk/owt/sdk/ic/backgroundblur.h" +#include "third_party/webrtc/rtc_base/logging.h" + +namespace owt { +namespace ic { + +bool ICManager::RegisterInferenceEnginePlugins( + const std::string& plugins_xml_path) { + if (!core_) { + try { + core_.reset(new InferenceEngine::Core(plugins_xml_path)); + } catch (const std::exception& e) { + RTC_LOG(LS_ERROR) << "Cannot initialize inference engine core: " + << e.what(); + return false; + } + } + try { + core_->RegisterPlugins(plugins_xml_path); + } catch (const std::exception& e) { + RTC_LOG(LS_ERROR) << "Cannot register inference engine plugins: " + << e.what(); + return false; + } + return true; +} + +std::shared_ptr +ICManager::CreatePostProcessor(ICPostProcessor processor) { + if (!core_ && !RegisterInferenceEnginePlugins({})) { + return nullptr; + } + switch (processor) { + case ICPostProcessor::BACKGROUND_BLUR: + return std::make_shared(*core_); + default: + RTC_LOG(LS_WARNING) << "CreatePostProcessor(" << processor + << ") is not implemented."; + } + return nullptr; +} + +} // namespace ic +} // namespace owt + +owt::ic::ICManagerInterface* CreateICManager() { + return new owt::ic::ICManager; +} + +void DestroyICManager(owt::ic::ICManagerInterface* ic_manager) { + delete ic_manager; +} diff --git a/talk/owt/sdk/ic/icmanager.h b/talk/owt/sdk/ic/icmanager.h new file mode 100644 index 000000000..b6ff879cb --- /dev/null +++ b/talk/owt/sdk/ic/icmanager.h @@ -0,0 +1,50 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_IC_ICMANAGER_H_ +#define OWT_IC_ICMANAGER_H_ + +#include "talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h" + +#include + +namespace InferenceEngine { +class Core; +} + +namespace owt { +namespace ic { + +class ICManager final : public ICManagerInterface { + public: + ICManager() = default; + + bool RegisterInferenceEnginePlugins( + const std::string& plugins_xml_path) override; + + std::shared_ptr CreatePostProcessor( + ICPostProcessor processor) override; + + protected: + std::shared_ptr core_; +}; + +} // namespace ic +} // namespace owt + +#ifdef WEBRTC_WIN +#ifdef OWT_IC_IMPL +#define OWT_IC_EXPORT extern "C" __declspec(dllexport) +#else +#define OWT_IC_EXPORT extern "C" __declspec(dllimport) +#endif +#else +#define OWT_IC_EXPORT extern "C" __attribute__((visibility("default"))) +#endif + +OWT_IC_EXPORT owt::ic::ICManagerInterface* CreateICManager(); + +OWT_IC_EXPORT void DestroyICManager(owt::ic::ICManagerInterface* ic_manager); + +#endif // OWT_IC_ICMANAGER_H_ diff --git a/talk/owt/sdk/ic/selfiesegmentationmodel.cc b/talk/owt/sdk/ic/selfiesegmentationmodel.cc new file mode 100644 index 000000000..37cbca42f --- /dev/null +++ b/talk/owt/sdk/ic/selfiesegmentationmodel.cc @@ -0,0 +1,55 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#include "selfiesegmentationmodel.h" + +namespace IE = InferenceEngine; + +namespace owt { +namespace ic { + +SelfieSegmentationModel::SelfieSegmentationModel(IE::Core& core) + : core_(core) {} + +void SelfieSegmentationModel::ReadModel(const std::string& modelXmlPath) { + network_ = core_.ReadNetwork(modelXmlPath); + auto input_info = network_.getInputsInfo(); + auto output_info = network_.getOutputsInfo(); + IE_ASSERT(input_info.size() == 1); + IE_ASSERT(output_info.size() == 1); + input_name_ = input_info.begin()->first; + output_name_ = output_info.begin()->first; + IE::DataPtr outputData = output_info.begin()->second; + outputData->setPrecision(IE::Precision::U8); + output_shape_ = outputData->getTensorDesc().getDims(); + IE::PreProcessInfo& preprocess = input_info[input_name_]->getPreProcess(); + preprocess.setResizeAlgorithm(IE::ResizeAlgorithm::RESIZE_BILINEAR); +} + +void SelfieSegmentationModel::LoadModel(const std::string& device) { + IE::ExecutableNetwork executableNetwork = core_.LoadNetwork(network_, device); + request_ = executableNetwork.CreateInferRequest(); +} + +bool SelfieSegmentationModel::IsLoaded() const { + return static_cast(request_); +} + +cv::Mat SelfieSegmentationModel::Predict(const cv::Mat& frame) { + IE_ASSERT(frame.type() == CV_32FC3); + IE::SizeVector inputShape = {1, 3, static_cast(frame.rows), + static_cast(frame.cols)}; + auto blob = IE::make_shared_blob( + IE::TensorDesc(IE::Precision::FP32, inputShape, IE::Layout::NHWC), + (float*)frame.data); + request_.SetBlob(input_name_, blob); + request_.Infer(); + const unsigned char* data = request_.GetBlob(output_name_)->buffer(); + size_t output_dims = output_shape_.size(); + return cv::Mat(output_shape_[output_dims - 2], output_shape_[output_dims - 1], + CV_8U, (void*)data); +} + +} // namespace ic +} // namespace owt diff --git a/talk/owt/sdk/ic/selfiesegmentationmodel.h b/talk/owt/sdk/ic/selfiesegmentationmodel.h new file mode 100644 index 000000000..3a0b9272c --- /dev/null +++ b/talk/owt/sdk/ic/selfiesegmentationmodel.h @@ -0,0 +1,39 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_IC_SELFIESEGMENTATIONMODEL_H_ +#define OWT_IC_SELFIESEGMENTATIONMODEL_H_ + +#include +#include + +#include "inference_engine.hpp" +#include "opencv2/core/mat.hpp" + +namespace owt { +namespace ic { + +class SelfieSegmentationModel { + public: + explicit SelfieSegmentationModel(InferenceEngine::Core& core); + + void ReadModel(const std::string& modelXmlPath); + void LoadModel(const std::string& device); + bool IsLoaded() const; + + cv::Mat Predict(const cv::Mat& frame); + + private: + InferenceEngine::Core &core_; + InferenceEngine::CNNNetwork network_; + InferenceEngine::InferRequest request_; + std::string input_name_; + std::string output_name_; + InferenceEngine::SizeVector output_shape_; +}; + +} // namespace ic +} // namespace owt + +#endif // OWT_IC_SELFIESEGMENTATIONMODEL_H_ diff --git a/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h b/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h index 1cb5847de..e86f25b98 100644 --- a/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h +++ b/talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h @@ -4,8 +4,11 @@ #ifndef OWT_BASE_LOCALCAMERASTREAMPARAMETERS_H_ #define OWT_BASE_LOCALCAMERASTREAMPARAMETERS_H_ +#include #include +#include #include "owt/base/commontypes.h" +#include "owt/base/videoframepostprocessor.h" namespace owt { namespace base{ /** @@ -51,6 +54,12 @@ class LocalCameraStreamParameters final { @param fps The frame rate of the video. */ void Fps(int fps); + /** + @brief Get the reference of video frame post processors, to be applied in + onFrame method. + */ + std::vector>& + PostProcessors(); /** @cond */ std::string CameraId() const { return camera_id_; } std::string StreamName() const { return stream_name_; } @@ -59,6 +68,10 @@ class LocalCameraStreamParameters final { int Fps() const { return fps_; } bool VideoEnabled() const { return video_enabled_; } bool AudioEnabled() const { return audio_enabled_; } + const std::vector>& + PostProcessors() const { + return post_processors_; + } /** @endcond */ private: std::string camera_id_; @@ -68,6 +81,8 @@ class LocalCameraStreamParameters final { int fps_; bool video_enabled_; bool audio_enabled_; + std::vector> + post_processors_; }; diff --git a/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h b/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h new file mode 100644 index 000000000..7761ca818 --- /dev/null +++ b/talk/owt/sdk/include/cpp/owt/base/pluginmanager.h @@ -0,0 +1,31 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_BASE_PLUGINMANAGER_H_ +#define OWT_BASE_PLUGINMANAGER_H_ + +#include "owt/ic/icmanagerinterface.h" + +namespace owt { +namespace base { + +/// Static class managing dll plugins. +class PluginManager { + public: +#if defined(WEBRTC_WIN) || defined(WEBRTC_LINUX) + /** + @brief Get the pointer of IC plugin. + @return The pointer to the instance of owt::ic::ICManagerInterface. If + loading owt_ic.dll failed, a nullptr will be returned. + */ + static owt::ic::ICManagerInterface* ICPlugin(); +#endif + + private: + PluginManager(); +}; +} // namespace base +} // namespace owt + +#endif // OWT_BASE_PLUGINMANAGER_H_ diff --git a/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h new file mode 100644 index 000000000..291b6625d --- /dev/null +++ b/talk/owt/sdk/include/cpp/owt/base/videoframepostprocessor.h @@ -0,0 +1,60 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_BASE_VIDEOFRAMEPOSTPROCESSOR_H_ +#define OWT_BASE_VIDEOFRAMEPOSTPROCESSOR_H_ + +#include + +namespace rtc { +template +class scoped_refptr; +} + +namespace webrtc { +class VideoFrameBuffer; +} + +namespace owt { +namespace base { + +/// Post processor that will be applied on the video frame. This is the abstract +/// base class of the post processors provided by OWT plugins. +class VideoFramePostProcessor { + public: + virtual ~VideoFramePostProcessor() = default; + + /** + @brief Read the model from the specific file path. + @param modelXmlPath The path to IR model description .xml file. + @return Whether the process succeeds. + */ + virtual bool ReadModel(const std::string& modelXmlPath) { return false; } + + /** + @brief Read the model to specific device. + @param device The device name. Must be "CPU" for now. + @return Whether the process succeeds. + */ + virtual bool LoadModel(const std::string& device) { return false; } + + /** + @brief Set the parameter of the processor. The content of key and value + should follow the definition of the sub-class, see documents for more + detail. + @return Whether the process succeeds. + */ + virtual bool SetParameter(const std::string& key, int value) { return false; } + + /** + @brief Process the VideoFrameBuffer. Implemented by OWT plugins. + */ + virtual rtc::scoped_refptr Process( + const rtc::scoped_refptr& buffer) = 0; +}; + +} // namespace base +} // namespace owt + +#endif // OWT_BASE_VIDEOFRAMEPOSTPROCESSOR_H_ diff --git a/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h b/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h new file mode 100644 index 000000000..501771c85 --- /dev/null +++ b/talk/owt/sdk/include/cpp/owt/ic/icmanagerinterface.h @@ -0,0 +1,46 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef OWT_IC_ICMANAGERINTERFACE_H_ +#define OWT_IC_ICMANAGERINTERFACE_H_ + +#include +#include + +#include "owt/base/videoframepostprocessor.h" + +namespace owt { +namespace ic { + +/// The type of IC post processor +enum class ICPostProcessor { BACKGROUND_BLUR }; + +/// The IC plugin manager, which creates the post processor instance. +class ICManagerInterface { + public: + virtual ~ICManagerInterface() = default; + + /** + @brief Register inference engine plugins. + @param plugins_xml_path The path to plugins.xml of OpenVINO Inference + Engine. + @return Whether the process succeeds. In case of failure, the error message + will be printed to RTC log. + */ + virtual bool RegisterInferenceEnginePlugins( + const std::string& plugins_xml_path) = 0; + + /** + @brief Create and get the IC post processor. + @param processor The type of processor to be created. + @return The shared pointer of the post processor. + */ + virtual std::shared_ptr + CreatePostProcessor(ICPostProcessor processor) = 0; +}; + +} // namespace ic +} // namespace owt + +#endif // OWT_IC_ICMANAGERINTERFACE_H_ diff --git a/talk/owt/sdk/sample/BUILD.gn b/talk/owt/sdk/sample/BUILD.gn new file mode 100644 index 000000000..bbf558718 --- /dev/null +++ b/talk/owt/sdk/sample/BUILD.gn @@ -0,0 +1,14 @@ +# Copyright (C) <2021> Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +import("//build_overrides/owt.gni") +import("//build_overrides/webrtc.gni") +import("//testing/test.gni") + +group("sample") { + deps = [] + if (is_win) { + deps += [ "win:sample_win" ] + } +} diff --git a/talk/owt/sdk/sample/win/BUILD.gn b/talk/owt/sdk/sample/win/BUILD.gn new file mode 100644 index 000000000..de1e99af6 --- /dev/null +++ b/talk/owt/sdk/sample/win/BUILD.gn @@ -0,0 +1,11 @@ +# Copyright (C) <2021> Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +import("//build_overrides/owt.gni") +import("//build_overrides/webrtc.gni") +import("//testing/test.gni") + +group("sample_win") { + deps = [ "sample_background_blur" ] +} diff --git a/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn b/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn new file mode 100644 index 000000000..52ad233b6 --- /dev/null +++ b/talk/owt/sdk/sample/win/sample_background_blur/BUILD.gn @@ -0,0 +1,14 @@ +# Copyright (C) <2021> Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +import("//build_overrides/owt.gni") +import("//build_overrides/webrtc.gni") +import("//testing/test.gni") + +executable("sample_background_blur") { + sources = [ "sample_background_blur.cc" ] + include_dirs = [ "//talk/owt/sdk/include/cpp" ] + deps = [ "//talk/owt:owt_sdk_base" ] + data_deps = [ "//talk/owt/sdk/ic:owt_ic" ] +} diff --git a/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc b/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc new file mode 100644 index 000000000..f6721b95a --- /dev/null +++ b/talk/owt/sdk/sample/win/sample_background_blur/sample_background_blur.cc @@ -0,0 +1,121 @@ +// Copyright (C) <2021> Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include "owt/base/deviceutils.h" +#include "owt/base/logging.h" +#include "owt/base/pluginmanager.h" +#include "owt/base/stream.h" + +class MainWindow : public owt::base::VideoRenderWindow { + static constexpr PCTSTR class_name = TEXT("MainWindow"); + + static LRESULT WindowProcedure(HWND window, + unsigned int msg, + WPARAM wp, + LPARAM lp) { + switch (msg) { + case WM_DESTROY: + PostQuitMessage(0); + return 0; + default: + return DefWindowProc(window, msg, wp, lp); + } + } + + public: + MainWindow(int width, int height, PCTSTR title) { + WNDCLASS cls = {}; + cls.lpfnWndProc = WindowProcedure; + cls.hInstance = GetModuleHandle(nullptr); + cls.lpszClassName = class_name; + RegisterClass(&cls); + HWND handle = CreateWindow(class_name, title, WS_OVERLAPPEDWINDOW, + CW_USEDEFAULT, CW_USEDEFAULT, width, height, + nullptr, nullptr, cls.hInstance, nullptr); + SetWindowHandle(handle); + } + + void Show() { ShowWindow(GetWindowHandle(), SW_SHOW); } + + int Exec() { + MSG msg; + while (GetMessage(&msg, 0, 0, 0)) { + DispatchMessage(&msg); + } + return 0; + } +}; + +int main(int argc, char* argv[]) { + if (argc != 2) { + std::cout << "Usage: sample_background_blur " << std::endl; + return 0; + } + std::string model_path = argv[1]; + + owt::base::Logging::Severity(owt::base::LoggingSeverity::kWarning); + + int id = 0; + std::string device_name = owt::base::DeviceUtils::GetDeviceNameByIndex(id); + std::string camera_id = owt::base::DeviceUtils::VideoCapturerIds()[id]; + owt::base::VideoTrackCapabilities capability = + owt::base::DeviceUtils::VideoCapturerSupportedCapabilities(camera_id)[id]; + + owt::base::LocalCameraStreamParameters param(false, true); + param.CameraId(camera_id); + param.Resolution(capability.width, capability.height); + param.Fps(capability.frameRate); + + // Load the owt_ic.dll and initialize ICManager + owt::ic::ICManagerInterface* ic_plugin = owt::base::PluginManager::ICPlugin(); + if (!ic_plugin) { + std::cerr << "Unable to initialize IC plugin." << std::endl; + return 1; + } + + // Initialize global inference engine core, to prevent slow first time + // initialization of background blur post processor + if (!ic_plugin->RegisterInferenceEnginePlugins("plugins.xml")) { + std::cerr << "Unable to register inference engine plugins" << std::endl; + return 2; + } + + std::shared_ptr background_blur = + ic_plugin->CreatePostProcessor(owt::ic::ICPostProcessor::BACKGROUND_BLUR); + if (!background_blur) { + std::cerr << "Create background blur failed." << std::endl; + return 3; + } + if (!background_blur->ReadModel(model_path)) { + std::cerr << "Failed to read the model." << std::endl; + return 4; + } + if (!background_blur->LoadModel("CPU")) { + std::cerr << "Failed to load the model." << std::endl; + return 5; + } + if (!background_blur->SetParameter("blur_radius", 55)) { + std::cerr << "Failed to set blur radius." << std::endl; + return 6; + } + param.PostProcessors().push_back(background_blur); + + int error = 0; + std::shared_ptr stream = + owt::base::LocalStream::Create(param, error); + if (error) { + std::cerr << "Create local stream failed, error code " << error << "." + << std::endl; + return 7; + } + + std::basic_string title(device_name.begin(), device_name.end()); + MainWindow w(capability.width, capability.height, title.c_str()); + stream->AttachVideoRenderer(w); + w.Show(); + return w.Exec(); +}