Skip to content
This repository was archived by the owner on Oct 25, 2024. It is now read-only.

Enable IC background blur feature #564

Open
wants to merge 30 commits into
base: main
Choose a base branch
from
Open

Enable IC background blur feature #564

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
dea547b
Enable IC background blur feature
zhibo1-wang Oct 20, 2021
6b305e9
Fixups
zhibo1-wang Oct 21, 2021
07e161c
Update thirdpartylicense.txt
zhibo1-wang Oct 21, 2021
091f187
Fixups
zhibo1-wang Oct 21, 2021
ec24bbf
Fixups
zhibo1-wang Oct 25, 2021
09530e6
Fixups
zhibo1-wang Oct 25, 2021
96ab951
Fixups
zhibo1-wang Oct 25, 2021
572b0f4
Fixups
zhibo1-wang Oct 28, 2021
65ee09d
Fixups
zhibo1-wang Oct 28, 2021
8e12bed
Fixups
zhibo1-wang Oct 28, 2021
51311bc
Fixups
zhibo1-wang Oct 29, 2021
09d1d4d
Fixups
zhibo1-wang Oct 29, 2021
59891e2
Fixups
zhibo1-wang Oct 29, 2021
9c7a7ce
Fixups
zhibo1-wang Oct 29, 2021
6ef5289
Fix FreeLibrary deadlocks
zhibo1-wang Nov 2, 2021
f9c7381
Fixups
zhibo1-wang Nov 3, 2021
a474a59
Naming fixups and etc.
zhibo1-wang Nov 4, 2021
13d8027
Fixups
zhibo1-wang Nov 5, 2021
4db6116
Update README.md
zhibo1-wang Nov 8, 2021
2f91956
Fixups (bugs)
zhibo1-wang Nov 11, 2021
e8c02e5
Update docs
zhibo1-wang Nov 17, 2021
7980fcf
Fix debug build
zhibo1-wang Nov 19, 2021
705860c
Fixups
zhibo1-wang Nov 22, 2021
ab80773
Update windows.jenkinsfile
zhibo1-wang Nov 24, 2021
6287a53
Fixups
zhibo1-wang Nov 25, 2021
f23c70c
Fixup
zhibo1-wang Nov 25, 2021
3e5dd15
Update windows.jenkinsfile
zhibo1-wang Nov 26, 2021
16de4f9
Update windows.jenkinsfile
zhibo1-wang Nov 29, 2021
d00bb7e
Update windows.jenkinsfile
zhibo1-wang Dec 13, 2021
32a61d7
Update windows.jenkinsfile
zhibo1-wang Dec 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion talk/owt/sdk/base/intelligentcollaborationparameters.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
#include "talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h"

namespace owt {
namespace ic {
namespace base {

std::vector<std::shared_ptr<owt::base::VideoFramePostProcessor>>&
IntelligentCollaborationParameters::PostProcessors() {
Expand Down
4 changes: 2 additions & 2 deletions talk/owt/sdk/base/localcamerastreamparameters.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@ LocalCameraStreamParameters::LocalCameraStreamParameters(bool audio_enabled,
void LocalCameraStreamParameters::Fps(int fps) {
fps_ = fps;
}
owt::ic::IntelligentCollaborationParameters&
IntelligentCollaborationParameters&
LocalCameraStreamParameters::ICParams() {
return ic_params_;
}
const owt::ic::IntelligentCollaborationParameters&
const IntelligentCollaborationParameters&
LocalCameraStreamParameters::ICParams() const {
return ic_params_;
}
Expand Down
4 changes: 2 additions & 2 deletions talk/owt/sdk/base/stream.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ class CapturerTrackSource : public webrtc::VideoTrackSource {
const size_t height,
const size_t fps,
int capture_device_idx,
const owt::ic::IntelligentCollaborationParameters& ic_params =
owt::ic::IntelligentCollaborationParameters()) {
const IntelligentCollaborationParameters& ic_params =
IntelligentCollaborationParameters()) {
std::unique_ptr<owt::base::VcmCapturer> capturer;
std::unique_ptr<webrtc::VideoCaptureModule::DeviceInfo> info(
webrtc::VideoCaptureFactory::CreateDeviceInfo());
Expand Down
48 changes: 24 additions & 24 deletions talk/owt/sdk/ic/backgroundblur.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,12 @@
namespace owt {
namespace ic {

BackgroundBlur::BackgroundBlur(InferenceEngine::Core& core) : model(core) {}
BackgroundBlur::BackgroundBlur(InferenceEngine::Core& core) : model_(core) {}

bool BackgroundBlur::SetParameter(const std::string& key,
const std::string& value) {
if (key == "model_path") {
return model.LoadModel(value);
return model_.LoadModel(value);
} else if (key == "blur_radius") {
blur_radius_ = stoi(value);
return true;
Expand All @@ -32,8 +32,8 @@ bool BackgroundBlur::SetParameter(const std::string& key,

rtc::scoped_refptr<webrtc::VideoFrameBuffer> BackgroundBlur::Process(
const rtc::scoped_refptr<webrtc::VideoFrameBuffer>& buffer) {
if (!model.IsLoaded()) {
RTC_LOG(LS_WARNING) << "Background blur model is not initialized.";
if (!model_.IsLoaded()) {
RTC_LOG(LS_WARNING) << "Background blur model_ is not initialized.";
return buffer;
}

Expand All @@ -46,39 +46,39 @@ rtc::scoped_refptr<webrtc::VideoFrameBuffer> BackgroundBlur::Process(

cv::Mat input;
frame.convertTo(input, CV_32FC3, 1. / UCHAR_MAX);
cv::Mat mask = model.predict(input); // mask is of 8UC1
cv::Mat mask = model_.predict(input); // mask is of 8UC1

cv::resize(mask, mask, {buffer->width(), buffer->height()});
cv::Mat foregroundMask;
cv::merge(std::vector<cv::Mat>{mask, mask, mask}, foregroundMask);
cv::Mat backgroundMask;
cv::Mat foreground_mask;
cv::merge(std::vector<cv::Mat>{mask, mask, mask}, foreground_mask);
cv::Mat background_mask;
cv::merge(std::vector<cv::Mat>{UCHAR_MAX - mask, UCHAR_MAX - mask,
UCHAR_MAX - mask},
backgroundMask);
background_mask);

// apply a masked blur on background
cv::Mat maskedFrame;
cv::multiply(frame, backgroundMask, maskedFrame, 1. / UCHAR_MAX);
// Apply a masked blur on background
cv::Mat masked_frame;
cv::multiply(frame, background_mask, masked_frame, 1. / UCHAR_MAX);
cv::Mat background;
cv::GaussianBlur(maskedFrame, background, {blur_radius_, blur_radius_}, 0);
cv::Mat blurredMask;
cv::GaussianBlur(backgroundMask, blurredMask, {blur_radius_, blur_radius_},
cv::GaussianBlur(masked_frame, background, {blur_radius_, blur_radius_}, 0);
cv::Mat blurred_mask;
cv::GaussianBlur(background_mask, blurred_mask, {blur_radius_, blur_radius_},
0);
cv::divide(background, blurredMask, background, UCHAR_MAX);
cv::multiply(background, backgroundMask, background, 1. / UCHAR_MAX);
cv::divide(background, blurred_mask, background, UCHAR_MAX);
cv::multiply(background, background_mask, background, 1. / UCHAR_MAX);

cv::Mat foreground;
cv::multiply(frame, foregroundMask, foreground, 1. / UCHAR_MAX);
cv::multiply(frame, foreground_mask, foreground, 1. / UCHAR_MAX);
frame = foreground + background;

auto newBuffer =
auto new_buffer =
webrtc::I420Buffer::Create(buffer->width(), buffer->height());
libyuv::RAWToI420(frame.data, frame.cols * 3, newBuffer->MutableDataY(),
newBuffer->StrideY(), newBuffer->MutableDataU(),
newBuffer->StrideU(), newBuffer->MutableDataV(),
newBuffer->StrideV(), buffer->width(), buffer->height());
libyuv::RAWToI420(frame.data, frame.cols * 3, new_buffer->MutableDataY(),
new_buffer->StrideY(), new_buffer->MutableDataU(),
new_buffer->StrideU(), new_buffer->MutableDataV(),
new_buffer->StrideV(), buffer->width(), buffer->height());

return newBuffer;
return new_buffer;
}

} // namespace ic
Expand Down
2 changes: 1 addition & 1 deletion talk/owt/sdk/ic/backgroundblur.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ class BackgroundBlur : public owt::base::VideoFramePostProcessor {
const rtc::scoped_refptr<webrtc::VideoFrameBuffer>& buffer) override;

private:
SelfieSegmentationModel model;
SelfieSegmentationModel model_;
int blur_radius_ = 55;
};

Expand Down
6 changes: 3 additions & 3 deletions talk/owt/sdk/ic/icmanager.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@

namespace owt {
namespace ic {
ICManager::ICManager() : core(new InferenceEngine::Core()) {
ICManager::ICManager() : core_(new InferenceEngine::Core()) {
}

bool ICManager::InitializeInferenceEngineCore(
const std::string& plugins_xml_path) {
try {
core->RegisterPlugins(plugins_xml_path);
core_->RegisterPlugins(plugins_xml_path);
return true;
} catch (std::exception &) {
return false;
Expand All @@ -26,7 +26,7 @@ std::shared_ptr<owt::base::VideoFramePostProcessor>
ICManager::CreatePostProcessor(ICPlugin plugin) {
switch (plugin) {
case ICPlugin::BACKGROUND_BLUR:
return std::make_shared<BackgroundBlur>(*core);
return std::make_shared<BackgroundBlur>(*core_);
default:;
}
return nullptr;
Expand Down
2 changes: 1 addition & 1 deletion talk/owt/sdk/ic/icmanager.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ class ICManager : public ICManagerInterface {
ICPlugin plugin) override;

protected:
std::shared_ptr<InferenceEngine::Core> core;
std::shared_ptr<InferenceEngine::Core> core_;
};

} // namespace ic
Expand Down
32 changes: 16 additions & 16 deletions talk/owt/sdk/ic/selfiesegmentationmodel.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,32 +15,32 @@ namespace IE = InferenceEngine;
namespace owt {
namespace ic {
SelfieSegmentationModel::SelfieSegmentationModel(InferenceEngine::Core& core)
: core(core) {}
: core_(core) {}

bool SelfieSegmentationModel::LoadModel(const std::string& xmlPath,
const std::string& device) {
InferenceEngine::CNNNetwork network = core.ReadNetwork(xmlPath);
InferenceEngine::CNNNetwork network = core_.ReadNetwork(xmlPath);
if (network.getInputsInfo().empty() || network.getOutputsInfo().empty()) {
RTC_LOG(LS_WARNING) << "Bad segmentation model: no input / output";
RTC_LOG(LS_WARNING) << "Bad segmentation model_: no input / output";
return false;
}
inputName = network.getInputsInfo().begin()->first;
outputName = network.getOutputsInfo().begin()->first;
input_name_ = network.getInputsInfo().begin()->first;
output_name_ = network.getOutputsInfo().begin()->first;
IE::DataPtr outputData = network.getOutputsInfo().begin()->second;
outputData->setPrecision(IE::Precision::U8);
outputSize = outputData->getTensorDesc().getDims();
output_size_ = outputData->getTensorDesc().getDims();

IE::PreProcessInfo& preprocess =
network.getInputsInfo()[inputName]->getPreProcess();
network.getInputsInfo()[input_name_]->getPreProcess();
preprocess.setResizeAlgorithm(IE::ResizeAlgorithm::RESIZE_BILINEAR);

InferenceEngine::ExecutableNetwork executableNetwork =
core.LoadNetwork(network, device);
request = executableNetwork.CreateInferRequest();
core_.LoadNetwork(network, device);
request_ = executableNetwork.CreateInferRequest();
}

bool SelfieSegmentationModel::IsLoaded() const {
return static_cast<bool>(request);
return static_cast<bool>(request_);
}

cv::Mat SelfieSegmentationModel::predict(const cv::Mat& frame) {
Expand All @@ -56,15 +56,15 @@ void SelfieSegmentationModel::predictAsync(const cv::Mat& frame) {
IE::TensorDesc(IE::Precision::FP32, {1, 3, height, width},
IE::Layout::NHWC),
(float*)frame.data);
request.SetBlob(inputName, blob);
request.StartAsync();
request_.SetBlob(input_name_, blob);
request_.StartAsync();
}

cv::Mat SelfieSegmentationModel::waitForFinished() {
request.Wait(IE::InferRequest::WaitMode::RESULT_READY);
const unsigned char* data = request.GetBlob(outputName)->buffer();
int height = outputSize[1];
int width = outputSize[2];
request_.Wait(IE::InferRequest::WaitMode::RESULT_READY);
const unsigned char* data = request_.GetBlob(output_name_)->buffer();
int height = output_size_[1];
int width = output_size_[2];
return cv::Mat(height, width, CV_8U, (void*)data);
}

Expand Down
10 changes: 5 additions & 5 deletions talk/owt/sdk/ic/selfiesegmentationmodel.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ class SelfieSegmentationModel {
cv::Mat waitForFinished();
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what's the point of public this?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sync/Async prediction are options provided by openvino. I just pass on these to provides potential caller with free usage.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't agree. If you want to provide async API, your API should include a callback param instead of asking developer to call another API just to wait for the result.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The waitForFinished function behave like await Promise, I believe this is another async call design except for callbacks. Anyway, we may remove the async api since the postprocessors use a sequential processing.


private:
InferenceEngine::Core &core;
InferenceEngine::InferRequest request;
std::string inputName;
std::string outputName;
InferenceEngine::SizeVector outputSize;
InferenceEngine::Core &core_;
InferenceEngine::InferRequest request_;
std::string input_name_;
std::string output_name_;
InferenceEngine::SizeVector output_size_;
};

} // namespace ic
Expand Down
6 changes: 3 additions & 3 deletions talk/owt/sdk/include/cpp/owt/base/localcamerastreamparameters.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ class LocalCameraStreamParameters final {
/**
@brief Get the reference of intelligent collaboration parameters
*/
owt::ic::IntelligentCollaborationParameters& ICParams();
const owt::ic::IntelligentCollaborationParameters& ICParams() const;
IntelligentCollaborationParameters& ICParams();
const IntelligentCollaborationParameters& ICParams() const;
/** @cond */
std::string CameraId() const { return camera_id_; }
std::string StreamName() const { return stream_name_; }
Expand All @@ -74,7 +74,7 @@ class LocalCameraStreamParameters final {
int fps_;
bool video_enabled_;
bool audio_enabled_;
owt::ic::IntelligentCollaborationParameters ic_params_;
IntelligentCollaborationParameters ic_params_;
};


Expand Down
2 changes: 1 addition & 1 deletion talk/owt/sdk/include/cpp/owt/ic/intelligentcollaborationparameters.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
#include "owt/base/videoframepostprocessor.h"

namespace owt {
namespace ic {
namespace base {

class IntelligentCollaborationParameters final {
public:
Expand Down