Skip to content

Commit 54f2812

Browse files
author
Linnea May
committed
small changes to styeleffect
1 parent 60c4fcb commit 54f2812

File tree

7 files changed

+243
-212
lines changed

7 files changed

+243
-212
lines changed

‎Samples/StyleTransfer/VideoEffect/StyleTransferEffectCpp/StyleTransferEffect.cpp‎

Lines changed: 163 additions & 136 deletions
Original file line numberDiff line numberDiff line change
@@ -11,140 +11,167 @@ using namespace concurrency;
1111

1212
namespace winrt::StyleTransferEffectCpp::implementation
1313
{
14-
StyleTransferEffect::StyleTransferEffect() :
15-
Session(nullptr)
16-
{
17-
for (int i = 0; i < swapChainEntryCount; i++)
18-
{
19-
bindings.push_back(std::make_unique<SwapChainEntry>());
20-
}
21-
}
22-
23-
IVectorView<VideoEncodingProperties> StyleTransferEffect::SupportedEncodingProperties() {
24-
VideoEncodingProperties props = VideoEncodingProperties();
25-
props.Subtype(L"ARGB32");
26-
return single_threaded_vector(std::vector<VideoEncodingProperties>{props}).GetView();
27-
}
28-
29-
bool StyleTransferEffect::TimeIndependent() { return true; }
30-
MediaMemoryTypes StyleTransferEffect::SupportedMemoryTypes() { return MediaMemoryTypes::GpuAndCpu; }
31-
bool StyleTransferEffect::IsReadOnly() { return false; }
32-
void StyleTransferEffect::DiscardQueuedFrames() {}
33-
34-
void StyleTransferEffect::Close(MediaEffectClosedReason) {
35-
OutputDebugString(L"Close Begin | ");
36-
std::lock_guard<mutex> guard{ Processing };
37-
// Make sure evalAsyncs are done before clearing resources
38-
for (int i = 0; i < swapChainEntryCount; i++) {
39-
if (bindings[i]->activetask != nullptr &&
40-
bindings[i]->binding != nullptr)
41-
{
42-
std::wostringstream ss;
43-
ss << i;
44-
OutputDebugString(ss.str().c_str());
45-
bindings[i]->activetask.get();
46-
bindings[i]->binding.Clear();
47-
}
48-
}
49-
if (Session != nullptr) Session.Close();
50-
OutputDebugString(L"Close\n");
51-
}
52-
53-
void StyleTransferEffect::SubmitEval(VideoFrame input, VideoFrame output) {
54-
auto currentBinding = bindings[0].get();
55-
if (currentBinding->activetask == nullptr
56-
|| currentBinding->activetask.Status() != Windows::Foundation::AsyncStatus::Started)
57-
{
58-
auto now = std::chrono::high_resolution_clock::now();
59-
OutputDebugString(L"PF Start new Eval ");
60-
std::wostringstream ss;
61-
ss << swapChainIndex;
62-
OutputDebugString(ss.str().c_str());
63-
OutputDebugString(L" | ");
64-
65-
currentBinding->binding.Bind(InputImageDescription, input);
66-
// submit an eval and wait for it to finish submitting work
67-
{
68-
std::lock_guard<mutex> guard{ Processing };
69-
std::rotate(bindings.begin(), bindings.begin() + 1, bindings.end());
70-
finishedIdx = (finishedIdx - 1 + swapChainEntryCount) % swapChainEntryCount;
71-
currentBinding->activetask = Session.EvaluateAsync(currentBinding->binding, ss.str().c_str());
72-
}
73-
currentBinding->activetask.Completed([&, currentBinding, now](auto&& asyncInfo, winrt::Windows::Foundation::AsyncStatus const) {
74-
OutputDebugString(L"PF Eval completed |");
75-
VideoFrame evalOutput = asyncInfo.GetResults().Outputs().Lookup(OutputImageDescription).try_as<VideoFrame>();
76-
int bindingIdx;
77-
bool finishedFrameUpdated;
78-
{
79-
std::lock_guard<mutex> guard{ Processing };
80-
auto binding = std::find_if(bindings.begin(), bindings.end(), [currentBinding](const auto& b) {return b.get() == currentBinding; });
81-
bindingIdx = std::distance(bindings.begin(), binding);
82-
finishedFrameUpdated = bindingIdx >= finishedIdx;
83-
finishedIdx = finishedFrameUpdated ? bindingIdx : finishedIdx;
84-
85-
}
86-
if (finishedFrameUpdated)
87-
{
88-
OutputDebugString(L"PF Copy | ");
89-
evalOutput.CopyToAsync(currentBinding->outputCache);
90-
}
91-
92-
auto timePassed = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - now);
93-
Notifier.SetFrameRate(1000.f / timePassed.count()); // Convert to FPS: milli to seconds, invert
94-
OutputDebugString(L"PF End ");
95-
});
96-
}
97-
if (bindings[finishedIdx]->outputCache != nullptr) {
98-
std::wostringstream ss;
99-
ss << finishedIdx;
100-
OutputDebugString(L"\nStart CopyAsync ");
101-
OutputDebugString(ss.str().c_str());
102-
bindings[finishedIdx]->outputCache.CopyToAsync(output).get();
103-
OutputDebugString(L" | Stop CopyAsync\n");
104-
}
105-
// return without waiting for the submit to finish, setup the completion handler
106-
}
107-
108-
void StyleTransferEffect::ProcessFrame(ProcessVideoFrameContext context) {
109-
OutputDebugString(L"PF Start | ");
110-
VideoFrame inputFrame = context.InputFrame();
111-
VideoFrame outputFrame = context.OutputFrame();
112-
113-
SubmitEval(inputFrame, outputFrame);
114-
115-
swapChainIndex = (++swapChainIndex) % swapChainEntryCount; // move on to the next entry after each call to PF.
116-
}
117-
118-
void StyleTransferEffect::SetEncodingProperties(VideoEncodingProperties props, IDirect3DDevice device) {
119-
encodingProperties = props;
120-
}
121-
122-
void StyleTransferEffect::SetProperties(IPropertySet config) {
123-
this->configuration = config;
124-
hstring modelName;
125-
bool useGpu;
126-
127-
IInspectable val = config.TryLookup(L"ModelName");
128-
if (val) modelName = unbox_value<hstring>(val);
129-
else winrt::throw_hresult(E_FAIL);
130-
val = configuration.TryLookup(L"UseGpu");
131-
if (val) useGpu = unbox_value<bool>(val);
132-
else winrt::throw_hresult(E_FAIL);
133-
val = configuration.TryLookup(L"Notifier");
134-
if (val) Notifier = val.try_as<StyleTransferEffectNotifier>();
135-
else winrt::throw_hresult(E_FAIL);
136-
137-
LearningModel m_model = LearningModel::LoadFromFilePath(modelName);
138-
LearningModelDeviceKind m_device = useGpu ? LearningModelDeviceKind::DirectX : LearningModelDeviceKind::Cpu;
139-
Session = LearningModelSession{ m_model, LearningModelDevice(m_device) };
140-
141-
InputImageDescription = L"inputImage";
142-
OutputImageDescription = L"outputImage";
143-
144-
// Create set of bindings to cycle through
145-
for (int i = 0; i < swapChainEntryCount; i++) {
146-
bindings[i]->binding = LearningModelBinding(Session);
147-
bindings[i]->binding.Bind(OutputImageDescription, VideoFrame(Windows::Graphics::Imaging::BitmapPixelFormat::Bgra8, 720, 720));
148-
}
149-
}
14+
StyleTransferEffect::StyleTransferEffect() :
15+
Session(nullptr)
16+
{
17+
for (int i = 0; i < swapChainEntryCount; i++)
18+
{
19+
bindings.push_back(std::make_unique<SwapChainEntry>());
20+
}
21+
}
22+
23+
IVectorView<VideoEncodingProperties> StyleTransferEffect::SupportedEncodingProperties() {
24+
VideoEncodingProperties props = VideoEncodingProperties();
25+
props.Subtype(L"ARGB32");
26+
return single_threaded_vector(std::vector<VideoEncodingProperties>{props}).GetView();
27+
}
28+
29+
bool StyleTransferEffect::TimeIndependent() { return true; }
30+
MediaMemoryTypes StyleTransferEffect::SupportedMemoryTypes() { return MediaMemoryTypes::GpuAndCpu; }
31+
bool StyleTransferEffect::IsReadOnly() { return false; }
32+
void StyleTransferEffect::DiscardQueuedFrames() {}
33+
34+
void StyleTransferEffect::Close(MediaEffectClosedReason) {
35+
OutputDebugString(L"Close Begin | ");
36+
std::lock_guard<mutex> guard{ Processing };
37+
// Make sure evalAsyncs are done before clearing resources
38+
for (int i = 0; i < swapChainEntryCount; i++) {
39+
if (bindings[i]->activetask != nullptr &&
40+
bindings[i]->binding != nullptr)
41+
{
42+
OutputDebugString(std::to_wstring(i).c_str());
43+
bindings[i]->activetask.get();
44+
bindings[i]->binding.Clear();
45+
}
46+
}
47+
if (Session != nullptr) Session.Close();
48+
OutputDebugString(L"Close\n");
49+
}
50+
51+
void StyleTransferEffect::SubmitEval(VideoFrame input, VideoFrame output) {
52+
auto currentBinding = bindings[0].get();
53+
if (currentBinding->activetask == nullptr
54+
|| currentBinding->activetask.Status() != Windows::Foundation::AsyncStatus::Started)
55+
{
56+
auto now = std::chrono::high_resolution_clock::now();
57+
OutputDebugString(L"PF Start new Eval ");
58+
OutputDebugString(std::to_wstring(swapChainIndex).c_str());
59+
OutputDebugString(L" | ");
60+
currentBinding->binding.Bind(InputImageDescription, input);
61+
// submit an eval and wait for it to finish submitting work
62+
{
63+
std::lock_guard<mutex> guard{ Processing };
64+
std::rotate(bindings.begin(), bindings.begin() + 1, bindings.end());
65+
finishedFrameIndex = (finishedFrameIndex - 1 + swapChainEntryCount) % swapChainEntryCount;
66+
currentBinding->activetask = Session.EvaluateAsync(
67+
currentBinding->binding,
68+
std::to_wstring(swapChainIndex).c_str());
69+
}
70+
currentBinding->activetask.Completed([&, currentBinding, now](auto&& asyncInfo, winrt::Windows::Foundation::AsyncStatus const) {
71+
OutputDebugString(L"PF Eval completed |");
72+
VideoFrame evalOutput = asyncInfo.GetResults()
73+
.Outputs()
74+
.Lookup(OutputImageDescription)
75+
.try_as<VideoFrame>();
76+
int bindingIdx;
77+
bool finishedFrameUpdated;
78+
{
79+
std::lock_guard<mutex> guard{ Processing };
80+
auto binding = std::find_if(bindings.begin(),
81+
bindings.end(),
82+
[currentBinding](const auto& b)
83+
{
84+
return b.get() == currentBinding;
85+
});
86+
bindingIdx = std::distance(bindings.begin(), binding);
87+
finishedFrameUpdated = bindingIdx >= finishedFrameIndex;
88+
finishedFrameIndex = finishedFrameUpdated ? bindingIdx : finishedFrameIndex;
89+
}
90+
if (finishedFrameUpdated)
91+
{
92+
OutputDebugString(L"PF Copy | ");
93+
evalOutput.CopyToAsync(currentBinding->outputCache);
94+
}
95+
96+
auto timePassed = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - now);
97+
// Convert to FPS: milli to seconds, invert
98+
Notifier.SetFrameRate(1000.f / timePassed.count());
99+
OutputDebugString(L"PF End ");
100+
});
101+
}
102+
if (bindings[finishedFrameIndex]->outputCache != nullptr) {
103+
OutputDebugString(L"\nStart CopyAsync ");
104+
OutputDebugString(std::to_wstring(finishedFrameIndex).c_str());
105+
bindings[finishedFrameIndex]->outputCache.CopyToAsync(output).get();
106+
OutputDebugString(L" | Stop CopyAsync\n");
107+
}
108+
// return without waiting for the submit to finish, setup the completion handler
109+
}
110+
111+
void StyleTransferEffect::ProcessFrame(ProcessVideoFrameContext context) {
112+
OutputDebugString(L"PF Start | ");
113+
VideoFrame inputFrame = context.InputFrame();
114+
VideoFrame outputFrame = context.OutputFrame();
115+
116+
SubmitEval(inputFrame, outputFrame);
117+
118+
swapChainIndex = (++swapChainIndex) % swapChainEntryCount;
119+
// move on to the next entry after each call to PF.
120+
}
121+
122+
void StyleTransferEffect::SetEncodingProperties(
123+
VideoEncodingProperties props,
124+
IDirect3DDevice device) {
125+
encodingProperties = props;
126+
}
127+
128+
void StyleTransferEffect::SetProperties(IPropertySet config) {
129+
this->configuration = config;
130+
hstring modelName;
131+
bool useGpu;
132+
133+
IInspectable val = config.TryLookup(L"ModelName");
134+
if (val)
135+
{
136+
modelName = unbox_value<hstring>(val);
137+
}
138+
else
139+
{
140+
winrt::throw_hresult(E_FAIL);
141+
}
142+
143+
val = configuration.TryLookup(L"UseGpu");
144+
if (val)
145+
{
146+
useGpu = unbox_value<bool>(val);
147+
}
148+
else
149+
{
150+
winrt::throw_hresult(E_FAIL);
151+
}
152+
153+
val = configuration.TryLookup(L"Notifier");
154+
if (val)
155+
{
156+
Notifier = val.try_as<StyleTransferEffectNotifier>();
157+
}
158+
else
159+
{
160+
winrt::throw_hresult(E_FAIL);
161+
}
162+
LearningModel _model = LearningModel::LoadFromFilePath(modelName);
163+
LearningModelDeviceKind _device = useGpu ? LearningModelDeviceKind::DirectX
164+
: LearningModelDeviceKind::Cpu;
165+
Session = LearningModelSession{ _model, LearningModelDevice(_device) };
166+
167+
InputImageDescription = L"inputImage";
168+
OutputImageDescription = L"outputImage";
169+
170+
// Create set of bindings to cycle through
171+
for (int i = 0; i < swapChainEntryCount; i++) {
172+
bindings[i]->binding = LearningModelBinding(Session);
173+
bindings[i]->binding.Bind(OutputImageDescription,
174+
VideoFrame(Windows::Graphics::Imaging::BitmapPixelFormat::Bgra8, 720, 720));
175+
}
176+
}
150177
}

‎Samples/StyleTransfer/VideoEffect/StyleTransferEffectCpp/StyleTransferEffect.h‎

Lines changed: 39 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -11,50 +11,50 @@ using namespace winrt::Windows::Media;
1111

1212
namespace winrt::StyleTransferEffectCpp::implementation
1313
{
14-
struct SwapChainEntry {
15-
LearningModelBinding binding;
16-
Windows::Foundation::IAsyncOperation<LearningModelEvaluationResult> activetask;
17-
VideoFrame outputCache;
18-
SwapChainEntry() :
19-
binding(nullptr),
20-
activetask(nullptr),
21-
outputCache(VideoFrame(Windows::Graphics::Imaging::BitmapPixelFormat::Bgra8, 720, 720)) {}
22-
};
14+
struct SwapChainEntry {
15+
LearningModelBinding binding;
16+
Windows::Foundation::IAsyncOperation<LearningModelEvaluationResult> activetask;
17+
VideoFrame outputCache;
18+
SwapChainEntry() :
19+
binding(nullptr),
20+
activetask(nullptr),
21+
outputCache(VideoFrame(Windows::Graphics::Imaging::BitmapPixelFormat::Bgra8, 720, 720)) {}
22+
};
2323

24-
struct StyleTransferEffect : StyleTransferEffectT<StyleTransferEffect>
25-
{
26-
StyleTransferEffect();
27-
IVectorView<VideoEncodingProperties> SupportedEncodingProperties();
28-
bool TimeIndependent();
29-
MediaMemoryTypes SupportedMemoryTypes();
30-
bool IsReadOnly();
31-
IPropertySet configuration;
24+
struct StyleTransferEffect : StyleTransferEffectT<StyleTransferEffect>
25+
{
26+
StyleTransferEffect();
27+
IVectorView<VideoEncodingProperties> SupportedEncodingProperties();
28+
bool TimeIndependent();
29+
MediaMemoryTypes SupportedMemoryTypes();
30+
bool IsReadOnly();
31+
IPropertySet configuration;
3232

33-
void DiscardQueuedFrames();
34-
void Close(MediaEffectClosedReason);
35-
void ProcessFrame(ProcessVideoFrameContext);
36-
void SetEncodingProperties(VideoEncodingProperties, IDirect3DDevice);
37-
void SetProperties(IPropertySet);
38-
void SubmitEval(VideoFrame, VideoFrame);
33+
void DiscardQueuedFrames();
34+
void Close(MediaEffectClosedReason);
35+
void ProcessFrame(ProcessVideoFrameContext);
36+
void SetEncodingProperties(VideoEncodingProperties, IDirect3DDevice);
37+
void SetProperties(IPropertySet);
38+
void SubmitEval(VideoFrame, VideoFrame);
3939

40-
private:
41-
VideoEncodingProperties encodingProperties;
42-
std::mutex Processing;
43-
StyleTransferEffectNotifier Notifier;
44-
LearningModelSession Session;
45-
Windows::Foundation::IAsyncOperation<LearningModelEvaluationResult> evalStatus;
46-
hstring InputImageDescription;
47-
hstring OutputImageDescription;
48-
int swapChainIndex = 0;
49-
static const int swapChainEntryCount = 5;
50-
std::vector < std::unique_ptr<SwapChainEntry>> bindings;
51-
int finishedIdx = 0;
52-
};
40+
private:
41+
VideoEncodingProperties encodingProperties;
42+
std::mutex Processing;
43+
StyleTransferEffectNotifier Notifier;
44+
LearningModelSession Session;
45+
Windows::Foundation::IAsyncOperation<LearningModelEvaluationResult> evalStatus;
46+
hstring InputImageDescription;
47+
hstring OutputImageDescription;
48+
int swapChainIndex = 0;
49+
static const int swapChainEntryCount = 5;
50+
std::vector < std::unique_ptr<SwapChainEntry>> bindings;
51+
int finishedFrameIndex = 0;
52+
};
5353
}
5454

5555
namespace winrt::StyleTransferEffectCpp::factory_implementation
5656
{
57-
struct StyleTransferEffect : StyleTransferEffectT<StyleTransferEffect, implementation::StyleTransferEffect>
58-
{
59-
};
57+
struct StyleTransferEffect : StyleTransferEffectT<StyleTransferEffect, implementation::StyleTransferEffect>
58+
{
59+
};
6060
}

0 commit comments

Comments
 (0)