Skip to content

Commit a5b639b

Browse files
linnealovespielinnealovespie
authored
Merge pull request #333 from microsoft/user/t-limay/StyleTransfer-optimize
User/t limay/style transfer optimize
2 parents c4202b5 + 0e2ced6 commit a5b639b

File tree

9 files changed

+172
-127
lines changed

9 files changed

+172
-127
lines changed

‎Samples/StyleTransfer/AppViewModel.cs‎

Lines changed: 37 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
using System.Collections.Generic;
33
using System.ComponentModel;
44
using System.Linq;
5-
using System.Text;
65
using System.Threading.Tasks;
76
using System.Windows.Input;
87
using Windows.Media.Core;
@@ -49,18 +48,27 @@ public AppViewModel()
4948
_saveEnabled = true;
5049
NotifyUser(true);
5150

52-
m_notifier = new StyleTransferEffectNotifier();
53-
m_notifier.FrameRateUpdated += async (_, e) => await DispatcherHelper.RunAsync(() => RenderFPS = e);
54-
_useGpu = true;
51+
m_notifier = new StyleTransferEffectCpp.StyleTransferEffectNotifier();
52+
m_notifier.FrameRateUpdated += async (_, e) =>
53+
{
54+
await DispatcherHelper.RunAsync(() =>
55+
{
56+
RenderFPS = e;
57+
});
58+
59+
};
60+
_useGpu = false;
5561
isPreviewing = false;
5662
}
5763

5864
// Media capture properties
5965
public Windows.Media.Capture.MediaCapture _mediaCapture;
60-
private List<MediaFrameSourceGroup> _mediaFrameSourceGroupList;
61-
private MediaFrameSourceGroup _selectedMediaFrameSourceGroup;
6266
private bool isPreviewing;
6367
private DisplayRequest displayRequest = new DisplayRequest();
68+
DeviceInformationCollection devices;
69+
System.Threading.Mutex Processing = new Mutex();
70+
private float _renderFPS;
71+
StyleTransferEffectCpp.StyleTransferEffectNotifier m_notifier;
6472

6573
// Style transfer effect properties
6674
private LearningModel m_model = null;
@@ -69,15 +77,8 @@ public AppViewModel()
6977
private LearningModelBinding m_binding;
7078
private string m_inputImageDescription;
7179
private string m_outputImageDescription;
72-
private IMediaExtension videoEffect;
73-
private VideoEffectDefinition videoEffectDefinition;
7480
// Activatable Class ID of the video effect.
7581
private String _videoEffectID = "StyleTransferEffectCpp.StyleTransferEffect";
76-
System.Threading.Mutex Processing = new Mutex();
77-
StyleTransferEffectCpp.StyleTransferEffectNotifier m_notifier;
78-
private float _renderFPS;
79-
DeviceInformationCollection devices;
80-
8182

8283
// Image style transfer properties
8384
uint m_inWidth, m_inHeight, m_outWidth, m_outHeight;
@@ -93,7 +94,6 @@ public AppModel CurrentApp
9394
{
9495
get { return _appModel; }
9596
}
96-
9797
public float RenderFPS
9898
{
9999
get { return (float)Math.Round(_renderFPS, 2); }
@@ -104,13 +104,9 @@ public float RenderFPS
104104
public float CaptureFPS
105105
{
106106
get
107-
{
108-
return _captureFPS;
109-
}
107+
{ return _captureFPS; }
110108
set
111-
{
112-
_captureFPS = value; OnPropertyChanged();
113-
}
109+
{ _captureFPS = value; OnPropertyChanged(); }
114110
}
115111
private bool _useGpu;
116112
public bool UseGpu
@@ -172,28 +168,26 @@ public async void SaveOutput()
172168
public async Task SetMediaSource(string src)
173169
{
174170
_appModel.InputMedia = src;
171+
await CleanupCameraAsync();
172+
CleanupInputImage();
175173

176174
NotifyUser(true);
177175
SaveEnabled = true;
178176

179-
// Changes media source while keeping all other controls
180177
switch (_appModel.InputMedia)
181178
{
182179
case "LiveStream":
183180
await StartLiveStream();
184181
break;
185182
case "AcquireImage":
186-
CleanupInputImage();
187183
await StartAcquireImage();
188184
break;
189185
case "FilePick":
190-
CleanupInputImage();
191186
await StartFilePick();
192187
break;
193-
case "Inking":
188+
default:
194189
break;
195190
}
196-
197191
return;
198192
}
199193

@@ -213,7 +207,7 @@ public async Task SetModelSource()
213207
case "FilePick":
214208
await ChangeImage();
215209
break;
216-
case "Inking":
210+
default:
217211
break;
218212
}
219213
}
@@ -255,9 +249,6 @@ public async Task StartFilePick()
255249
var file = await StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///Assets/{_DefaultImageFileName}"));
256250
_appModel.InputFrame = await ImageHelper.LoadVideoFrameFromStorageFileAsync(file);
257251
}
258-
else
259-
{
260-
}
261252
await ChangeImage();
262253
}
263254
catch (Exception ex)
@@ -276,7 +267,6 @@ public async Task ChangeImage()
276267
var file = await StorageFile.GetFileFromApplicationUriAsync(new Uri($"ms-appx:///Assets/{_DefaultImageFileName}"));
277268
_appModel.InputFrame = await ImageHelper.LoadVideoFrameFromStorageFileAsync(file);
278269
}
279-
280270
await EvaluateVideoFrameAsync();
281271
}
282272

@@ -374,7 +364,6 @@ public async Task ChangeLiveStream()
374364
var settings = new MediaCaptureInitializationSettings
375365
{
376366
VideoDeviceId = device.Id,
377-
SourceGroup = _selectedMediaFrameSourceGroup,
378367
PhotoCaptureSource = PhotoCaptureSource.Auto,
379368
MemoryPreference = UseGpu ? MediaCaptureMemoryPreference.Auto : MediaCaptureMemoryPreference.Cpu,
380369
StreamingCaptureMode = StreamingCaptureMode.Video,
@@ -389,11 +378,11 @@ public async Task ChangeLiveStream()
389378
_appModel.OutputCaptureElement = capture;
390379

391380
var modelPath = Path.GetFullPath($"./Assets/{_appModel.ModelSource}.onnx");
392-
videoEffectDefinition = new VideoEffectDefinition(_videoEffectID, new PropertySet() {
381+
VideoEffectDefinition videoEffectDefinition = new VideoEffectDefinition(_videoEffectID, new PropertySet() {
393382
{"ModelName", modelPath },
394383
{"UseGpu", UseGpu },
395384
{ "Notifier", m_notifier} });
396-
videoEffect = await _mediaCapture.AddVideoEffectAsync(videoEffectDefinition, MediaStreamType.VideoPreview);
385+
IMediaExtension videoEffect = await _mediaCapture.AddVideoEffectAsync(videoEffectDefinition, MediaStreamType.VideoPreview);
397386

398387
var props = _mediaCapture.VideoDeviceController.GetMediaStreamProperties(MediaStreamType.VideoPreview) as VideoEncodingProperties;
399388
CaptureFPS = props.FrameRate.Numerator / props.FrameRate.Denominator;
@@ -519,14 +508,22 @@ private void CleanupInputImage()
519508
{
520509
try
521510
{
522-
InputSoftwareBitmapSource?.Dispose();
523-
OutputSoftwareBitmapSource?.Dispose();
524-
525-
InputSoftwareBitmapSource = new SoftwareBitmapSource();
526-
OutputSoftwareBitmapSource = new SoftwareBitmapSource();
527-
528-
_appModel.OutputFrame?.Dispose();
511+
if (InputSoftwareBitmapSource != null)
512+
{
513+
InputSoftwareBitmapSource.Dispose();
514+
InputSoftwareBitmapSource = new SoftwareBitmapSource();
515+
}
516+
if (OutputSoftwareBitmapSource != null)
517+
{
518+
OutputSoftwareBitmapSource.Dispose();
519+
OutputSoftwareBitmapSource = new SoftwareBitmapSource();
520+
}
521+
if (_appModel.OutputFrame != null)
522+
{
523+
_appModel.OutputFrame.Dispose();
524+
}
529525
_appModel.OutputFrame = new VideoFrame(BitmapPixelFormat.Bgra8, (int)m_outWidth, (int)m_outHeight);
526+
530527
}
531528
catch (Exception e)
532529
{

‎Samples/StyleTransfer/Package.appxmanifest‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
<Identity
1010
Name="7fafd263-abdf-4f88-87a1-ffb0922489d1"
1111
Publisher="CN=t-limay"
12-
Version="1.0.9.0" />
12+
Version="1.0.11.0" />
1313

1414
<mp:PhoneIdentity PhoneProductId="7fafd263-abdf-4f88-87a1-ffb0922489d1" PhonePublisherId="00000000-0000-0000-0000-000000000000"/>
1515

‎Samples/StyleTransfer/StyleTransfer.csproj‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,7 @@
212212
</ItemGroup>
213213
<ItemGroup>
214214
<PackageReference Include="Microsoft.AI.MachineLearning">
215-
<Version>1.3.1-dev-20200711-0122-00cc8041c</Version>
215+
<Version>1.4.0</Version>
216216
</PackageReference>
217217
<PackageReference Include="Microsoft.NETCore.UniversalWindowsPlatform">
218218
<Version>6.2.10</Version>
@@ -221,7 +221,7 @@
221221
<Version>2.4.2</Version>
222222
</PackageReference>
223223
<PackageReference Include="Microsoft.Windows.CppWinRT">
224-
<Version>2.0.200703.9</Version>
224+
<Version>2.0.200729.8</Version>
225225
</PackageReference>
226226
<PackageReference Include="Microsoft.Xaml.Behaviors.Uwp.Managed">
227227
<Version>2.0.1</Version>
@@ -241,10 +241,10 @@
241241
<None Include="StyleTransfer_TemporaryKey.pfx" />
242242
</ItemGroup>
243243
<ItemGroup>
244-
<ProjectReference Include="VideoEffect\StyleTransferEffectCpp\StyleTransferEffectCpp.vcxproj">
245-
<Project>{05dffb01-62c4-458c-be13-7a97e8583e43}</Project>
246-
<Name>StyleTransferEffectCpp</Name>
247-
</ProjectReference>
244+
<Reference Include="StyleTransferEffectCpp, Version=255.255.255.255, Culture=neutral, processorArchitecture=MSIL">
245+
<SpecificVersion>False</SpecificVersion>
246+
<HintPath>VideoEffect\StyleTransferEffectCpp\x64\Debug\StyleTransferEffectCpp\StyleTransferEffectCpp.winmd</HintPath>
247+
</Reference>
248248
</ItemGroup>
249249
<PropertyGroup Condition=" '$(VisualStudioVersion)' == '' or '$(VisualStudioVersion)' &lt; '14.0' ">
250250
<VisualStudioVersion>14.0</VisualStudioVersion>

‎Samples/StyleTransfer/StyleTransfer.sln‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@ VisualStudioVersion = 16.0.30204.135
55
MinimumVisualStudioVersion = 10.0.40219.1
66
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StyleTransfer", "StyleTransfer.csproj", "{9CC66287-64B8-4321-B701-7E52F200A027}"
77
EndProject
8-
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "StyleTransferEffectCpp", "VideoEffect\StyleTransferEffectCpp\StyleTransferEffectCpp.vcxproj", "{05DFFB01-62C4-458C-BE13-7A97E8583E43}"
9-
EndProject
108
Global
119
GlobalSection(SolutionConfigurationPlatforms) = preSolution
1210
Debug|ARM = Debug|ARM
@@ -43,22 +41,6 @@ Global
4341
{9CC66287-64B8-4321-B701-7E52F200A027}.Release|x86.ActiveCfg = Release|x86
4442
{9CC66287-64B8-4321-B701-7E52F200A027}.Release|x86.Build.0 = Release|x86
4543
{9CC66287-64B8-4321-B701-7E52F200A027}.Release|x86.Deploy.0 = Release|x86
46-
{05DFFB01-62C4-458C-BE13-7A97E8583E43}.Debug|ARM.ActiveCfg = Debug|ARM
47-
{05DFFB01-62C4-458C-BE13-7A97E8583E43}.Debug|ARM.Build.0 = Debug|ARM
48-
{05DFFB01-62C4-458C-BE13-7A97E8583E43}.Debug|ARM64.ActiveCfg = Debug|ARM64
49-
{05DFFB01-62C4-458C-BE13-7A97E8583E43}.Debug|ARM64.Build.0 = Debug|ARM64
50-
{05DFFB01-62C4-458C-BE13-7A97E8583E43}.Debug|x64.ActiveCfg = Debug|x64
51-
{05DFFB01-62C4-458C-BE13-7A97E8583E43}.Debug|x64.Build.0 = Debug|x64
52-
{05DFFB01-62C4-458C-BE13-7A97E8583E43}.Debug|x86.ActiveCfg = Debug|Win32
53-
{05DFFB01-62C4-458C-BE13-7A97E8583E43}.Debug|x86.Build.0 = Debug|Win32
54-
{05DFFB01-62C4-458C-BE13-7A97E8583E43}.Release|ARM.ActiveCfg = Release|ARM
55-
{05DFFB01-62C4-458C-BE13-7A97E8583E43}.Release|ARM.Build.0 = Release|ARM
56-
{05DFFB01-62C4-458C-BE13-7A97E8583E43}.Release|ARM64.ActiveCfg = Release|ARM64
57-
{05DFFB01-62C4-458C-BE13-7A97E8583E43}.Release|ARM64.Build.0 = Release|ARM64
58-
{05DFFB01-62C4-458C-BE13-7A97E8583E43}.Release|x64.ActiveCfg = Release|x64
59-
{05DFFB01-62C4-458C-BE13-7A97E8583E43}.Release|x64.Build.0 = Release|x64
60-
{05DFFB01-62C4-458C-BE13-7A97E8583E43}.Release|x86.ActiveCfg = Release|Win32
61-
{05DFFB01-62C4-458C-BE13-7A97E8583E43}.Release|x86.Build.0 = Release|Win32
6244
EndGlobalSection
6345
GlobalSection(SolutionProperties) = preSolution
6446
HideSolutionNode = FALSE

0 commit comments

Comments
 (0)