trying to optimize with copying to rectangle of model inputframe

Author: linnealovespie

Samples/StyleTransfer/AppViewModel.cs

@@ -50,8 +50,17 @@ public AppViewModel()
             NotifyUser(true);
 
             m_notifier = new StyleTransferEffectNotifier();
-            m_notifier.FrameRateUpdated += async (_, e) => await DispatcherHelper.RunAsync(() => RenderFPS = e);
-            _useGpu = true;
+            m_notifier.FrameRateUpdated += async (_, e) =>
+            {
+                await DispatcherHelper.RunAsync(() =>
+                {
+                    // Running average just to see where it levels out at
+                    float temp = RenderFPS;
+                    RenderFPS = ((temp * numFrames) + e) / ++numFrames;
+                });
+
+            };
+            _useGpu = false;
             isPreviewing = false;
         }
 
@@ -61,6 +70,7 @@ public AppViewModel()
         private MediaFrameSourceGroup _selectedMediaFrameSourceGroup;
         private bool isPreviewing;
         private DisplayRequest displayRequest = new DisplayRequest();
+        private int numFrames = 0;
 
         // Style transfer effect properties
         private LearningModel m_model = null;
@@ -172,6 +182,8 @@ public async void SaveOutput()
         public async Task SetMediaSource(string src)
         {
             _appModel.InputMedia = src;
+            await CleanupCameraAsync();
+            CleanupInputImage();
 
             NotifyUser(true);
             SaveEnabled = true;
@@ -183,11 +195,9 @@ public async Task SetMediaSource(string src)
                     await StartLiveStream();
                     break;
                 case "AcquireImage":
-                    CleanupInputImage();
                     await StartAcquireImage();
                     break;
                 case "FilePick":
-                    CleanupInputImage();
                     await StartFilePick();
                     break;
                 case "Inking":
@@ -519,14 +529,22 @@ private void CleanupInputImage()
         {
             try
             {
-                InputSoftwareBitmapSource?.Dispose();
-                OutputSoftwareBitmapSource?.Dispose();
-
-                InputSoftwareBitmapSource = new SoftwareBitmapSource();
-                OutputSoftwareBitmapSource = new SoftwareBitmapSource();
-
-                _appModel.OutputFrame?.Dispose();
+                if (InputSoftwareBitmapSource != null)
+                {
+                    InputSoftwareBitmapSource.Dispose();
+                    InputSoftwareBitmapSource = new SoftwareBitmapSource();
+                }
+                if (OutputSoftwareBitmapSource != null)
+                {
+                    OutputSoftwareBitmapSource.Dispose();
+                    OutputSoftwareBitmapSource = new SoftwareBitmapSource();
+                }
+                if (_appModel.OutputFrame != null)
+                {
+                    _appModel.OutputFrame.Dispose();
+                }
                 _appModel.OutputFrame = new VideoFrame(BitmapPixelFormat.Bgra8, (int)m_outWidth, (int)m_outHeight);
+
             }
             catch (Exception e)
             {

Samples/StyleTransfer/VideoEffect/StyleTransferEffectCpp/StyleTransferEffect.cpp

@@ -7,7 +7,10 @@ using namespace winrt::Windows::Storage::Streams;
 
 namespace winrt::StyleTransferEffectCpp::implementation
 {
-	StyleTransferEffect::StyleTransferEffect() : outputTransformed(VideoFrame(Windows::Graphics::Imaging::BitmapPixelFormat::Bgra8, 720, 720)),
+	StyleTransferEffect::StyleTransferEffect() :
+		outputTransformed(VideoFrame(Windows::Graphics::Imaging::BitmapPixelFormat::Bgra8, 720, 720)),
+		inputTransformed(VideoFrame(Windows::Graphics::Imaging::BitmapPixelFormat::Bgra8, 720, 720)),
+		copyBounds(Windows::Graphics::Imaging::BitmapBounds{ 0, 0, 640, 360 }),
 		Session(nullptr),
 		Binding(nullptr)
 	{
@@ -38,29 +41,32 @@ namespace winrt::StyleTransferEffectCpp::implementation
 		std::lock_guard<mutex> guard{ Processing };
 		auto now = std::chrono::high_resolution_clock::now();
 		std::chrono::milliseconds timePassed;
-		// If the first time calling ProcessFrame, just start the timer 
+		// If the first time calling ProcessFrame, just start the timer
 		if (firstProcessFrameCall) {
 			m_StartTime = now;
 			firstProcessFrameCall = false;
 		}
-		// On the second and any proceding process, 
+		// On the second and any proceding process,
 		else {
 			timePassed = std::chrono::duration_cast<std::chrono::milliseconds>(now - m_StartTime);
 			m_StartTime = now;
-			Notifier.SetFrameRate(1000.f / timePassed.count()); // Convert to FPS: milli to seconds, invert 
+			Notifier.SetFrameRate(1000.f / timePassed.count()); // Convert to FPS: milli to seconds, invert
 		}
 
 		OutputDebugString(L"PF Start | ");
 
 		VideoFrame inputFrame = context.InputFrame();
 		VideoFrame outputFrame = context.OutputFrame();
 
+		// X, Y, Width, Height
+		//inputFrame.CopyToAsync(inputTransformed, copyBounds, copyBounds).get();
 		OutputDebugString(L"PF Locked | ");
 		Binding.Bind(InputImageDescription, inputFrame);
 		Binding.Bind(OutputImageDescription, outputTransformed);
 
 		OutputDebugString(L"PF Eval | ");
 		Session.Evaluate(Binding, L"test");
+		OutputDebugString(L"PF Copy | ");
 		outputTransformed.CopyToAsync(outputFrame).get();
 
 		OutputDebugString(L"PF End\n ");

Samples/StyleTransfer/VideoEffect/StyleTransferEffectCpp/StyleTransferEffect.h

@@ -15,6 +15,7 @@ namespace winrt::StyleTransferEffectCpp::implementation
 	{
 		StyleTransferEffect();
 		VideoFrame outputTransformed;
+		VideoFrame inputTransformed;
 
 		IVectorView<VideoEncodingProperties> SupportedEncodingProperties();
 		bool TimeIndependent();
@@ -38,6 +39,7 @@ namespace winrt::StyleTransferEffectCpp::implementation
 		StyleTransferEffectNotifier Notifier;
 		std::chrono::time_point<std::chrono::steady_clock> m_StartTime;
 		bool firstProcessFrameCall = true;
+		Windows::Graphics::Imaging::BitmapBounds copyBounds;
 	};
 }