send framerate from processframe

linnealovespie · linnealovespie · commit ae57c955a31b · 2020-07-14T16:23:04.000-04:00
diff --git a/Samples/StyleTransfer/AppViewModel.cs b/Samples/StyleTransfer/AppViewModel.cs
@@ -49,13 +49,8 @@ public AppViewModel()
             NotifyUser(true);
 
             m_notifier = new StyleTransferEffectNotifier();
-            m_notifier.AccountIsInDebit += M_notifier_AccountIsInDebit;
-            m_notifier.AdjustBalance(-4.0f);
-        }
+            m_notifier.FrameRateUpdated += async (_, e) => await DispatcherHelper.RunAsync(() => RenderFPS = e);
 
-        private void M_notifier_AccountIsInDebit(object sender, float e)
-        {
-            Debug.WriteLine("Account in Debit: " + e);
         }
 
         // Media capture properties
@@ -77,8 +72,8 @@ private void M_notifier_AccountIsInDebit(object sender, float e)
         // Activatable Class ID of the video effect. 
         private String _videoEffectID = "StyleTransferEffectCpp.StyleTransferEffect";
         System.Threading.Mutex Processing = new Mutex();
-        EventRegistrationToken m_eventToken;
         StyleTransferEffectCpp.StyleTransferEffectNotifier m_notifier;
+        private float _renderFPS;
 
         // Image style transfer properties
         uint m_inWidth, m_inHeight, m_outWidth, m_outHeight;
@@ -95,6 +90,24 @@ public AppModel CurrentApp
             get { return _appModel; }
         }
 
+        public float RenderFPS
+        {
+            get { return (float)Math.Round(_renderFPS, 2); }
+            set { _renderFPS = value; OnPropertyChanged(); }
+        }
+
+        private float _captureFPS;
+        public float CaptureFPS
+        {
+            get
+            {
+                return _captureFPS;
+            }
+            set
+            {
+                _captureFPS = value; OnPropertyChanged();
+            }
+        }
         private SoftwareBitmapSource _inputSoftwareBitmapSource;
         public SoftwareBitmapSource InputSoftwareBitmapSource
         {
@@ -350,7 +363,11 @@ public async Task ChangeLiveStream()
                 videoEffect = await _mediaCapture.AddVideoEffectAsync(videoEffectDefinition, MediaStreamType.VideoPreview);
                 videoEffect.SetProperties(new PropertySet() {
                     {"ModelName", modelPath },
-                    {"UseGPU", _appModel.UseGPU }});
+                    {"UseGPU", _appModel.UseGPU },
+                    { "Notifier", m_notifier} });
+
+                var props = _mediaCapture.VideoDeviceController.GetMediaStreamProperties(MediaStreamType.VideoPreview) as VideoEncodingProperties;
+                CaptureFPS = props.FrameRate.Numerator / props.FrameRate.Denominator;
 
                 await _mediaCapture.StartPreviewAsync();
                 isPreviewing = true;
@@ -425,6 +442,7 @@ private async Task CleanupCameraAsync()
             Debug.WriteLine("CleanupCameraAsync");
             try
             {
+                RenderFPS = 0;
                 // Clean up the media capture
                 if (_mediaCapture != null)
                 {
diff --git a/Samples/StyleTransfer/MainPage.xaml b/Samples/StyleTransfer/MainPage.xaml
@@ -320,9 +320,9 @@
               RelativePanel.AlignVerticalCenterWithPanel="True"
               Visibility="{Binding CurrentApp.StreamingControlsVisible, Converter={StaticResource BoolToVis}, Mode=OneWay}">
             <Run x:Uid="TxtCapture" FontWeight="Bold" />
-            <Run Text="{x:Bind dummyFPS}" />
+            <Run Text="{Binding CaptureFPS}" />
             <Run x:Uid="TxtRender" FontWeight="Bold" />
-            <Run FontWeight="Normal" Text="{x:Bind dummyFPS}" />
+            <Run FontWeight="Normal" Text="{Binding RenderFPS}" />
           </TextBlock>
         </RelativePanel>
       </Border>
diff --git a/Samples/StyleTransfer/VideoEffect/StyleTransferEffectCpp/StyleTransferEffect.cpp b/Samples/StyleTransfer/VideoEffect/StyleTransferEffectCpp/StyleTransferEffect.cpp
@@ -35,21 +35,25 @@ namespace winrt::StyleTransferEffectCpp::implementation
 	}
 
 	void StyleTransferEffect::ProcessFrame(ProcessVideoFrameContext context) {
-		Processing.lock();
 		OutputDebugString(L"Start ProcessFrame | ");
+		auto startSync = std::chrono::high_resolution_clock::now();
+
 		VideoFrame inputFrame = context.InputFrame();
 		VideoFrame outputFrame = context.OutputFrame();
 
-		OutputDebugString(L"PF Binding | ");
+		Processing.lock();
+		OutputDebugString(L"PF Locked | ");
 		Binding.Bind(InputImageDescription, inputFrame);
 		Binding.Bind(OutputImageDescription, outputTransformed);
 
 		OutputDebugString(L"PF Eval | ");
 		Session.Evaluate(Binding, L"test");
 		outputTransformed.CopyToAsync(context.OutputFrame());
-		OutputDebugString(L"Stop ProcessFrame | ");
 		Processing.unlock();
-		OutputDebugString(L"End Lock ProcessFrame\n");
+		OutputDebugString(L"PF Unlocked");
+
+		auto syncTime = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - startSync);
+		Notifier.SetFrameRate(1000.f / syncTime.count()); // Convert to FPS: milli to seconds, invert 
 	}
 
 	void StyleTransferEffect::SetEncodingProperties(VideoEncodingProperties props, IDirect3DDevice device) {
@@ -66,9 +70,10 @@ namespace winrt::StyleTransferEffectCpp::implementation
 		modelName = unbox_value<hstring>(val);
 		val = configuration.TryLookup(L"UseGPU");
 		bool useGpu = unbox_value<bool>(val);
-		OutputDebugString(modelName.c_str());
-		LearningModel m_model = LearningModel::LoadFromFilePath(modelName);
+		val = configuration.TryLookup(L"Notifier");
+		Notifier = val.try_as<StyleTransferEffectNotifier>();
 
+		LearningModel m_model = LearningModel::LoadFromFilePath(modelName);
 		LearningModelDeviceKind m_device = useGpu ? LearningModelDeviceKind::DirectX : LearningModelDeviceKind::Cpu;
 		Session = LearningModelSession{ m_model, LearningModelDevice(m_device) };
 		Binding = LearningModelBinding{ Session };
diff --git a/Samples/StyleTransfer/VideoEffect/StyleTransferEffectCpp/StyleTransferEffect.h b/Samples/StyleTransfer/VideoEffect/StyleTransferEffectCpp/StyleTransferEffect.h
@@ -35,6 +35,7 @@ namespace winrt::StyleTransferEffectCpp::implementation
 		hstring OutputImageDescription;
 		VideoEncodingProperties encodingProperties;
 		std::mutex Processing;
+		StyleTransferEffectNotifier Notifier;
 	};
 }
 
diff --git a/Samples/StyleTransfer/VideoEffect/StyleTransferEffectCpp/StyleTransferEffectNotifier.cpp b/Samples/StyleTransfer/VideoEffect/StyleTransferEffectCpp/StyleTransferEffectNotifier.cpp
@@ -0,0 +1,21 @@
+#include "pch.h"
+#include "StyleTransferEffectNotifier.h"
+#include "StyleTransferEffectNotifier.g.cpp"
+
+
+namespace winrt::StyleTransferEffectCpp::implementation
+{
+	void StyleTransferEffectNotifier::SetFrameRate(float value)
+	{
+		m_balance = value;
+		m_frameRateUpdatedEvent(*this, m_balance);
+	}
+	winrt::event_token StyleTransferEffectNotifier::FrameRateUpdated(Windows::Foundation::EventHandler<float> const& handler)
+	{
+		return m_frameRateUpdatedEvent.add(handler);
+	}
+	void StyleTransferEffectNotifier::FrameRateUpdated(winrt::event_token const& token) noexcept
+	{
+		m_frameRateUpdatedEvent.remove(token);
+	}
+}
diff --git a/Samples/StyleTransfer/VideoEffect/StyleTransferEffectCpp/StyleTransferEffectNotifier.h b/Samples/StyleTransfer/VideoEffect/StyleTransferEffectCpp/StyleTransferEffectNotifier.h
@@ -0,0 +1,24 @@
+#pragma once
+#include "StyleTransferEffectNotifier.g.h"
+
+
+namespace winrt::StyleTransferEffectCpp::implementation
+{
+	struct StyleTransferEffectNotifier : StyleTransferEffectNotifierT<StyleTransferEffectNotifier>
+	{
+		StyleTransferEffectNotifier() = default;
+
+		void SetFrameRate(float value);
+		winrt::event_token FrameRateUpdated(Windows::Foundation::EventHandler<float> const& handler);
+		void FrameRateUpdated(winrt::event_token const& token) noexcept;
+	private:
+		winrt::event<Windows::Foundation::EventHandler<float>> m_frameRateUpdatedEvent;
+		float m_balance{ 0.f };
+	};
+}
+namespace winrt::StyleTransferEffectCpp::factory_implementation
+{
+	struct StyleTransferEffectNotifier : StyleTransferEffectNotifierT<StyleTransferEffectNotifier, implementation::StyleTransferEffectNotifier>
+	{
+	};
+}
diff --git a/Samples/StyleTransfer/VideoEffect/StyleTransferEffectCpp/StyleTransferEffectNotifier.idl b/Samples/StyleTransfer/VideoEffect/StyleTransferEffectCpp/StyleTransferEffectNotifier.idl
@@ -0,0 +1,9 @@
+namespace StyleTransferEffectCpp
+{
+	runtimeclass StyleTransferEffectNotifier
+	{
+		StyleTransferEffectNotifier();
+		void SetFrameRate(Single value);
+		event Windows.Foundation.EventHandler<Single> FrameRateUpdated;
+	}
+}

Original file line number	Diff line number	Diff line change
`@@ -35,6 +35,7 @@ namespace winrt::StyleTransferEffectCpp::implementation`
`35`	`35`	`hstring OutputImageDescription;`
`36`	`36`	`VideoEncodingProperties encodingProperties;`
`37`	`37`	`std::mutex Processing;`
	`38`	`+ StyleTransferEffectNotifier Notifier;`
`38`	`39`	`};`
`39`	`40`	`}`
`40`	`41`