#43. Use threshold technique against difference image. Add onStopDetect callback. Use TimeSpan in MotionConfig. Removed RGB16 support from ConvolutionBase and FrameDiffAnalyser due to exceptions being thrown by GDI+.

techyian · techyian · commit 55f29e462335 · 2020-02-27T12:33:10.000Z
diff --git a/src/MMALSharp.Processing/Handlers/CircularBufferCaptureHandler.cs b/src/MMALSharp.Processing/Handlers/CircularBufferCaptureHandler.cs
@@ -25,6 +25,7 @@ public sealed class CircularBufferCaptureHandler : VideoStreamCaptureHandler, IM
         private Stopwatch _recordingElapsed;
         private IFrameAnalyser _analyser;
         private MotionConfig _config;
+        private Action _onStopDetect;
                 
         /// <summary>
         /// The circular buffer object responsible for storing image data.
@@ -65,8 +66,6 @@ public override void Process(ImageContext context)
                 {
                     this.Buffer.PushBack(context.Data[i]);
                 }
-
-                this.CheckRecordingProgress();
             }
             else
             {
@@ -107,12 +106,14 @@ public override void Process(ImageContext context)
                     this.Processed += context.Data.Length;
                 }
             }
-
+            
             if (_shouldDetectMotion && !_recordToFileStream)
             {
-                _analyser.Apply(context);
+                _analyser?.Apply(context);
             }
 
+            this.CheckRecordingProgress();
+
             // Not calling base method to stop data being written to the stream when not recording.
             this.ImageContext = context;
         }
@@ -122,11 +123,12 @@ public override void Process(ImageContext context)
         /// </summary>
         /// <param name="config">The motion configuration.</param>
         /// <param name="onDetect">A callback for when motion is detected.</param>
-        public void DetectMotion(MotionConfig config, Action onDetect)
+        /// <param name="onStopDetect">An optional callback for when the record duration has passed.</param>
+        public void ConfigureMotionDetection(MotionConfig config, Action onDetect, Action onStopDetect = null)
         {
             _config = config;
-            _shouldDetectMotion = true;
-
+            _onStopDetect = onStopDetect;
+            
             if (this.MotionType == MotionType.FrameDiff)
             {
                 _analyser = new FrameDiffAnalyser(config, onDetect);
@@ -135,6 +137,28 @@ public void DetectMotion(MotionConfig config, Action onDetect)
             {
                 // TODO: Motion vector analyser
             }
+
+            this.EnableMotionDetection();
+        }
+
+        /// <summary>
+        /// Enables motion detection. When configured, this will instruct the capture handler to detect motion.
+        /// </summary>
+        public void EnableMotionDetection()
+        {
+            _shouldDetectMotion = true;
+
+            MMALLog.Logger.LogInformation("Enabling motion detection.");
+        }
+
+        /// <summary>
+        /// Disables motion detection. When configured, this will instruct the capture handler not to detect motion.
+        /// </summary>
+        public void DisableMotionDetection()
+        {
+            _shouldDetectMotion = false;
+
+            MMALLog.Logger.LogInformation("Disabling motion detection.");
         }
 
         /// <summary>
@@ -165,10 +189,7 @@ public void StopRecording()
         /// <inheritdoc />
         public override void Dispose()
         {
-            if (_shouldDetectMotion)
-            {
-                this.CurrentStream?.Dispose();
-            }
+            this.CurrentStream?.Dispose();
         }
 
         /// <inheritdoc />
@@ -181,9 +202,22 @@ private void CheckRecordingProgress()
         {
             if (_recordingElapsed != null && _config != null)
             {
-                if (_recordingElapsed.Elapsed >= _config.RecordDuration.TimeOfDay)
+                if (_recordingElapsed.Elapsed >= _config.RecordDuration)
                 {
-                    this.StopRecording();
+                    if (_onStopDetect != null)
+                    {
+                        _onStopDetect();
+                    }
+                    else
+                    {
+                        this.StopRecording();
+                    }
+
+                    if (_analyser is FrameDiffAnalyser)
+                    {
+                        var fdAnalyser = _analyser as FrameDiffAnalyser;
+                        fdAnalyser?.ResetAnalyser();
+                    }
                 }
             }
         }
diff --git a/src/MMALSharp.Processing/Handlers/IMotionCaptureHandler.cs b/src/MMALSharp.Processing/Handlers/IMotionCaptureHandler.cs
@@ -5,7 +5,6 @@
 
 using MMALSharp.Processors.Motion;
 using System;
-using MMALSharp.Common;
 
 namespace MMALSharp.Handlers
 {
@@ -20,10 +19,21 @@ public interface IMotionCaptureHandler
         MotionType MotionType { get; set; }
 
         /// <summary>
-        /// Call to enable motion detection.
+        /// Call to configure motion detection.
         /// </summary>
         /// <param name="config">The motion configuration.</param>
         /// <param name="onDetect">A callback for when motion is detected.</param>
-        void DetectMotion(MotionConfig config, Action onDetect);
+        /// <param name="onStopDetect">An optional callback for when the record duration has passed.</param>
+        void ConfigureMotionDetection(MotionConfig config, Action onDetect, Action onStopDetect = null);
+
+        /// <summary>
+        /// Enables motion detection. When configured, this will instruct the capture handler to detect motion.
+        /// </summary>
+        void EnableMotionDetection();
+
+        /// <summary>
+        /// Disables motion detection. When configured, this will instruct the capture handler not to detect motion.
+        /// </summary>
+        void DisableMotionDetection();
     }
 }
diff --git a/src/MMALSharp.Processing/Processors/Effects/ConvolutionBase.cs b/src/MMALSharp.Processing/Processors/Effects/ConvolutionBase.cs
@@ -110,11 +110,7 @@ private Bitmap LoadBitmap(ImageContext imageContext, MemoryStream stream)
             {
                 PixelFormat format = default;
 
-                if (imageContext.PixelFormat == MMALEncoding.RGB16)
-                {
-                    format = PixelFormat.Format16bppRgb565;
-                }
-
+                // RGB16 doesn't appear to be supported by GDI?
                 if (imageContext.PixelFormat == MMALEncoding.RGB24)
                 {
                     format = PixelFormat.Format24bppRgb;
diff --git a/src/MMALSharp.Processing/Processors/Motion/FrameDiffAnalyser.cs b/src/MMALSharp.Processing/Processors/Motion/FrameDiffAnalyser.cs
@@ -13,7 +13,6 @@
 using Microsoft.Extensions.Logging;
 using MMALSharp.Common;
 using MMALSharp.Common.Utility;
-using MMALSharp.Processors.Effects;
 
 namespace MMALSharp.Processors.Motion
 {
@@ -75,12 +74,7 @@ public override void Apply(ImageContext context)
                 if (context.Eos)
                 {
                     this.FullTestFrame = true;
-                    MMALLog.Logger.LogDebug("EOS reached for test frame. Applying edge detection.");
-
-                    // We want to apply Edge Detection to the test frame to make it easier to detect changes.
-                    var edgeDetection = new EdgeDetection(this.MotionConfig.Sensitivity);
-                    this.ImageContext.Data = this.TestFrame.ToArray();
-                    edgeDetection.ApplyConvolution(edgeDetection.Kernel, EdgeDetection.KernelWidth, EdgeDetection.KernelHeight, this.ImageContext);
+                    MMALLog.Logger.LogDebug("EOS reached for test frame.");
                 }
             }
 
@@ -92,19 +86,27 @@ public override void Apply(ImageContext context)
                 this.CheckForChanges(this.OnDetect);
             }
         }
-        
-        private void CheckForChanges(Action onDetect)
+
+        /// <summary>
+        /// Resets the test and working frames this analyser is using.
+        /// </summary>
+        public void ResetAnalyser()
         {
-            var edgeDetection = new EdgeDetection(EDStrength.Medium);
-            this.ImageContext.Data = this.WorkingData.ToArray();
-            edgeDetection.ApplyConvolution(EdgeDetection.MediumStrengthKernel, 3, 3, this.ImageContext);
-            var diff = this.Analyse();
+            this.TestFrame = new List<byte>();
+            this.WorkingData = new List<byte>();
+            this.FullFrame = false;
+            this.FullTestFrame = false;
+        }
 
-            MMALLog.Logger.LogDebug($"Diff size: {diff}");
+        private void CheckForChanges(Action onDetect)
+        {
+            this.PrepareDifferenceImage(this.ImageContext, this.MotionConfig.Threshold);
 
+            var diff = this.Analyse();
+            
             if (diff >= this.MotionConfig.Threshold)
             {
-                MMALLog.Logger.LogInformation("Motion detected!");
+                MMALLog.Logger.LogInformation($"Motion detected! Frame difference {diff}.");
                 onDetect();
             }
         }
@@ -115,11 +117,7 @@ private Bitmap LoadBitmap(MemoryStream stream)
             {
                 PixelFormat format = default;
 
-                if (this.ImageContext.PixelFormat == MMALEncoding.RGB16)
-                {
-                    format = PixelFormat.Format16bppRgb565;
-                }
-
+                // RGB16 doesn't appear to be supported by GDI?
                 if (this.ImageContext.PixelFormat == MMALEncoding.RGB24)
                 {
                     format = PixelFormat.Format24bppRgb;
@@ -198,11 +196,114 @@ private int Analyse()
 
                 testBmp.UnlockBits(testBmpData);
                 currentBmp.UnlockBits(currentBmpData);
-             
+
                 return diff;
             }
         }
 
+        private void PrepareDifferenceImage(ImageContext context, int threshold)
+        {
+            BitmapData bmpData = null;
+            IntPtr pNative = IntPtr.Zero;
+            int bytes;
+            byte[] store = null;
+
+            using (var ms = new MemoryStream(context.Data))
+            using (var bmp = this.LoadBitmap(ms))
+            {
+                bmpData = bmp.LockBits(new Rectangle(0, 0,
+                        bmp.Width,
+                        bmp.Height),
+                    ImageLockMode.ReadWrite,
+                    bmp.PixelFormat);
+
+                if (context.Raw)
+                {
+                    this.InitBitmapData(bmpData, ms.ToArray());
+                }
+
+                pNative = bmpData.Scan0;
+
+                // Split image into 4 quadrants and process individually.
+                var quadA = new Rectangle(0, 0, bmpData.Width / 2, bmpData.Height / 2);
+                var quadB = new Rectangle(bmpData.Width / 2, 0, bmpData.Width / 2, bmpData.Height / 2);
+                var quadC = new Rectangle(0, bmpData.Height / 2, bmpData.Width / 2, bmpData.Height / 2);
+                var quadD = new Rectangle(bmpData.Width / 2, bmpData.Height / 2, bmpData.Width / 2, bmpData.Height / 2);
+
+                bytes = bmpData.Stride * bmp.Height;
+
+                var rgbValues = new byte[bytes];
+
+                // Copy the RGB values into the array.
+                Marshal.Copy(pNative, rgbValues, 0, bytes);
+
+                var bpp = Image.GetPixelFormatSize(bmp.PixelFormat) / 8;
+
+                var t1 = Task.Run(() =>
+                {
+                    this.ApplyThreshold(quadA, bmpData, bpp, threshold);
+                });
+                var t2 = Task.Run(() =>
+                {
+                    this.ApplyThreshold(quadB, bmpData, bpp, threshold);
+                });
+                var t3 = Task.Run(() =>
+                {
+                    this.ApplyThreshold(quadC, bmpData, bpp, threshold);
+                });
+                var t4 = Task.Run(() =>
+                {
+                    this.ApplyThreshold(quadD, bmpData, bpp, threshold);
+                });
+
+                Task.WaitAll(t1, t2, t3, t4);
+
+                if (context.Raw)
+                {
+                    store = new byte[bytes];
+                    Marshal.Copy(pNative, store, 0, bytes);
+                }
+
+                bmp.UnlockBits(bmpData);
+            }
+
+            context.Data = store;
+        }
+
+        private void ApplyThreshold(Rectangle quad, BitmapData bmpData, int pixelDepth, int threshold)
+        {
+            unsafe
+            {
+                // Declare an array to hold the bytes of the bitmap.
+                var stride = bmpData.Stride;
+
+                byte* ptr1 = (byte*)bmpData.Scan0;
+
+                for (int column = quad.X; column < quad.X + quad.Width; column++)
+                {
+                    for (int row = quad.Y; row < quad.Y + quad.Height; row++)
+                    {
+                        var rgb1 = ptr1[(column * pixelDepth) + (row * stride)] +
+                                   ptr1[(column * pixelDepth) + (row * stride) + 1] +
+                                   ptr1[(column * pixelDepth) + (row * stride) + 2];
+
+                        if (rgb1 > threshold)
+                        {
+                            ptr1[(column * pixelDepth) + (row * stride)] = 255;
+                            ptr1[(column * pixelDepth) + (row * stride) + 1] = 255;
+                            ptr1[(column * pixelDepth) + (row * stride) + 2] = 255;
+                        }
+                        else
+                        {
+                            ptr1[(column * pixelDepth) + (row * stride)] = 0;
+                            ptr1[(column * pixelDepth) + (row * stride) + 1] = 0;
+                            ptr1[(column * pixelDepth) + (row * stride) + 2] = 0;
+                        }
+                    }
+                }
+            }
+        }
+
         private int CheckDiff(Rectangle quad, BitmapData bmpData, BitmapData bmpData2, int pixelDepth, int threshold)
         {
             unsafe
@@ -228,7 +329,7 @@ private int CheckDiff(Rectangle quad, BitmapData bmpData, BitmapData bmpData2, i
                         ptr2[(column * pixelDepth) + (row * stride2) + 1] +
                         ptr2[(column * pixelDepth) + (row * stride2) + 2];
                         
-                        if (rgb2 > rgb1 + threshold)
+                        if (rgb2 - rgb1 > threshold)
                         {
                             diff++;
 
@@ -252,6 +353,17 @@ private int CheckDiff(Rectangle quad, BitmapData bmpData, BitmapData bmpData2, i
                                 highestX = column;
                             }
                         }
+
+                        // If the threshold has been exceeded, we want to exit from this method immediately for performance reasons.
+                        if (diff > threshold)
+                        {
+                            break;
+                        }
+                    }
+
+                    if (diff > threshold)
+                    {
+                        break;
                     }
                 }
 
diff --git a/src/MMALSharp.Processing/Processors/Motion/MotionConfig.cs b/src/MMALSharp.Processing/Processors/Motion/MotionConfig.cs
diff --git a/src/MMALSharp/MMALCamera.cs b/src/MMALSharp/MMALCamera.cs

Original file line number	Diff line number	Diff line change
`@@ -25,6 +25,7 @@ public sealed class CircularBufferCaptureHandler : VideoStreamCaptureHandler, IM`
`25`	`25`	`private Stopwatch _recordingElapsed;`
`26`	`26`	`private IFrameAnalyser _analyser;`
`27`	`27`	`private MotionConfig _config;`
	`28`	`+ private Action _onStopDetect;`
`28`	`29`
`29`	`30`	`/// <summary>`
`30`	`31`	`/// The circular buffer object responsible for storing image data.`
`@@ -65,8 +66,6 @@ public override void Process(ImageContext context)`
`65`	`66`	`{`
`66`	`67`	`this.Buffer.PushBack(context.Data[i]);`
`67`	`68`	`}`
`68`		`-`
`69`		`- this.CheckRecordingProgress();`
`70`	`69`	`}`
`71`	`70`	`else`
`72`	`71`	`{`
`@@ -107,12 +106,14 @@ public override void Process(ImageContext context)`
`107`	`106`	`this.Processed += context.Data.Length;`
`108`	`107`	`}`
`109`	`108`	`}`
`110`		`-`
	`109`	`+`
`111`	`110`	`if (_shouldDetectMotion && !_recordToFileStream)`
`112`	`111`	`{`
`113`		`- _analyser.Apply(context);`
	`112`	`+ _analyser?.Apply(context);`
`114`	`113`	`}`
`115`	`114`
	`115`	`+ this.CheckRecordingProgress();`
	`116`	`+`
`116`	`117`	`// Not calling base method to stop data being written to the stream when not recording.`
`117`	`118`	`this.ImageContext = context;`
`118`	`119`	`}`
`@@ -122,11 +123,12 @@ public override void Process(ImageContext context)`
`122`	`123`	`/// </summary>`
`123`	`124`	`/// <param name="config">The motion configuration.</param>`
`124`	`125`	`/// <param name="onDetect">A callback for when motion is detected.</param>`
`125`		`- public void DetectMotion(MotionConfig config, Action onDetect)`
	`126`	`+ /// <param name="onStopDetect">An optional callback for when the record duration has passed.</param>`
	`127`	`+ public void ConfigureMotionDetection(MotionConfig config, Action onDetect, Action onStopDetect = null)`
`126`	`128`	`{`
`127`	`129`	`_config = config;`
`128`		`- _shouldDetectMotion = true;`
`129`		`-`
	`130`	`+ _onStopDetect = onStopDetect;`
	`131`	`+`
`130`	`132`	`if (this.MotionType == MotionType.FrameDiff)`
`131`	`133`	`{`
`132`	`134`	`_analyser = new FrameDiffAnalyser(config, onDetect);`
`@@ -135,6 +137,28 @@ public void DetectMotion(MotionConfig config, Action onDetect)`
`135`	`137`	`{`
`136`	`138`	`// TODO: Motion vector analyser`
`137`	`139`	`}`
	`140`	`+`
	`141`	`+ this.EnableMotionDetection();`
	`142`	`+ }`
	`143`	`+`
	`144`	`+ /// <summary>`
	`145`	`+ /// Enables motion detection. When configured, this will instruct the capture handler to detect motion.`
	`146`	`+ /// </summary>`
	`147`	`+ public void EnableMotionDetection()`
	`148`	`+ {`
	`149`	`+ _shouldDetectMotion = true;`
	`150`	`+`
	`151`	`+ MMALLog.Logger.LogInformation("Enabling motion detection.");`
	`152`	`+ }`
	`153`	`+`
	`154`	`+ /// <summary>`
	`155`	`+ /// Disables motion detection. When configured, this will instruct the capture handler not to detect motion.`
	`156`	`+ /// </summary>`
	`157`	`+ public void DisableMotionDetection()`
	`158`	`+ {`
	`159`	`+ _shouldDetectMotion = false;`
	`160`	`+`
	`161`	`+ MMALLog.Logger.LogInformation("Disabling motion detection.");`
`138`	`162`	`}`
`139`	`163`
`140`	`164`	`/// <summary>`
`@@ -165,10 +189,7 @@ public void StopRecording()`
`165`	`189`	`/// <inheritdoc />`
`166`	`190`	`public override void Dispose()`
`167`	`191`	`{`
`168`		`- if (_shouldDetectMotion)`
`169`		`- {`
`170`		`- this.CurrentStream?.Dispose();`
`171`		`- }`
	`192`	`+ this.CurrentStream?.Dispose();`
`172`	`193`	`}`
`173`	`194`
`174`	`195`	`/// <inheritdoc />`
`@@ -181,9 +202,22 @@ private void CheckRecordingProgress()`
`181`	`202`	`{`
`182`	`203`	`if (_recordingElapsed != null && _config != null)`
`183`	`204`	`{`
`184`		`- if (_recordingElapsed.Elapsed >= _config.RecordDuration.TimeOfDay)`
	`205`	`+ if (_recordingElapsed.Elapsed >= _config.RecordDuration)`
`185`	`206`	`{`
`186`		`- this.StopRecording();`
	`207`	`+ if (_onStopDetect != null)`
	`208`	`+ {`
	`209`	`+ _onStopDetect();`
	`210`	`+ }`
	`211`	`+ else`
	`212`	`+ {`
	`213`	`+ this.StopRecording();`
	`214`	`+ }`
	`215`	`+`
	`216`	`+ if (_analyser is FrameDiffAnalyser)`
	`217`	`+ {`
	`218`	`+ var fdAnalyser = _analyser as FrameDiffAnalyser;`
	`219`	`+ fdAnalyser?.ResetAnalyser();`
	`220`	`+ }`
`187`	`221`	`}`
`188`	`222`	`}`
`189`	`223`	`}`
Original file line number	Diff line number	Diff line change
`@@ -110,11 +110,7 @@ private Bitmap LoadBitmap(ImageContext imageContext, MemoryStream stream)`
`110`	`110`	`{`
`111`	`111`	`PixelFormat format = default;`
`112`	`112`
`113`		`- if (imageContext.PixelFormat == MMALEncoding.RGB16)`
`114`		`- {`
`115`		`- format = PixelFormat.Format16bppRgb565;`
`116`		`- }`
`117`		`-`
	`113`	`+ // RGB16 doesn't appear to be supported by GDI?`
`118`	`114`	`if (imageContext.PixelFormat == MMALEncoding.RGB24)`
`119`	`115`	`{`
`120`	`116`	`format = PixelFormat.Format24bppRgb;`