[scanner] Extend merge window to the sum of pre and post event length #72

Author: Breakthrough

dvr_scan/scanner.py

@@ -263,8 +263,12 @@ def __init__(
 
         # Motion Event Parameters (set_event_params)
         self._min_event_len = None  # -l/--min-event-length
-        self._pre_event_len = None  # -tb/--time-before-event
-        self._post_event_len = None  # -tp/--time-post-event
+        # TODO(v1.7): We really need three variables here:
+        #  - how much time after an event ends should we wait for more motion  <-- window
+        #  - how much time before/after an event should we include in the video
+
+        self._time_before_event = None  # -tb/--time-before-event
+        self._time_post_event = None  # -tp/--time-post-event
         self._use_pts = None  # --use_pts
 
         # Region Parameters (set_region)
@@ -471,8 +475,8 @@ def set_event_params(
         """Set motion event parameters."""
         assert self._input.framerate is not None
         self._min_event_len = FrameTimecode(min_event_len, self._input.framerate)
-        self._pre_event_len = FrameTimecode(time_pre_event, self._input.framerate)
-        self._post_event_len = FrameTimecode(time_post_event, self._input.framerate)
+        self._time_before_event = FrameTimecode(time_pre_event, self._input.framerate)
+        self._time_post_event = FrameTimecode(time_post_event, self._input.framerate)
         self._use_pts = use_pts
 
     def set_thumbnail_params(self, thumbnails: str = None):
@@ -693,8 +697,8 @@ def scan(self) -> Optional[DetectionResult]:
         )
 
         # Correct event length parameters to account frame skip.
-        post_event_len: int = self._post_event_len.frame_num // (self._frame_skip + 1)
-        pre_event_len: int = self._pre_event_len.frame_num // (self._frame_skip + 1)
+        post_event_len: int = self._time_post_event.frame_num // (self._frame_skip + 1)
+        pre_event_len: int = self._time_before_event.frame_num // (self._frame_skip + 1)
         min_event_len: int = max(self._min_event_len.frame_num // (self._frame_skip + 1), 1)
 
         # Calculations below rely on min_event_len always being >= 1 (cannot be zero)
@@ -705,21 +709,19 @@ def scan(self) -> Optional[DetectionResult]:
         # need to compensate for rounding errors when we corrected it for frame skip. This is
         # important as this affects the number of frames we consider for the actual motion event.
         if not self._use_pts:
-            start_event_shift: int = self._pre_event_len.frame_num + min_event_len * (
+            start_event_shift: int = self._time_before_event.frame_num + min_event_len * (
                 self._frame_skip + 1
             )
         else:
             start_event_shift_ms: float = (
-                self._pre_event_len.get_seconds() + self._min_event_len.get_seconds()
+                self._time_before_event.get_seconds() + self._min_event_len.get_seconds()
             ) * 1000
 
         # Length of buffer we require in memory to keep track of all frames required for -l and -tb.
         buff_len = pre_event_len + min_event_len
         event_end = self._input.position
-        if not self._use_pts:
-            last_frame_above_threshold = 0
-        else:
-            last_frame_above_threshold_ms = 0
+        last_frame_above_threshold = 0
+        last_frame_above_threshold_ms = 0
 
         if self._bounding_box:
             self._bounding_box.set_corrections(
@@ -825,6 +827,11 @@ def scan(self) -> Optional[DetectionResult]:
                     )
                 )
 
+            #
+            # TODO: Make a small state diagram and create a new state enum to handle different
+            # merging modes, etc.
+            #
+
             # Last frame was part of a motion event, or still within the post-event window.
             if in_motion_event:
                 # If this frame still has motion, reset the post-event window.
@@ -839,67 +846,39 @@ def scan(self) -> Optional[DetectionResult]:
                 #
                 # TODO(#72): We should wait until the max of *both* the pre-event and post-
                 # event windows have passed. Right now we just consider the post-event window.
+                # We should also allow configuring overlap behavior:
+                #  - normal: If any new motion is found within max(time_pre_event, time_post_event),
+                #            it will be merged with the preceeding event.
+                #  - extended: Events that have a gap of size (time_pre_event + time_post_event)
+                #              between each other will be merged.
                 else:
                     num_frames_post_event += 1
-                    if num_frames_post_event >= post_event_len:
+                    if num_frames_post_event >= (pre_event_len + post_event_len):
                         in_motion_event = False
-
-                        logger.debug(
-                            "event %d high score %f" % (1 + self._num_events, self._highscore)
+                        # TODO: We can't throw these frames away, they might be needed for the
+                        # next event to satisfy it's own pre_event_len.
+                        buffered_frames = []
+                        event = self._on_event_end(
+                            last_frame_above_threshold,
+                            last_frame_above_threshold_ms,
+                            event_start,
                         )
-                        if self._thumbnails == "highscore":
-                            video_name = get_filename(
-                                path=self._input.paths[0], include_extension=False
-                            )
-                            output_path = (
-                                self._comp_file
-                                if self._comp_file
-                                else OUTPUT_FILE_TEMPLATE.format(
-                                    VIDEO_NAME=video_name,
-                                    EVENT_NUMBER="%04d" % (1 + self._num_events),
-                                    EXTENSION="jpg",
-                                )
-                            )
-                            if self._output_dir:
-                                output_path = os.path.join(self._output_dir, output_path)
-                            cv2.imwrite(output_path, self._highframe)
-                            self._highscore = 0
-                            self._highframe = None
-
-                        # Calculate event end based on the last frame we had with motion plus
-                        # the post event length time. We also need to compensate for the number
-                        # of frames that we skipped that could have had motion.
-                        # We also add 1 to include the presentation duration of the last frame.
-                        if not self._use_pts:
-                            event_end = FrameTimecode(
-                                1
-                                + last_frame_above_threshold
-                                + self._post_event_len.frame_num
-                                + self._frame_skip,
-                                self._input.framerate,
-                            )
-                            assert event_end.frame_num >= event_start.frame_num
-                        else:
-                            event_end = FrameTimecode(
-                                (last_frame_above_threshold_ms / 1000)
-                                + self._post_event_len.get_seconds(),
-                                self._input.framerate,
-                            )
-                            assert event_end.get_seconds() >= event_start.get_seconds()
-                        event_list.append(MotionEvent(start=event_start, end=event_end))
+                        event_list.append(event)
                         if self._output_mode != OutputMode.SCAN_ONLY:
                             encode_queue.put(MotionEvent(start=event_start, end=event_end))
-
                 # Send frame to encode thread.
                 if in_motion_event and self._output_mode == OutputMode.OPENCV:
-                    encode_queue.put(
-                        EncodeFrameEvent(
-                            frame_bgr=frame.frame_bgr,
-                            timecode=frame.timecode,
-                            bounding_box=bounding_box,
-                            score=frame_score,
-                        )
+                    encode_frame = EncodeFrameEvent(
+                        frame_bgr=frame.frame_bgr,
+                        timecode=frame.timecode,
+                        bounding_box=bounding_box,
+                        score=frame_score,
                     )
+                    if num_frames_post_event < post_event_len:
+                        encode_queue.put(encode_frame)
+                    else:
+                        buffered_frames.append(encode_frame)
+
             # Not already in a motion event, look for a new one.
             else:
                 # Buffer the required amount of frames and overlay data until we find an event.
@@ -1017,6 +996,48 @@ def scan(self) -> Optional[DetectionResult]:
 
         return DetectionResult(event_list, frames_processed)
 
+    def _on_event_end(
+        self,
+        last_frame_above_threshold,
+        last_frame_above_threshold_ms,
+        event_start,
+    ) -> MotionEvent:
+        logger.debug("event %d high score %f" % (1 + self._num_events, self._highscore))
+        if self._thumbnails == "highscore":
+            video_name = get_filename(path=self._input.paths[0], include_extension=False)
+            output_path = (
+                self._comp_file
+                if self._comp_file
+                else OUTPUT_FILE_TEMPLATE.format(
+                    VIDEO_NAME=video_name,
+                    EVENT_NUMBER="%04d" % (1 + self._num_events),
+                    EXTENSION="jpg",
+                )
+            )
+            if self._output_dir:
+                output_path = os.path.join(self._output_dir, output_path)
+            cv2.imwrite(output_path, self._highframe)
+            self._highscore = 0
+            self._highframe = None
+
+        # Calculate event end based on the last frame we had with motion plus
+        # the post event length time. We also need to compensate for the number
+        # of frames that we skipped that could have had motion.
+        # We also add 1 to include the presentation duration of the last frame.
+        if not self._use_pts:
+            event_end = FrameTimecode(
+                1 + last_frame_above_threshold + self._time_post_event.frame_num + self._frame_skip,
+                self._input.framerate,
+            )
+            assert event_end.frame_num >= event_start.frame_num
+        else:
+            event_end = FrameTimecode(
+                (last_frame_above_threshold_ms / 1000) + self._time_post_event.get_seconds(),
+                self._input.framerate,
+            )
+            assert event_end.get_seconds() >= event_start.get_seconds()
+        return MotionEvent(start=event_start, end=event_end)
+
     def _decode_thread(self, decode_queue: queue.Queue):
         try:
             while not self._stop.is_set():

tests/test_scan_context.py

@@ -237,3 +237,21 @@ def test_start_duration(traffic_camera_video):
     event_list = [(event.start.frame_num, event.end.frame_num) for event in event_list]
     # The set duration should only cover the middle event.
     compare_event_lists(event_list, TRAFFIC_CAMERA_EVENTS[1:2], EVENT_FRAME_TOLERANCE)
+
+
+TRAFFIC_CAMERA_EVENTS_MERGE_WITHIN_TIME_BEFORE = [
+    (2, 149),
+    (306, 576),
+]
+
+
+def test_merge_within_time_before(traffic_camera_video):
+    """Test setting time_pre_event."""
+    scanner = MotionScanner([traffic_camera_video])
+    scanner.set_regions(regions=[TRAFFIC_CAMERA_ROI])
+    scanner.set_event_params(min_event_len=4, time_pre_event=52)
+    event_list = scanner.scan().event_list
+    event_list = [(event.start.frame_num, event.end.frame_num) for event in event_list]
+    compare_event_lists(
+        event_list, TRAFFIC_CAMERA_EVENTS_MERGE_WITHIN_TIME_BEFORE, EVENT_FRAME_TOLERANCE
+    )