[backends] Land initial VFR support for PyAV

This required extensive API changes to the SceneDetector interface and SceneManager.

Author: Breakthrough

scenedetect/_cli/commands.py

@@ -29,7 +29,7 @@
 import scenedetect
 from scenedetect._cli.config import XmlFormat
 from scenedetect._cli.context import CliContext
-from scenedetect.frame_timecode import FrameTimecode
+from scenedetect.common import FrameTimecode
 from scenedetect.platform import get_and_create_path
 from scenedetect.scene_manager import (
     CutList,

scenedetect/_cli/config.py

@@ -25,9 +25,9 @@
 
 from platformdirs import user_config_dir
 
+from scenedetect.common import FrameTimecode
 from scenedetect.detector import FlashFilter
 from scenedetect.detectors import ContentDetector
-from scenedetect.frame_timecode import FrameTimecode
 from scenedetect.scene_manager import Interpolation
 from scenedetect.video_splitter import DEFAULT_FFMPEG_ARGS
 

scenedetect/_cli/context.py

@@ -24,6 +24,7 @@
     ConfigRegistry,
     CropValue,
 )
+from scenedetect.common import MAX_FPS_DELTA, FrameTimecode
 from scenedetect.detector import FlashFilter, SceneDetector
 from scenedetect.detectors import (
     AdaptiveDetector,
@@ -32,7 +33,6 @@
     HistogramDetector,
     ThresholdDetector,
 )
-from scenedetect.frame_timecode import MAX_FPS_DELTA, FrameTimecode
 from scenedetect.platform import init_logger
 from scenedetect.scene_manager import Interpolation, SceneManager
 from scenedetect.stats_manager import StatsManager

scenedetect/_cli/controller.py

@@ -20,7 +20,7 @@
 
 from scenedetect._cli.context import CliContext
 from scenedetect.backends import VideoStreamCv2, VideoStreamMoviePy
-from scenedetect.frame_timecode import FrameTimecode
+from scenedetect.common import FrameTimecode
 from scenedetect.platform import get_and_create_path
 from scenedetect.scene_manager import CutList, SceneList, get_scenes_from_cuts
 from scenedetect.video_stream import SeekError

scenedetect/backends/moviepy.py

@@ -24,7 +24,7 @@
 from moviepy.video.io.ffmpeg_reader import FFMPEG_VideoReader
 
 from scenedetect.backends.opencv import VideoStreamCv2
-from scenedetect.frame_timecode import FrameTimecode
+from scenedetect.common import FrameTimecode
 from scenedetect.platform import get_file_name
 from scenedetect.video_stream import SeekError, VideoOpenFailure, VideoStream
 

scenedetect/backends/opencv.py

@@ -26,8 +26,7 @@
 import cv2
 import numpy as np
 
-from scenedetect.common import Timecode
-from scenedetect.frame_timecode import MAX_FPS_DELTA, FrameTimecode
+from scenedetect.common import MAX_FPS_DELTA, FrameTimecode, Timecode, _USE_PTS_IN_DEVELOPMENT
 from scenedetect.platform import get_file_name
 from scenedetect.video_stream import (
     FrameRateUnavailable,
@@ -46,8 +45,6 @@
     " ! ",  # gstreamer pipe
 )
 
-_USE_PTS_IN_DEVELOPMENT = False
-
 
 def _get_aspect_ratio(cap: cv2.VideoCapture, epsilon: float = 0.0001) -> float:
     """Display/pixel aspect ratio of the VideoCapture as a float (1.0 represents square pixels)."""

scenedetect/backends/pyav.py

@@ -17,17 +17,14 @@
 import av
 import numpy as np
 
-from scenedetect.common import Timecode
-from scenedetect.frame_timecode import MAX_FPS_DELTA, FrameTimecode
+from scenedetect.common import MAX_FPS_DELTA, FrameTimecode, Timecode, _USE_PTS_IN_DEVELOPMENT
 from scenedetect.platform import get_file_name
 from scenedetect.video_stream import FrameRateUnavailable, VideoOpenFailure, VideoStream
 
 logger = getLogger("pyscenedetect")
 
 VALID_THREAD_MODES = ["NONE", "SLICE", "FRAME", "AUTO"]
 
-_USE_PTS_IN_DEVELOPMENT = False
-
 
 class VideoStreamAv(VideoStream):
     """PyAV `av.InputContainer` backend."""
@@ -205,7 +202,12 @@ def frame_number(self) -> int:
         """Current position within stream as the frame number.
 
         Will return 0 until the first frame is `read`."""
+
         if self._frame:
+            if _USE_PTS_IN_DEVELOPMENT:
+                return FrameTimecode(
+                    round(self._frame.time * self.frame_rate), self.frame_rate
+                ).frame_num
             return self.position.frame_num + 1
         return 0
 

scenedetect/common.py

@@ -66,6 +66,9 @@
 from dataclasses import dataclass
 from fractions import Fraction
 
+
+_USE_PTS_IN_DEVELOPMENT = False
+
 ##
 ## Type Aliases
 ##

scenedetect/detector.py

@@ -29,6 +29,7 @@
 
 import numpy
 
+from scenedetect.common import FrameTimecode, _USE_PTS_IN_DEVELOPMENT
 from scenedetect.stats_manager import StatsManager
 
 
@@ -46,6 +47,7 @@ class SceneDetector:
 
     # TODO(v0.7): Make this a proper abstract base class.
 
+    # TODO(v0.7): This should be a property.
     stats_manager: ty.Optional[StatsManager] = None
     """Optional :class:`StatsManager <scenedetect.stats_manager.StatsManager>` to
     use for caching frame metrics to and from."""
@@ -67,7 +69,9 @@ def get_metrics(self) -> ty.List[str]:
         """
         return []
 
-    def process_frame(self, frame_num: int, frame_img: numpy.ndarray) -> ty.List[int]:
+    def process_frame(
+        self, timecode: FrameTimecode, frame_img: numpy.ndarray
+    ) -> ty.List[FrameTimecode]:
         """Process the next frame. `frame_num` is assumed to be sequential.
 
         Args:
@@ -84,7 +88,7 @@ def process_frame(self, frame_num: int, frame_img: numpy.ndarray) -> ty.List[int
         """
         return []
 
-    def post_process(self, frame_num: int) -> ty.List[int]:
+    def post_process(self, timecode: int) -> ty.List[FrameTimecode]:
         """Post Process: Performs any processing after the last frame has been read.
 
         Prototype method, no actual detection.
@@ -130,15 +134,17 @@ def __init__(self, mode: Mode, length: int):
     def max_behind(self) -> int:
         return 0 if self._mode == FlashFilter.Mode.SUPPRESS else self._filter_length
 
-    def filter(self, frame_num: int, above_threshold: bool) -> ty.List[int]:
+    def filter(self, timecode: FrameTimecode, above_threshold: bool) -> ty.List[FrameTimecode]:
         if not self._filter_length > 0:
-            return [frame_num] if above_threshold else []
+            return [timecode] if above_threshold else []
+        if _USE_PTS_IN_DEVELOPMENT:
+            raise NotImplementedError("TODO: Change filter to use units of time instead of frames.")
         if self._last_above is None:
-            self._last_above = frame_num
+            self._last_above = timecode
         if self._mode == FlashFilter.Mode.MERGE:
-            return self._filter_merge(frame_num=frame_num, above_threshold=above_threshold)
+            return self._filter_merge(frame_num=timecode, above_threshold=above_threshold)
         elif self._mode == FlashFilter.Mode.SUPPRESS:
-            return self._filter_suppress(frame_num=frame_num, above_threshold=above_threshold)
+            return self._filter_suppress(frame_num=timecode, above_threshold=above_threshold)
         raise RuntimeError("Unhandled FlashFilter mode.")
 
     def _filter_suppress(self, frame_num: int, above_threshold: bool) -> ty.List[int]:

scenedetect/detectors/adaptive_detector.py

@@ -21,6 +21,7 @@
 
 import numpy as np
 
+from scenedetect.common import FrameTimecode
 from scenedetect.detectors import ContentDetector
 
 logger = getLogger("pyscenedetect")
@@ -109,7 +110,9 @@ def stats_manager_required(self) -> bool:
         """Not required for AdaptiveDetector."""
         return False
 
-    def process_frame(self, frame_num: int, frame_img: ty.Optional[np.ndarray]) -> ty.List[int]:
+    def process_frame(
+        self, timecode: FrameTimecode, frame_img: ty.Optional[np.ndarray]
+    ) -> ty.List[int]:
         """Process the next frame. `frame_num` is assumed to be sequential.
 
         Args:
@@ -124,14 +127,14 @@ def process_frame(self, frame_num: int, frame_img: ty.Optional[np.ndarray]) -> t
 
         # TODO(#283): Merge this with ContentDetector and turn it on by default.
 
-        super().process_frame(frame_num=frame_num, frame_img=frame_img)
+        super().process_frame(timecode=timecode, frame_img=frame_img)
 
         # Initialize last scene cut point at the beginning of the frames of interest.
         if self._last_cut is None:
-            self._last_cut = frame_num
+            self._last_cut = timecode
 
         required_frames = 1 + (2 * self.window_width)
-        self._buffer.append((frame_num, self._frame_score))
+        self._buffer.append((timecode, self._frame_score))
         if not len(self._buffer) >= required_frames:
             return []
         self._buffer = self._buffer[-required_frames:]
@@ -156,7 +159,7 @@ def process_frame(self, frame_num: int, frame_img: ty.Optional[np.ndarray]) -> t
         threshold_met: bool = (
             adaptive_ratio >= self.adaptive_threshold and target_score >= self.min_content_val
         )
-        min_length_met: bool = (frame_num - self._last_cut) >= self.min_scene_len
+        min_length_met: bool = (timecode - self._last_cut) >= self.min_scene_len
         if threshold_met and min_length_met:
             self._last_cut = target_frame
             return [target_frame]