Merge pull request #1283 from roflcoopter/feature/ffmpeg-mjpeg-transcode

fix(ffmpeg) auto transcode to h264 for mjpeg sources

Author: roflcoopter

docs/src/pages/components-explorer/components/ffmpeg/config.json

@@ -2377,9 +2377,9 @@
                   {
                     "type": "string",
                     "name": "codec",
-                    "description": "FFmpeg video encoder codec, eg <code>h264_nvenc</code>.",
+                    "description": "FFmpeg video encoder codec, eg <code>h264_nvenc</code>. Defaults to <code>copy</code> except for MJPEG streams where the default is <code>h264</code>.",
                     "optional": true,
-                    "default": "copy"
+                    "default": null
                   },
                   {
                     "type": "string",

docs/src/pages/components-explorer/components/gstreamer/config.json

@@ -2119,9 +2119,9 @@
                   {
                     "type": "string",
                     "name": "codec",
-                    "description": "FFmpeg video encoder codec, eg <code>h264_nvenc</code>.",
+                    "description": "FFmpeg video encoder codec, eg <code>h264_nvenc</code>. Defaults to <code>copy</code> except for MJPEG streams where the default is <code>h264</code>.",
                     "optional": true,
-                    "default": "copy"
+                    "default": null
                   },
                   {
                     "type": "string",

tests/components/ffmpeg/test_stream.py

@@ -1,4 +1,5 @@
 """FFmpeg stream tests."""
+
 from __future__ import annotations
 
 from contextlib import nullcontext
@@ -23,6 +24,7 @@
     CONFIG_PROTOCOL,
     CONFIG_RECORDER,
     CONFIG_RECORDER_AUDIO_CODEC,
+    CONFIG_RECORDER_CODEC,
     CONFIG_STREAM_FORMAT,
     CONFIG_SUBSTREAM,
     CONFIG_USERNAME,
@@ -40,12 +42,14 @@
 )
 from viseron.components.ffmpeg.stream import FFprobe, Stream
 from viseron.const import (
+    CAMERA_SEGMENT_DURATION,
     ENV_CUDA_SUPPORTED,
     ENV_JETSON_NANO,
     ENV_RASPBERRYPI3,
     ENV_RASPBERRYPI4,
 )
 from viseron.exceptions import StreamInformationError
+from viseron.helpers.validators import UNDEFINED
 
 from tests.common import MockCamera
 
@@ -71,19 +75,17 @@
 
 CONFIG_WITH_SUBSTREAM: dict[str, Any] = {
     **CONFIG,
-    **{
-        CONFIG_SUBSTREAM: {
-            CONFIG_PROTOCOL: DEFAULT_PROTOCOL,
-            CONFIG_STREAM_FORMAT: DEFAULT_STREAM_FORMAT,
-            CONFIG_PORT: 1234,
-            CONFIG_PATH: "/",
-            CONFIG_WIDTH: 1921,
-            CONFIG_HEIGHT: 1081,
-            CONFIG_FPS: 31,
-            CONFIG_CODEC: "h265",
-            CONFIG_AUDIO_CODEC: DEFAULT_AUDIO_CODEC,
-            CONFIG_PIX_FMT: "yuv420p",
-        },
+    CONFIG_SUBSTREAM: {
+        CONFIG_PROTOCOL: DEFAULT_PROTOCOL,
+        CONFIG_STREAM_FORMAT: DEFAULT_STREAM_FORMAT,
+        CONFIG_PORT: 1234,
+        CONFIG_PATH: "/",
+        CONFIG_WIDTH: 1921,
+        CONFIG_HEIGHT: 1081,
+        CONFIG_FPS: 31,
+        CONFIG_CODEC: "h265",
+        CONFIG_AUDIO_CODEC: DEFAULT_AUDIO_CODEC,
+        CONFIG_PIX_FMT: "yuv420p",
     },
 }
 
@@ -140,16 +142,20 @@ def test_init(
     ) -> None:
         """Test that the stream is correctly initialized."""
         mocked_camera = MockCamera(identifier="test_camera_identifier")
-        with raises, patch.object(
-            FFprobe, "stream_information", MagicMock(return_value=stream_information)
-        ) as mock_stream_information, patch.object(
-            Stream, "create_symlink", MagicMock()
+        with (
+            raises,
+            patch.object(
+                FFprobe,
+                "stream_information",
+                MagicMock(return_value=stream_information),
+            ) as mock_stream_information,
+            patch.object(Stream, "create_symlink", MagicMock()),
         ):
             stream = Stream(config, mocked_camera, "test_camera_identifier")
             assert mock_stream_information.call_count == (
                 2 if config.get(CONFIG_SUBSTREAM) else 1
             )
-            assert stream._camera == mocked_camera  # pylint: disable=protected-access
+            assert stream._camera == mocked_camera
             assert stream.width == expected_width
             assert stream.height == expected_height
             assert stream.fps == expected_fps
@@ -180,7 +186,7 @@ def test_get_decoder_codec(
             Stream, "__init__", MagicMock(spec=Stream, return_value=None)
         ):
             stream = Stream(config, mocked_camera, "test_camera_identifier")
-            stream._logger = MagicMock()  # pylint: disable=protected-access
+            stream._logger = MagicMock()
             assert stream.get_decoder_codec(config, stream_codec) == expected_cmd
 
     @pytest.mark.parametrize(
@@ -206,8 +212,8 @@ def test_get_encoder_audio_codec(
             Stream, "__init__", MagicMock(spec=Stream, return_value=None)
         ):
             stream = Stream(config, mocked_camera, "test_camera_identifier")
-            stream._logger = MagicMock()  # pylint: disable=protected-access
-            stream._config = config  # pylint: disable=protected-access
+            stream._logger = MagicMock()
+            stream._config = config
             assert (
                 stream.get_encoder_audio_codec(stream_audio_codec) == expected_audio_cmd
             )
@@ -235,7 +241,7 @@ def test_get_stream_url(self, username, password, expected_url) -> None:
             Stream, "__init__", MagicMock(spec=Stream, return_value=None)
         ):
             stream = Stream(CONFIG, mocked_camera, "test_camera_identifier")
-            stream._config = config  # pylint: disable=protected-access
+            stream._config = config
             assert stream.get_stream_url(config) == expected_url
 
     def test_get_stream_information(self):
@@ -245,10 +251,11 @@ def test_get_stream_information(self):
         config[CONFIG_CODEC] = DEFAULT_CODEC
         config[CONFIG_AUDIO_CODEC] = DEFAULT_AUDIO_CODEC
 
-        with patch.object(
-            Stream, "__init__", MagicMock(spec=Stream, return_value=None)
-        ), patch.object(
-            Stream, "get_stream_url", MagicMock(return_value="test_stream_url")
+        with (
+            patch.object(Stream, "__init__", MagicMock(spec=Stream, return_value=None)),
+            patch.object(
+                Stream, "get_stream_url", MagicMock(return_value="test_stream_url")
+            ),
         ):
             stream = Stream(config, mocked_camera, "test_camera_identifier")
             mock_ffprobe = MagicMock(spec=FFprobe)
@@ -259,9 +266,9 @@ def test_get_stream_information(self):
                 "h264",
                 "aac",
             )
-            stream._ffprobe = mock_ffprobe  # pylint: disable=protected-access
+            stream._ffprobe = mock_ffprobe
             mock_logger = MagicMock()
-            stream._logger = mock_logger  # pylint: disable=protected-access
+            stream._logger = mock_logger
 
             result = stream.get_stream_information(config)
 
@@ -282,10 +289,11 @@ def test_get_stream_information_missing_parameters(self):
         config[CONFIG_CODEC] = DEFAULT_CODEC
         config[CONFIG_AUDIO_CODEC] = DEFAULT_AUDIO_CODEC
 
-        with patch.object(
-            Stream, "__init__", MagicMock(spec=Stream, return_value=None)
-        ), patch.object(
-            Stream, "get_stream_url", MagicMock(return_value="test_stream_url")
+        with (
+            patch.object(Stream, "__init__", MagicMock(spec=Stream, return_value=None)),
+            patch.object(
+                Stream, "get_stream_url", MagicMock(return_value="test_stream_url")
+            ),
         ):
             stream = Stream(config, mocked_camera, "test_camera_identifier")
             mock_ffprobe = MagicMock(spec=FFprobe)
@@ -296,9 +304,9 @@ def test_get_stream_information_missing_parameters(self):
                 "h264",
                 "mp4",
             )
-            stream._ffprobe = mock_ffprobe  # pylint: disable=protected-access
+            stream._ffprobe = mock_ffprobe
             mock_logger = MagicMock()
-            stream._logger = mock_logger  # pylint: disable=protected-access
+            stream._logger = mock_logger
 
             with pytest.raises(StreamInformationError) as excinfo:
                 stream.get_stream_information(config)
@@ -308,3 +316,84 @@ def test_get_stream_information_missing_parameters(self):
             mock_ffprobe.stream_information.assert_called_once_with(
                 "test_stream_url", ANY
             )
+
+    @pytest.mark.parametrize(
+        "stream_format, codec, expected_args",
+        [
+            ("mjpeg", "h264", ["-tune", "zerolatency"]),
+            ("mjpeg", "hevc", []),
+            ("rtsp", None, []),
+            ("rtmp", None, []),
+        ],
+    )
+    def test_encoder_tuning_args(self, stream_format, codec, expected_args) -> None:
+        """Test that encoder tuning args are only added for MJPEG streams."""
+        with patch.object(
+            Stream, "__init__", MagicMock(spec=Stream, return_value=None)
+        ):
+            stream = Stream.__new__(Stream)
+            stream._config = {
+                CONFIG_STREAM_FORMAT: stream_format,
+                CONFIG_RECORDER: {
+                    CONFIG_RECORDER_CODEC: codec,
+                },
+            }
+            result = stream._encoder_tuning_args()
+            assert result == expected_args
+
+    @pytest.mark.parametrize(
+        "stream_format, expected_args",
+        [
+            (
+                "mjpeg",
+                [
+                    "-force_key_frames",
+                    f"expr:gte(t,n_forced*{CAMERA_SEGMENT_DURATION})",
+                ],
+            ),
+            ("rtsp", []),
+            ("rtmp", []),
+        ],
+    )
+    def test_force_keyframe_args(self, stream_format, expected_args) -> None:
+        """Test that force keyframe args are only added for MJPEG streams."""
+        with patch.object(
+            Stream, "__init__", MagicMock(spec=Stream, return_value=None)
+        ):
+            stream = Stream.__new__(Stream)
+            stream._config = {
+                CONFIG_STREAM_FORMAT: stream_format,
+            }
+            result = stream._force_keyframe_args()
+            assert result == expected_args
+
+    @pytest.mark.parametrize(
+        "recorder_codec, stream_format, expected_cmd",
+        [
+            # MJPEG with no user-set codec should auto-convert to h264
+            (UNDEFINED, "mjpeg", ["-c:v", "h264"]),
+            # Non-MJPEG with no user-set codec should copy
+            (UNDEFINED, "rtsp", ["-c:v", "copy"]),
+            (UNDEFINED, "rtmp", ["-c:v", "copy"]),
+            # User-set codec should always take precedence
+            ("libx264", "mjpeg", ["-c:v", "libx264"]),
+            ("h264_nvenc", "rtsp", ["-c:v", "h264_nvenc"]),
+            ("hevc", "rtmp", ["-c:v", "hevc"]),
+        ],
+    )
+    def test_get_encoder_codec(
+        self, recorder_codec, stream_format, expected_cmd
+    ) -> None:
+        """Test that the correct encoder codec is returned."""
+        with patch.object(
+            Stream, "__init__", MagicMock(spec=Stream, return_value=None)
+        ):
+            stream = Stream.__new__(Stream)
+            stream._logger = MagicMock()
+            stream._config = {
+                CONFIG_STREAM_FORMAT: stream_format,
+                CONFIG_RECORDER: {
+                    CONFIG_RECORDER_CODEC: recorder_codec,
+                },
+            }
+            assert stream.get_encoder_codec() == expected_cmd

viseron/components/ffmpeg/camera.py

@@ -26,6 +26,7 @@
 from viseron.helpers import escape_string, utcnow
 from viseron.helpers.logs import SensitiveInformationFilter
 from viseron.helpers.validators import (
+    UNDEFINED,
     CameraIdentifier,
     CoerceNoneToDict,
     Deprecated,
@@ -87,7 +88,6 @@
     DEFAULT_RECORD_ONLY,
     DEFAULT_RECORDER_AUDIO_CODEC,
     DEFAULT_RECORDER_AUDIO_FILTERS,
-    DEFAULT_RECORDER_CODEC,
     DEFAULT_RECORDER_HWACCEL_ARGS,
     DEFAULT_RECORDER_OUTPUT_ARGS,
     DEFAULT_RECORDER_VIDEO_FILTERS,
@@ -225,9 +225,9 @@ def get_default_hwaccel_args() -> list[str]:
         ): [str],
         vol.Optional(
             CONFIG_RECORDER_CODEC,
-            default=DEFAULT_RECORDER_CODEC,
+            default=UNDEFINED,
             description=DESC_RECORDER_CODEC,
-        ): str,
+        ): Maybe(str),
         vol.Optional(
             CONFIG_RECORDER_AUDIO_CODEC,
             default=DEFAULT_RECORDER_AUDIO_CODEC,

viseron/components/ffmpeg/const.py

@@ -153,15 +153,18 @@
 CONFIG_RECORDER_OUPTUT_ARGS = "output_args"
 
 DEFAULT_RECORDER_HWACCEL_ARGS: list[str] = []
-DEFAULT_RECORDER_CODEC = "copy"
 DEFAULT_RECORDER_AUDIO_CODEC = "unset"
 DEFAULT_RECORDER_VIDEO_FILTERS: list[str] = []
 DEFAULT_RECORDER_AUDIO_FILTERS: list[str] = []
 DEFAULT_RECORDER_OUTPUT_ARGS: list[str] = []
 DEFAULT_SEGMENTS_FOLDER = "/segments"
 
 DESC_RECORDER_HWACCEL_ARGS = "FFmpeg encoder hardware acceleration arguments."
-DESC_RECORDER_CODEC = "FFmpeg video encoder codec, eg <code>h264_nvenc</code>."
+DESC_RECORDER_CODEC = (
+    "FFmpeg video encoder codec, eg <code>h264_nvenc</code>. "
+    "Defaults to <code>copy</code> except for MJPEG streams where the default is "
+    "<code>h264</code>."
+)
 DESC_RECORDER_AUDIO_CODEC = (
     "FFmpeg audio encoder codec, eg <code>aac</code>.<br>"
     "If your source has audio and you want to remove it, set this to <code>null</code>."

viseron/components/ffmpeg/stream.py

@@ -19,6 +19,7 @@
 from viseron.exceptions import FFprobeError, FFprobeTimeout, StreamInformationError
 from viseron.helpers import escape_string
 from viseron.helpers.logs import LogPipe, UnhelpfullLogFilter
+from viseron.helpers.validators import UNDEFINED
 from viseron.watchdog.subprocess_watchdog import RestartablePopen
 
 from .const import (
@@ -313,6 +314,15 @@ def get_decoder_codec(
 
     def get_encoder_codec(self) -> list[str]:
         """Return encoder codec set in config."""
+        if self._config[CONFIG_RECORDER][CONFIG_RECORDER_CODEC] == UNDEFINED:
+            if self._config[CONFIG_STREAM_FORMAT] == "mjpeg":
+                self._logger.warning(
+                    "MJPEG stream detected. Defaulting to h264 encoder codec. "
+                    "Consider setting `codec: h264` under the cameras `recorder` "
+                    "section in your config.yaml to avoid this warning. "
+                )
+                return ["-c:v", "h264"]
+            return ["-c:v", "copy"]
         return ["-c:v", self._config[CONFIG_RECORDER][CONFIG_RECORDER_CODEC]]
 
     def stream_command(
@@ -390,6 +400,36 @@ def recorder_audio_filter_args(self) -> list[str] | list:
             ]
         return []
 
+    def _encoder_tuning_args(self) -> list[str]:
+        """Return encoder tuning args for MJPEG streams.
+
+        The libx264 encoder sometimes buffers frames when encoding MJPEG streams,
+        causing a delay in the output.
+        Using -tune zerolatency eliminates this buffering delay.
+        """
+        if self._config[CONFIG_STREAM_FORMAT] == "mjpeg" and self._config[
+            CONFIG_RECORDER
+        ][CONFIG_RECORDER_CODEC].lower() in [
+            "libx264",
+            "h264",
+        ]:
+            return ["-tune", "zerolatency"]
+        return []
+
+    def _force_keyframe_args(self) -> list[str]:
+        """Return force keyframe args for MJPEG streams.
+
+        The HLS muxer never triggers segment splits based on hls_time for some
+        MJPEG streams. Force keyframes at the segment
+        duration interval to ensure segments are created.
+        """
+        if self._config[CONFIG_STREAM_FORMAT] == "mjpeg":
+            return [
+                "-force_key_frames",
+                f"expr:gte(t,n_forced*{CAMERA_SEGMENT_DURATION})",
+            ]
+        return []
+
     def segment_args(self) -> list[str]:
         """Generate FFmpeg segment args."""
         return (
@@ -413,6 +453,8 @@ def segment_args(self) -> list[str]:
                 ),
             ]
             + self.get_encoder_codec()
+            + self._encoder_tuning_args()
+            + self._force_keyframe_args()
             + self.recorder_video_filter_args()
             + self.get_encoder_audio_codec(self._mainstream.audio_codec)
             + self.recorder_audio_filter_args()