✨(back) cleanup transcoding temp files

Once a video transcoded we want to delete the first transcoded video
that is using a temporary file. Also a management command cleaning old
temporary transcoded video is added

Author: kernicPanel

CHANGELOG.md

@@ -8,6 +8,10 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+### Added
+
+- Cleanup transcoding temp files
+
 ## [5.2.0] - 2024-10-22
 
 ### Added

src/backend/marsha/core/management/commands/delete_transcoding_temp_files.py

@@ -0,0 +1,20 @@
+"""Management command to delete transcoding temp files."""
+
+import logging
+
+from django.core.management import BaseCommand
+
+from marsha.core.utils.transcode import delete_transcoding_temp_files
+
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """Delete transcoding temp files."""
+
+    help = "Delete transcoding temp files"
+
+    def handle(self, *args, **options):
+        """Delete all transcoding temp files."""
+        delete_transcoding_temp_files()

src/backend/marsha/core/tests/management_commands/test_delete_transcoding_temp_files.py

@@ -0,0 +1,23 @@
+"""Test transcript_video command."""
+
+from unittest.mock import patch
+
+from django.core.management import call_command
+from django.test import TestCase
+
+from marsha.core.utils import transcode
+
+
+@patch.object(transcode, "delete_transcoding_temp_files")
+class TranscriptVideosTestCase(TestCase):
+    """
+    Test case for the transcript_videos command.
+    """
+
+    maxDiff = None
+
+    def test_delete_transcoding_temp_files(self, mock_delete_temp_files):
+        """Should call delete_transcoding_temp_files function."""
+        call_command("delete_transcoding_temp_files")
+
+        mock_delete_temp_files.assert_called_once()

src/backend/marsha/core/tests/utils/test_transcode.py

@@ -1,7 +1,13 @@
 """Tests for the `core.utils.transcode` module."""
 
+from unittest.mock import patch
+from uuid import uuid4
+
 from django.test import TestCase
 
+from django_peertube_runner_connector.factories import (
+    VideoFactory as TranscodedVideoFactory,
+)
 from django_peertube_runner_connector.models import (
     Video as TranscodedVideo,
     VideoFile,
@@ -12,11 +18,14 @@
 from marsha.core import defaults
 from marsha.core.factories import VideoFactory
 from marsha.core.utils.time_utils import to_datetime
-from marsha.core.utils.transcode import transcoding_ended_callback
+from marsha.core.utils.transcode import (
+    delete_transcoding_temp_files,
+    transcoding_ended_callback,
+)
 
 
-class TranscodeTestCase(TestCase):
-    """Test the `transcode` functions."""
+class TranscodingEndedCallbackTestCase(TestCase):
+    """Test the `transcoding_ended_callback` function."""
 
     def setUp(self):
         # Create a test video
@@ -31,6 +40,7 @@ def setUp(self):
             state=VideoState.PUBLISHED,
             directory=f"vod/{self.video.pk}/video/1698941501",
         )
+
         self.video_playlist = VideoStreamingPlaylist.objects.create(
             video=self.transcoded_video
         )
@@ -50,31 +60,59 @@ def setUp(self):
             extname="mp4",
         )
 
-    def test_transcoding_ended_callback(self):
+    @patch("marsha.core.utils.transcode.delete_temp_file")
+    def test_transcoding_ended_callback(self, mock_delete_temp_file):
         """The marsha video should correctly be updated."""
         transcoding_ended_callback(self.transcoded_video)
 
         self.video.refresh_from_db()
-
         self.assertEqual(self.video.uploaded_on, to_datetime("1698941501"))
-
         self.assertEqual(self.video.resolutions, [720, 1080])
-
         self.assertEqual(self.video.upload_state, defaults.READY)
         self.assertEqual(self.video.transcode_pipeline, defaults.PEERTUBE_PIPELINE)
+        mock_delete_temp_file.assert_called_once_with(
+            self.transcoded_video, f"tmp/{self.video.pk}/video/1698941501"
+        )
 
-    def test_transcoding_ended_callback_with_error(self):
+    @patch("marsha.core.utils.transcode.delete_temp_file")
+    def test_transcoding_ended_callback_with_error(self, mock_delete_temp_file):
         """The marsha video should be set with state on error and nothing else should be done."""
         self.transcoded_video.state = VideoState.TRANSCODING_FAILED
 
         transcoding_ended_callback(self.transcoded_video)
 
         self.video.refresh_from_db()
-
         self.assertEqual(self.video.upload_state, defaults.ERROR)
-
         self.assertEqual(self.video.uploaded_on, None)
-
         self.assertEqual(self.video.resolutions, [])
-
         self.assertEqual(self.video.transcode_pipeline, None)
+        mock_delete_temp_file.assert_called_once_with(
+            self.transcoded_video, f"tmp/{self.video.pk}/video/1698941501"
+        )
+
+
+class DeleteTranscodingTempFilesTestCase(TestCase):
+    """Test the `delete_transcoding_temp_files` function."""
+
+    @patch("marsha.core.utils.transcode.delete_temp_file")
+    def test_transcoding_delete_transcoding_temp_files(self, mock_delete_temp_file):
+        """The temp files should be deleted."""
+        video_id = uuid4()
+        video_file = TranscodedVideoFactory(
+            directory=f"vod/{video_id}/video/1698941501"
+        )
+
+        delete_transcoding_temp_files()
+
+        mock_delete_temp_file.assert_called_once_with(
+            video_file, f"tmp/{video_id}/video/1698941501"
+        )
+
+    @patch("marsha.core.utils.transcode.delete_temp_file")
+    def test_transcoding_delete_transcoding_temp_files_all(self, mock_delete_temp_file):
+        """All video files should be processed."""
+        TranscodedVideoFactory.create_batch(15)
+
+        delete_transcoding_temp_files()
+
+        self.assertEqual(mock_delete_temp_file.call_count, 15)

src/backend/marsha/core/utils/transcode.py

@@ -1,15 +1,22 @@
 """ Utils related to transcoding """
 
 from django_peertube_runner_connector.models import Video as TranscodedVideo, VideoState
-
-from marsha.core.defaults import ERROR, PEERTUBE_PIPELINE, READY
+from django_peertube_runner_connector.utils.files import delete_temp_file
+
+from marsha.core.defaults import (
+    ERROR,
+    PEERTUBE_PIPELINE,
+    READY,
+    TMP_VIDEOS_STORAGE_BASE_DIRECTORY,
+    VOD_VIDEOS_STORAGE_BASE_DIRECTORY,
+)
 from marsha.core.models.video import Video
 from marsha.core.utils.time_utils import to_datetime
 
 
 def transcoding_ended_callback(transcoded_video: TranscodedVideo):
     """
-    Callback used when the a Peertube runnner has finished
+    Callback used when a Peertube runner has finished
     to transcode a video.
 
     Parameters
@@ -23,6 +30,11 @@ def transcoding_ended_callback(transcoded_video: TranscodedVideo):
     uploaded_on = directory[-1]
     video_id = directory[-3]
     video = Video.objects.get(pk=video_id)
+    tmp_filename = transcoded_video.directory.replace(
+        VOD_VIDEOS_STORAGE_BASE_DIRECTORY, TMP_VIDEOS_STORAGE_BASE_DIRECTORY
+    )
+
+    delete_temp_file(transcoded_video, tmp_filename)
 
     if transcoded_video.state == VideoState.TRANSCODING_FAILED:
         video.update_upload_state(ERROR, None)
@@ -39,3 +51,12 @@ def transcoding_ended_callback(transcoded_video: TranscodedVideo):
         to_datetime(uploaded_on),
         **{"resolutions": resolutions},
     )
+
+
+def delete_transcoding_temp_files():
+    """Delete all transcoding temp files."""
+    for transcoded_video in TranscodedVideo.objects.all():
+        tmp_filename = transcoded_video.directory.replace(
+            VOD_VIDEOS_STORAGE_BASE_DIRECTORY, TMP_VIDEOS_STORAGE_BASE_DIRECTORY
+        )
+        delete_temp_file(transcoded_video, tmp_filename)