✨(back) add celery task to copy BBB video recordings

To replace AWS lambda for video conversion, adding a Celery task that fetches
the videos from the Scalelite API, and uploads them to Scaleway S3.
This allow the Peertube pipeline to be used instead of the AWS-based pipeline.

Author: wilbrdt

CHANGELOG.md

@@ -8,9 +8,13 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+### Added
+
+- Add Celery task to fetch recording from Scalelite API
+
 ### Fixed
 
-- Restore download, transcription and shared media icons on the video player 
+- Restore download, transcription and shared media icons on the video player
 
 ## [5.6.0] - 2025-03-06
 

src/aws/lambda-convert/testfiles/bbb-video-template.html

@@ -41,4 +41,4 @@ <h2 class="visually-hidden">Chat Messages</h2>
 
     <footer>Recorded with <a href="http://bigbluebutton.org">BigBlueButton</a>.</footer>
   </body>
-</html>
+</html>

src/backend/marsha/bbb/api.py

@@ -10,6 +10,7 @@
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 
+from celery import chain
 import django_filters
 import jwt
 from rest_framework import filters, mixins, viewsets
@@ -38,7 +39,8 @@
 from marsha.core.api import APIViewMixin, BulkDestroyModelMixin, ObjectPkMixin
 from marsha.core.defaults import VOD_CONVERT
 from marsha.core.models import ADMINISTRATOR, INSTRUCTOR, Video
-from marsha.core.utils.convert_lambda_utils import invoke_lambda_convert
+from marsha.core.tasks.recording import copy_video_recording
+from marsha.core.tasks.video import launch_video_transcoding
 from marsha.core.utils.s3_utils import create_presigned_post
 from marsha.core.utils.time_utils import to_timestamp
 
@@ -763,7 +765,7 @@ def create_vod(self, request, pk=None, classroom_id=None):
         classroom_recording.vod = Video.objects.create(
             title=request.data.get("title"),
             playlist=classroom_recording.classroom.playlist,
-            transcode_pipeline=defaults.AWS_PIPELINE,
+            transcode_pipeline=defaults.PEERTUBE_PIPELINE,
         )
         classroom_recording.save()
 
@@ -774,12 +776,26 @@ def create_vod(self, request, pk=None, classroom_id=None):
         now = timezone.now()
         stamp = to_timestamp(now)
 
-        # we need an used url to convert the record in VOD
-        invoke_lambda_convert(
-            get_recording_url(record_id=classroom_recording.record_id),
-            classroom_recording.vod.get_source_s3_key(stamp=stamp),
+        if settings.TRANSCODING_CALLBACK_DOMAIN:
+            domain = settings.TRANSCODING_CALLBACK_DOMAIN
+        else:
+            domain = f"{request.scheme}://{request.get_host()}"
+
+        process_chain = chain(
+            copy_video_recording.si(
+                record_url=get_recording_url(record_id=classroom_recording.record_id),
+                video_pk=classroom_recording.vod.pk,
+                stamp=stamp,
+            ),
+            launch_video_transcoding.si(
+                video_pk=classroom_recording.vod.pk,
+                stamp=stamp,
+                domain=domain,
+            ),
         )
 
+        process_chain.delay()
+
         return Response(serializer.data, status=201)
 
     def destroy(self, request, *args, **kwargs):

src/backend/marsha/bbb/tests/api/classroom/recordings/test_create_vod.py

@@ -17,7 +17,6 @@
     UserAccessTokenFactory,
 )
 from marsha.core.tests.testing_utils import reload_urlconf
-from marsha.core.utils.time_utils import to_timestamp
 
 
 # We don't enforce arguments documentation in tests
@@ -46,7 +45,7 @@ def test_api_classroom_recording_create_anonymous(self):
         """An anonymous should not be able to convert a recording to a VOD."""
         recording = ClassroomRecordingFactory()
 
-        with mock.patch("marsha.bbb.api.invoke_lambda_convert"):
+        with mock.patch("marsha.bbb.api.chain"):
             response = self.client.post(
                 f"/api/classrooms/{recording.classroom.id}/recordings/{recording.id}/create-vod/",
             )
@@ -63,7 +62,7 @@ def test_api_classroom_recording_create_anonymous_unknown_recording(self):
         """An anonymous should not be able to convert an unknown recording to a VOD."""
         recording = ClassroomRecordingFactory()
 
-        with mock.patch("marsha.bbb.api.invoke_lambda_convert"):
+        with mock.patch("marsha.bbb.api.chain"):
             response = self.client.post(
                 f"/api/classrooms/{recording.classroom.id}"
                 f"/recordings/{recording.classroom.id}/create-vod/",
@@ -82,7 +81,7 @@ def test_api_classroom_recording_create_vod_student(self):
         recording = ClassroomRecordingFactory()
         jwt_token = StudentLtiTokenFactory(playlist=recording.classroom.playlist)
 
-        with mock.patch("marsha.bbb.api.invoke_lambda_convert"):
+        with mock.patch("marsha.bbb.api.chain"):
             response = self.client.post(
                 f"/api/classrooms/{recording.classroom.id}/recordings/{recording.id}/create-vod/",
                 HTTP_AUTHORIZATION=f"Bearer {jwt_token}",
@@ -163,12 +162,14 @@ def test_api_classroom_recording_create_vod_instructor_or_admin(self):
             status=200,
         )
 
-        with mock.patch(
-            "marsha.bbb.api.invoke_lambda_convert"
-        ) as mock_invoke_lambda_convert, mock.patch.object(
-            timezone, "now", return_value=now
-        ), self.assertNumQueries(
-            9
+        with (
+            mock.patch("marsha.bbb.api.chain") as mock_chain,
+            mock.patch("marsha.bbb.api.copy_video_recording.si") as mock_copy,
+            mock.patch(
+                "marsha.bbb.api.launch_video_transcoding.si"
+            ) as mock_transcoding,
+            mock.patch.object(timezone, "now", return_value=now),
+            self.assertNumQueries(9),
         ):
             response = self.client.post(
                 f"/api/classrooms/{recording.classroom.id}/recordings/{recording.id}/create-vod/",
@@ -177,7 +178,7 @@ def test_api_classroom_recording_create_vod_instructor_or_admin(self):
             )
 
         self.assertEqual(Video.objects.count(), 1)
-        self.assertEqual(Video.objects.first().transcode_pipeline, "AWS")
+        self.assertEqual(Video.objects.first().transcode_pipeline, "peertube")
         self.assertEqual(response.status_code, 201)
 
         recording.refresh_from_db()
@@ -202,13 +203,19 @@ def test_api_classroom_recording_create_vod_instructor_or_admin(self):
 
         self.assertEqual(Video.objects.first(), recording.vod)
         self.assertEqual(recording.vod.upload_state, PENDING)
-        mock_invoke_lambda_convert.assert_called_once_with(
-            (
+        stamp = str(int(now.timestamp()))
+        mock_copy.assert_called_once_with(
+            record_url=(
                 "https://10.7.7.1/presentation/"
                 "c62c9c205d37815befe1b75ae6ef5878d8da5bb6-1673282694493/meeting.mp4"
             ),
-            recording.vod.get_source_s3_key(stamp=to_timestamp(now)),
+            video_pk=recording.vod.pk,
+            stamp=stamp,
         )
+        mock_transcoding.assert_called_once_with(
+            video_pk=recording.vod.pk, stamp=stamp, domain="http://testserver"
+        )
+        mock_chain.assert_called_once_with(mock_copy(), mock_transcoding())
 
     def test_api_classroom_recording_create_vod_instructor_or_admin_unknown_recording(
         self,
@@ -225,9 +232,10 @@ def test_api_classroom_recording_create_vod_instructor_or_admin_unknown_recordin
 
         now = timezone.now()
 
-        with mock.patch.object(
-            timezone, "now", return_value=now
-        ), self.assertNumQueries(1):
+        with (
+            mock.patch.object(timezone, "now", return_value=now),
+            self.assertNumQueries(1),
+        ):
             response = self.client.post(
                 f"/api/classrooms/{recording.classroom.id}"
                 f"/recordings/{recording.classroom.id}/create-vod/",
@@ -251,7 +259,7 @@ def test_api_classroom_recording_create_vod_user_access_token(self):
         )
         jwt_token = UserAccessTokenFactory(user=organization_access.user)
 
-        with mock.patch("marsha.bbb.api.invoke_lambda_convert"):
+        with mock.patch("marsha.bbb.api.chain"):
             response = self.client.post(
                 f"/api/classrooms/{recording.classroom.id}/recordings/{recording.id}/create-vod/",
                 HTTP_AUTHORIZATION=f"Bearer {jwt_token}",
@@ -323,15 +331,15 @@ def test_api_classroom_recording_create_vod_user_access_token_organization_admin
             status=200,
         )
 
-        with mock.patch("marsha.bbb.api.invoke_lambda_convert"):
+        with mock.patch("marsha.bbb.api.chain"):
             response = self.client.post(
                 f"/api/classrooms/{recording.classroom.id}/recordings/{recording.id}/create-vod/",
                 HTTP_AUTHORIZATION=f"Bearer {jwt_token}",
             )
 
         self.assertEqual(response.status_code, 201)
         self.assertEqual(Video.objects.count(), 1)
-        self.assertEqual(Video.objects.first().transcode_pipeline, "AWS")
+        self.assertEqual(Video.objects.first().transcode_pipeline, "peertube")
 
     @responses.activate
     def test_api_classroom_recording_create_vod_from_standalone_site_no_consumer_site(
@@ -402,15 +410,15 @@ def test_api_classroom_recording_create_vod_from_standalone_site_no_consumer_sit
             status=200,
         )
 
-        with mock.patch("marsha.bbb.api.invoke_lambda_convert"):
+        with mock.patch("marsha.bbb.api.chain"):
             response = self.client.post(
                 f"/api/classrooms/{recording.classroom.id}/recordings/{recording.id}/create-vod/",
                 HTTP_AUTHORIZATION=f"Bearer {jwt_token}",
             )
 
         self.assertEqual(response.status_code, 201)
         self.assertEqual(Video.objects.count(), 1)
-        self.assertEqual(Video.objects.first().transcode_pipeline, "AWS")
+        self.assertEqual(Video.objects.first().transcode_pipeline, "peertube")
 
     def test_api_classroom_recording_create_vod_from_standalone_site_inactive_conversion(
         self,
@@ -430,7 +438,7 @@ def test_api_classroom_recording_create_vod_from_standalone_site_inactive_conver
         )
         jwt_token = UserAccessTokenFactory(user=organization_access.user)
 
-        with mock.patch("marsha.bbb.api.invoke_lambda_convert"):
+        with mock.patch("marsha.bbb.api.chain"):
             response = self.client.post(
                 f"/api/classrooms/{recording.classroom.id}/recordings/{recording.id}/create-vod/",
                 HTTP_AUTHORIZATION=f"Bearer {jwt_token}",
@@ -502,15 +510,15 @@ def test_api_classroom_recording_create_vod_user_access_token_playlist_admin(
             status=200,
         )
 
-        with mock.patch("marsha.bbb.api.invoke_lambda_convert"):
+        with mock.patch("marsha.bbb.api.chain"):
             response = self.client.post(
                 f"/api/classrooms/{recording.classroom.id}/recordings/{recording.id}/create-vod/",
                 HTTP_AUTHORIZATION=f"Bearer {jwt_token}",
             )
 
         self.assertEqual(response.status_code, 201)
         self.assertEqual(Video.objects.count(), 1)
-        self.assertEqual(Video.objects.first().transcode_pipeline, "AWS")
+        self.assertEqual(Video.objects.first().transcode_pipeline, "peertube")
 
     @responses.activate
     def test_api_classroom_recording_create_vod_user_access_token_playlist_instructor(
@@ -576,15 +584,15 @@ def test_api_classroom_recording_create_vod_user_access_token_playlist_instructo
             status=200,
         )
 
-        with mock.patch("marsha.bbb.api.invoke_lambda_convert"):
+        with mock.patch("marsha.bbb.api.chain"):
             response = self.client.post(
                 f"/api/classrooms/{recording.classroom.id}/recordings/{recording.id}/create-vod/",
                 HTTP_AUTHORIZATION=f"Bearer {jwt_token}",
             )
 
         self.assertEqual(response.status_code, 201)
         self.assertEqual(Video.objects.count(), 1)
-        self.assertEqual(Video.objects.first().transcode_pipeline, "AWS")
+        self.assertEqual(Video.objects.first().transcode_pipeline, "peertube")
 
     def test_api_classroom_recording_create_vod_user_access_token_playlist_student(
         self,
@@ -617,7 +625,7 @@ def test_api_classroom_recording_create_vod_user_access_token_other_playlist_adm
         classroom_other = ClassroomFactory(playlist=playlist_access.playlist)
         jwt_token = UserAccessTokenFactory(user=playlist_access.user)
 
-        with mock.patch("marsha.bbb.api.invoke_lambda_convert"):
+        with mock.patch("marsha.bbb.api.chain"):
             response = self.client.post(
                 f"/api/classrooms/{classroom_other.id}/recordings/{recording.id}/create-vod/",
                 HTTP_AUTHORIZATION=f"Bearer {jwt_token}",
@@ -639,7 +647,7 @@ def test_api_classroom_recording_create_vod_admin_other_playlist_access(
         )
         jwt_token = UserAccessTokenFactory(user=other_playlist_access.user)
 
-        with mock.patch("marsha.bbb.api.invoke_lambda_convert"):
+        with mock.patch("marsha.bbb.api.chain"):
             response = self.client.post(
                 f"/api/classrooms/{recording.classroom.id}/recordings/{recording.id}/create-vod/",
                 HTTP_AUTHORIZATION=f"Bearer {jwt_token}",
@@ -666,12 +674,10 @@ def test_api_classroom_recording_create_vod_instructor_or_admin_inactive_convers
 
         now = timezone.now()
 
-        with mock.patch(
-            "marsha.bbb.api.invoke_lambda_convert"
-        ) as mock_invoke_lambda_convert, mock.patch.object(
-            timezone, "now", return_value=now
-        ), self.assertNumQueries(
-            1
+        with (
+            mock.patch("marsha.bbb.api.chain") as mock_chain,
+            mock.patch.object(timezone, "now", return_value=now),
+            self.assertNumQueries(1),
         ):
             response = self.client.post(
                 f"/api/classrooms/{recording.classroom.id}/recordings/{recording.id}/create-vod/",
@@ -688,4 +694,4 @@ def test_api_classroom_recording_create_vod_instructor_or_admin_inactive_convers
             {"error": "VOD conversion is disabled."},
         )
         self.assertEqual(Video.objects.count(), 0)
-        mock_invoke_lambda_convert.assert_not_called()
+        mock_chain.assert_not_called()

src/backend/marsha/core/defaults.py

@@ -73,8 +73,12 @@
 
 # UPLOAD ERROR REASONS
 MAX_RESOLUTION_EXCEDEED = "max_resolution_excedeed"
+RECORDING_SOURCE_ERROR = "recording_source_error"
 
-UPLOAD_ERROR_REASON_CHOICES = ((MAX_RESOLUTION_EXCEDEED, _("max_resolution_excedeed")),)
+UPLOAD_ERROR_REASON_CHOICES = (
+    (MAX_RESOLUTION_EXCEDEED, _("max_resolution_excedeed")),
+    (RECORDING_SOURCE_ERROR, _("recording_source_error")),
+)
 
 # LIVE TYPE
 (RAW, JITSI) = ("raw", "jitsi")