🚚(back) serve legacy shared live medias from Scaleway S3

Shared live medias are now served from the aws/ directory in Scaleway S3.
They are served using the django-storage already in place.
As the newer shared live medias are already stored in Scaleway, we do not
rename these files and allow them to be served without requiring signed URLs.

Author: wilbrdt

src/backend/marsha/core/models/video.py

@@ -1261,44 +1261,6 @@ class Meta:
         verbose_name = _("shared live media")
         verbose_name_plural = _("shared live medias")
 
-    def get_source_s3_key(self, stamp=None, extension=None):
-        """Compute the S3 key in the source bucket.
-
-        It is built from the video ID + ID of the shared live media + version stamp.
-
-        Parameters
-        ----------
-        stamp: Type[string]
-            Passing a value for this argument will return the source S3 key for the shared live
-            media assuming its active stamp is set to this value. This is useful to create an
-            upload policy for this prospective version of the track, so that the client can
-            upload the file to S3 and the confirmation lambda can set the `uploaded_on` field
-            to this value only after the file upload and processing is successful.
-
-
-        extension: Type[string]
-            The extension used by the uploaded media. This extension is added at the end of the key
-            to keep a record of the extension. We will use it in the update-state endpoint to
-            record it in the database.
-
-        Returns
-        -------
-        string
-            The S3 key for the shared live media in the source bucket, where uploaded files are
-            stored before they are converted and copied to the destination bucket.
-
-        """
-        # We don't want to deal with None value so we set it with an empty string
-        extension = extension or ""
-
-        # We check if the extension starts with a leading dot or not. If it's not the case we add
-        # it at the beginning of the string
-        if extension and not extension.startswith("."):
-            extension = "." + extension
-
-        stamp = stamp or self.uploaded_on_stamp()
-        return f"{self.video.pk}/sharedlivemedia/{self.pk}/{stamp}{extension}"
-
     def get_storage_prefix(
         self,
         stamp=None,

src/backend/marsha/core/serializers/shared_live_media.py

@@ -1,20 +1,15 @@
 """Structure of SharedLiveMedia related models API responses with DRF serializers."""
 
-from urllib.parse import quote_plus
-
-from django.conf import settings
-
 from rest_framework import serializers
 
-from marsha.core.defaults import CELERY_PIPELINE
+from marsha.core.defaults import AWS_STORAGE_BASE_DIRECTORY, CELERY_PIPELINE
 from marsha.core.models import SharedLiveMedia
 from marsha.core.serializers.base import (
     TimestampField,
     UploadableFileWithExtensionSerializerMixin,
-    get_video_cloudfront_url_params,
 )
 from marsha.core.storage.storage_class import file_storage
-from marsha.core.utils import cloudfront_utils, time_utils
+from marsha.core.utils import time_utils
 
 
 class SharedLiveMediaSerializer(
@@ -100,33 +95,20 @@ def get_aws_pipeline_url(self, obj):
         urls = {}
 
         stamp = time_utils.to_timestamp(obj.uploaded_on)
-        base = (
-            f"{settings.AWS_S3_URL_PROTOCOL}://{settings.CLOUDFRONT_DOMAIN}/"
-            f"{obj.video_id}/sharedlivemedia/{obj.pk}/{stamp}"
-        )
 
-        if settings.CLOUDFRONT_SIGNED_URLS_ACTIVE:
-            params = get_video_cloudfront_url_params(obj.video_id)
+        base = obj.get_storage_prefix(stamp=stamp, base_dir=AWS_STORAGE_BASE_DIRECTORY)
 
         pages = {}
         for page_number in range(1, obj.nb_pages + 1):
-            url = f"{base:s}_{page_number:d}.svg"
-            if settings.CLOUDFRONT_SIGNED_URLS_ACTIVE:
-                url = cloudfront_utils.build_signed_url(url, params)
-            pages[page_number] = url
+            url = f"{base}_{page_number}.svg"
+            pages[page_number] = file_storage.url(url)
 
         urls["pages"] = pages
 
-        # Downloadable link can be generated only when cloudfront request is signed
-        if (
-            self.context.get("is_admin") or obj.show_download
-        ) and settings.CLOUDFRONT_SIGNED_URLS_ACTIVE:
+        if self.context.get("is_admin") or obj.show_download:
             extension = f".{obj.extension}" if obj.extension else ""
-            urls["media"] = cloudfront_utils.build_signed_url(
-                f"{base}{extension}?response-content-disposition="
-                f"{quote_plus('attachment; filename=' + self.get_filename(obj))}",
-                params,
-            )
+            url = f"{base}{extension}"
+            urls["media"] = file_storage.url(url)
 
         return urls
 

src/backend/marsha/core/tests/api/shared_live_media/test_list.py

@@ -72,6 +72,9 @@ def test_api_shared_live_media_list_student(self):
 
         self.assertEqual(response.status_code, 403)
 
+    @override_settings(
+        MEDIA_URL="https://abc.svc.edge.scw.cloud/",
+    )
     def test_api_shared_live_media_list_instructor(self):
         """An instructor can list shared live media details."""
         video = VideoFactory()
@@ -136,20 +139,24 @@ def test_api_shared_live_media_list_instructor(self):
                         "title": "python expressions",
                         "upload_state": "ready",
                         "urls": {
+                            "media": (
+                                f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
+                                f"sharedlivemedia/{shared_live_media2.id}/1638230400.pdf"
+                            ),
                             "pages": {
                                 "1": (
-                                    f"https://abc.cloudfront.net/{video.id}/"
+                                    f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
                                     f"sharedlivemedia/{shared_live_media2.id}/1638230400_1.svg"
                                 ),
                                 "2": (
-                                    f"https://abc.cloudfront.net/{video.id}/"
+                                    f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
                                     f"sharedlivemedia/{shared_live_media2.id}/1638230400_2.svg"
                                 ),
                                 "3": (
-                                    f"https://abc.cloudfront.net/{video.id}/"
+                                    f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
                                     f"sharedlivemedia/{shared_live_media2.id}/1638230400_3.svg"
                                 ),
-                            }
+                            },
                         },
                         "video": str(video.id),
                     },
@@ -198,10 +205,9 @@ def test_api_shared_live_media_list_instructor_other_video(self):
         )
 
     @override_settings(
-        CLOUDFRONT_SIGNED_URLS_ACTIVE=True,
-        CLOUDFRONT_SIGNED_PUBLIC_KEY_ID="cloudfront-access-key-id",
+        MEDIA_URL="https://abc.svc.edge.scw.cloud/",
     )
-    def test_api_shared_live_media_list_instructor_ready_to_show_and_signed_url_active(
+    def test_api_shared_live_media_list_instructor_ready_to_show(
         self,
     ):
         """An instructor can list shared live media details ready to show and signed url on."""
@@ -236,8 +242,11 @@ def test_api_shared_live_media_list_instructor_ready_to_show_and_signed_url_acti
 
         # fix the time so that the url signature is deterministic and can be checked
         now = datetime(2021, 11, 30, tzinfo=baseTimezone.utc)
-        with mock.patch.object(timezone, "now", return_value=now), mock.patch(
-            "builtins.open", new_callable=mock.mock_open, read_data=RSA_KEY_MOCK
+        with (
+            mock.patch.object(timezone, "now", return_value=now),
+            mock.patch(
+                "builtins.open", new_callable=mock.mock_open, read_data=RSA_KEY_MOCK
+            ),
         ):
             response = self.client.get(
                 self._get_url(video),
@@ -246,19 +255,6 @@ def test_api_shared_live_media_list_instructor_ready_to_show_and_signed_url_acti
 
         self.assertEqual(response.status_code, 200)
 
-        expected_cloudfront_signature = (
-            "Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0"
-            "cHM6Ly9hYmMuY2xvdWRmcm9udC5uZXQvZDlkNzA0OWMtNWEzZi00MDcwLWE0OTQt"
-            "ZTZiZjBiZDhiOWZiLyoiLCJDb25kaXRpb24iOnsiRGF0ZUxlc3NUaGFuIjp7IkFX"
-            "UzpFcG9jaFRpbWUiOjE2MzgyMzc2MDB9fX1dfQ__&Signature=IVWMFfS7WQVTK"
-            "LZl~gKgGES~BS~wVLBIOncSE6yVgg9zIrEI1Epq3AVkOsI7z10dyjgInNbPviArn"
-            "xmlV~DQeN-ykgEWmGy7aT4lRCx61oXuHFtNkq8Qx-we~UY87mZ4~UTqmM~JVuuLd"
-            "uMiRQB-I3XKaRQGRlsok5yGu0RhvLcZntVFp6QgYui3WtGvxSs2LjW0IakR1qepS"
-            "Dl9LXI-F2bgl9Vd1U9eapPBhhoD0okebXm7NGg9gUMLXlmUo-RvsrAzzEteKctPp"
-            "0Xzkydk~tcnMkJs4jfbQxKrpyF~N9OuCRYCs68ONhHvypOYU3K-wQEoAFlERBLia"
-            "OzDZUzlyA__&Key-Pair-Id=cloudfront-access-key-id"
-        )
-
         content = json.loads(response.content)
         self.assertEqual(
             content,
@@ -278,26 +274,21 @@ def test_api_shared_live_media_list_instructor_ready_to_show_and_signed_url_acti
                         "upload_state": "ready",
                         "urls": {
                             "media": (
-                                "https://abc.cloudfront.net/d9d7049c-5a3f-4070-a494-e6bf0bd8b9fb/"
-                                "sharedlivemedia/7520c16b-5846-41ca-822b-52b446a96809/1638230400."
-                                "pdf?response-content-disposition=attachment%3B+filename%3Dpython"
-                                f"-expressions.pdf&{expected_cloudfront_signature}"
+                                f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
+                                f"sharedlivemedia/{shared_live_media.id}/1638230400.pdf"
                             ),
                             "pages": {
                                 "1": (
-                                    "https://abc.cloudfront.net/d9d7049c-5a3f-4070-a494-e6bf0bd8b9"
-                                    "fb/sharedlivemedia/7520c16b-5846-41ca-822b-52b446a96809/16382"
-                                    f"30400_1.svg?{expected_cloudfront_signature}"
+                                    f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
+                                    f"sharedlivemedia/{shared_live_media.id}/1638230400_1.svg"
                                 ),
                                 "2": (
-                                    "https://abc.cloudfront.net/d9d7049c-5a3f-4070-a494-e6bf0bd8b9"
-                                    "fb/sharedlivemedia/7520c16b-5846-41ca-822b-52b446a96809/16382"
-                                    f"30400_2.svg?{expected_cloudfront_signature}"
+                                    f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
+                                    f"sharedlivemedia/{shared_live_media.id}/1638230400_2.svg"
                                 ),
                                 "3": (
-                                    "https://abc.cloudfront.net/d9d7049c-5a3f-4070-a494-e6bf0bd8b9"
-                                    "fb/sharedlivemedia/7520c16b-5846-41ca-822b-52b446a96809/16382"
-                                    f"30400_3.svg?{expected_cloudfront_signature}"
+                                    f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
+                                    f"sharedlivemedia/{shared_live_media.id}/1638230400_3.svg"
                                 ),
                             },
                         },
@@ -346,6 +337,9 @@ def test_api_shared_live_media_list_by_user_with_no_access(self):
 
         self.assertEqual(response.status_code, 403)
 
+    @override_settings(
+        MEDIA_URL="https://abc.svc.edge.scw.cloud/",
+    )
     def test_api_shared_live_media_list_by_video_playlist_admin(self):
         """
         Playlist administrator token user list shared live medias for a video.
@@ -418,17 +412,21 @@ def test_api_shared_live_media_list_by_video_playlist_admin(self):
                         "title": "python extensions",
                         "upload_state": "ready",
                         "urls": {
+                            "media": (
+                                f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
+                                f"sharedlivemedia/{shared_live_media2.id}/1638230400.pdf"
+                            ),
                             "pages": {
                                 "1": (
-                                    f"https://abc.cloudfront.net/{video.id}/"
+                                    f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
                                     f"sharedlivemedia/{shared_live_media2.id}/1638230400_1.svg"
                                 ),
                                 "2": (
-                                    f"https://abc.cloudfront.net/{video.id}/"
+                                    f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
                                     f"sharedlivemedia/{shared_live_media2.id}/1638230400_2.svg"
                                 ),
                                 "3": (
-                                    f"https://abc.cloudfront.net/{video.id}/"
+                                    f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
                                     f"sharedlivemedia/{shared_live_media2.id}/1638230400_3.svg"
                                 ),
                             },
@@ -503,6 +501,9 @@ def test_api_shared_live_media_list_by_video_organization_instructor(self):
 
         self.assertEqual(response.status_code, 403)
 
+    @override_settings(
+        MEDIA_URL="https://abc.svc.edge.scw.cloud/",
+    )
     def test_api_shared_live_media_list_by_video_organization_admin(self):
         """
         Organization administrator token user list shared live medias for a video.
@@ -578,20 +579,24 @@ def test_api_shared_live_media_list_by_video_organization_admin(self):
                         "title": "python extensions",
                         "upload_state": "ready",
                         "urls": {
+                            "media": (
+                                f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
+                                f"sharedlivemedia/{shared_live_media2.id}/1638230400.pdf"
+                            ),
                             "pages": {
                                 "1": (
-                                    f"https://abc.cloudfront.net/{video.id}/"
+                                    f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
                                     f"sharedlivemedia/{shared_live_media2.id}/1638230400_1.svg"
                                 ),
                                 "2": (
-                                    f"https://abc.cloudfront.net/{video.id}/"
+                                    f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
                                     f"sharedlivemedia/{shared_live_media2.id}/1638230400_2.svg"
                                 ),
                                 "3": (
-                                    f"https://abc.cloudfront.net/{video.id}/"
+                                    f"https://abc.svc.edge.scw.cloud/aws/{video.id}/"
                                     f"sharedlivemedia/{shared_live_media2.id}/1638230400_3.svg"
                                 ),
-                            }
+                            },
                         },
                         "video": str(video.id),
                     },