🚚(back) serve legacy classroom documents from Scaleway S3

wilbrdt · wilbrdt · commit 661550063de4 · 2025-07-01T10:36:23.000+02:00
Classroom documents are now served from the `aws/` directory in Scaleway S3.
They are served using the django-storage already in place.
As content headers cannot be set to Scaleway Edge services URLs, legacy
classroom documents need to be renamed to their filenames. Adding a management
command to do that.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,10 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+### Changed
+
+- Serve legacy classroom documents from Scaleway S3 after AWS migration
+
 ## [5.9.1] - 2025-06-25
 
 ### Fixed
diff --git a/src/backend/marsha/bbb/api.py b/src/backend/marsha/bbb/api.py
@@ -599,19 +599,19 @@ def initiate_upload(self, request, pk=None, classroom_id=None):
         """Get an upload policy for a classroom document.
 
         Calling the endpoint resets the upload state to `pending` and returns an upload policy to
-        our AWS S3 source bucket.
+        our S3 bucket.
 
         Parameters
         ----------
         request : Type[django.http.request.HttpRequest]
             The request on the API endpoint
         pk: string
-            The primary key of the shared live media
+            The primary key of the classroom document
 
         Returns
         -------
         Type[rest_framework.response.Response]
-            HttpResponse carrying the AWS S3 upload policy as a JSON object.
+            HttpResponse carrying the S3 upload policy as a JSON object.
 
         """
         classroom_document = self.get_object()  # check permissions first
diff --git a/src/backend/marsha/bbb/management/commands/rename_classroom_documents.py b/src/backend/marsha/bbb/management/commands/rename_classroom_documents.py
@@ -0,0 +1,84 @@
+"""Rename classroom documents in Scaleway S3 for serving them with the right name."""
+
+import logging
+from os.path import splitext
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+import boto3
+
+from marsha.bbb.models import ClassroomDocument
+from marsha.core.defaults import AWS_STORAGE_BASE_DIRECTORY
+from marsha.core.storage.storage_class import file_storage
+from marsha.core.utils import time_utils
+
+
+logger = logging.getLogger(__name__)
+
+scw_credentials = {
+    "aws_access_key_id": settings.STORAGE_S3_ACCESS_KEY,
+    "aws_secret_access_key": settings.STORAGE_S3_SECRET_KEY,
+    "region_name": settings.STORAGE_S3_REGION_NAME,
+    "endpoint_url": settings.STORAGE_S3_ENDPOINT_URL,
+}
+
+# Configure medialive client
+s3_client = boto3.client("s3", **scw_credentials)
+
+
+class Command(BaseCommand):
+    """Rename classroom documents in Scaleway S3 to their filename."""
+
+    help = "Rename classroom documents in Scaleway S3 to their filename."
+
+    def validate_filename(self, value):
+        """Transform filename to make it valid."""
+
+        value = value.replace("/", "_")
+        value = value.replace("\\", "_")
+        value = value.lstrip(".")
+
+        return value
+
+    def handle(self, *args, **options):
+        """Execute management command."""
+
+        for document in ClassroomDocument.objects.all():
+            # Get the file stored on Scaleway S3 under `aws/`
+            stamp = time_utils.to_timestamp(document.uploaded_on)
+            extension = ""
+            if "." in document.filename:
+                extension = splitext(document.filename)[1]
+
+            file_key_src = document.get_storage_key(
+                filename=f"{stamp}{extension}", base_dir=AWS_STORAGE_BASE_DIRECTORY
+            )
+            copy_source = {
+                "Bucket": settings.STORAGE_S3_BUCKET_NAME,
+                "Key": file_key_src,
+            }
+
+            filename = self.validate_filename(document.filename)
+
+            # Override document filename with the validated S3-compatible filename
+            if filename != document.filename:
+                document.filename = filename
+                document.save()
+
+            # Compute file key destination which should be the document filename
+            file_key_dest = document.get_storage_key(
+                filename, base_dir=AWS_STORAGE_BASE_DIRECTORY
+            )
+            if file_storage.exists(file_key_dest):
+                logger.info("Object %s already exists", file_key_dest)
+                continue
+
+            logger.info("Copying %s to %s", file_key_src, file_key_dest)
+            s3_client.copy_object(
+                Bucket=settings.STORAGE_S3_BUCKET_NAME,
+                CopySource=copy_source,
+                Key=file_key_dest,
+            )
+
+        logger.info("Finished copying!")
diff --git a/src/backend/marsha/bbb/serializers.py b/src/backend/marsha/bbb/serializers.py
@@ -3,7 +3,6 @@
 from datetime import datetime
 import mimetypes
 from os.path import splitext
-from urllib.parse import quote_plus
 from uuid import uuid4
 
 from django.conf import settings
@@ -22,15 +21,14 @@
     ClassroomSession,
 )
 from marsha.bbb.utils.bbb_utils import get_recording_url, get_url as get_document_url
-from marsha.core.defaults import CLASSROOM_RECORDINGS_KEY_CACHE, SCW_S3, VOD_CONVERT
+from marsha.core.defaults import CLASSROOM_RECORDINGS_KEY_CACHE, VOD_CONVERT
 from marsha.core.serializers import (
     BaseInitiateUploadSerializer,
     PlaylistLiteSerializer,
     ReadOnlyModelSerializer,
     UploadableFileWithExtensionSerializerMixin,
     VideoFromRecordingSerializer,
 )
-from marsha.core.storage.storage_class import file_storage
 
 
 class ClassroomRecordingSerializer(ReadOnlyModelSerializer):
@@ -405,25 +403,11 @@ def get_url(self, obj):
         Returns
         -------
         String or None
-            the url to fetch the classroom document on CloudFront/Storage
+            the url to fetch the classroom document on Storage
             None if the classroom document is still not uploaded to S3 with success
 
         """
-        if not obj.uploaded_on:
-            return None
-
-        if obj.storage_location == SCW_S3:
-            file_key = obj.get_storage_key(obj.filename)
-
-            return file_storage.url(file_key)
-
-        # Default AWS fallback
-        url = get_document_url(obj)
-
-        return (
-            f"{url}?response-content-disposition="
-            f"{quote_plus('attachment; filename=' + obj.filename)}"
-        )
+        return get_document_url(obj)
 
 
 class ClassroomDocumentInitiateUploadSerializer(BaseInitiateUploadSerializer):
diff --git a/src/backend/marsha/bbb/tests/api/classroomdocument/test_list.py b/src/backend/marsha/bbb/tests/api/classroomdocument/test_list.py
@@ -112,8 +112,7 @@ def test_api_list_classroom_documents_instructor(self):
         )
 
     @override_settings(
-        CLOUDFRONT_SIGNED_URLS_ACTIVE=True,
-        CLOUDFRONT_SIGNED_PUBLIC_KEY_ID="cloudfront-access-key-id",
+        MEDIA_URL="https://abc.svc.edge.scw.cloud/",
     )
     def test_api_list_classroom_documents_on_aws(self):
         """Classroom documents should not been signed."""
@@ -156,10 +155,8 @@ def test_api_list_classroom_documents_on_aws(self):
                         "upload_state": "pending",
                         "uploaded_on": "2018-08-08T00:00:00Z",
                         "url": (
-                            f"https://abc.cloudfront.net/{classroom.id}/classroomdocument/"
-                            f"{classroom_documents[3].id}/1533686400"
-                            f"?response-content-disposition"
-                            f"=attachment%3B+filename%3D{classroom_documents[3].filename}"
+                            f"https://abc.svc.edge.scw.cloud/aws/{classroom.id}/classroomdocument/"
+                            f"{classroom_documents[3].id}/{classroom_documents[3].filename}"
                         ),
                     },
                     {
@@ -170,11 +167,8 @@ def test_api_list_classroom_documents_on_aws(self):
                         "upload_state": "pending",
                         "uploaded_on": "2018-08-08T00:00:00Z",
                         "url": (
-                            f"https://abc.cloudfront.net/{classroom.id}/classroomdocument/"
-                            f"{classroom_documents[2].id}/1533686400"
-                            f".{classroom_documents[2].filename.split('.')[-1]}"
-                            f"?response-content-disposition"
-                            f"=attachment%3B+filename%3D{classroom_documents[2].filename}"
+                            f"https://abc.svc.edge.scw.cloud/aws/{classroom.id}/classroomdocument/"
+                            f"{classroom_documents[2].id}/{classroom_documents[2].filename}"
                         ),
                     },
                     {
@@ -185,11 +179,8 @@ def test_api_list_classroom_documents_on_aws(self):
                         "upload_state": "pending",
                         "uploaded_on": "2018-08-08T00:00:00Z",
                         "url": (
-                            f"https://abc.cloudfront.net/{classroom.id}/classroomdocument/"
-                            f"{classroom_documents[1].id}/1533686400"
-                            f".{classroom_documents[1].filename.split('.')[-1]}"
-                            f"?response-content-disposition"
-                            f"=attachment%3B+filename%3D{classroom_documents[1].filename}"
+                            f"https://abc.svc.edge.scw.cloud/aws/{classroom.id}/classroomdocument/"
+                            f"{classroom_documents[1].id}/{classroom_documents[1].filename}"
                         ),
                     },
                     {
@@ -200,11 +191,8 @@ def test_api_list_classroom_documents_on_aws(self):
                         "upload_state": "pending",
                         "uploaded_on": "2018-08-08T00:00:00Z",
                         "url": (
-                            f"https://abc.cloudfront.net/{classroom.id}/classroomdocument/"
-                            f"{classroom_documents[0].id}/1533686400"
-                            f".{classroom_documents[0].filename.split('.')[-1]}"
-                            f"?response-content-disposition"
-                            f"=attachment%3B+filename%3D{classroom_documents[0].filename}"
+                            f"https://abc.svc.edge.scw.cloud/aws/{classroom.id}/classroomdocument/"
+                            f"{classroom_documents[0].id}/{classroom_documents[0].filename}"
                         ),
                     },
                 ],
diff --git a/src/backend/marsha/bbb/tests/test_command_rename_classroom_documents.py b/src/backend/marsha/bbb/tests/test_command_rename_classroom_documents.py
@@ -0,0 +1,111 @@
+"""Test the ``rename_classroom_documents`` management command."""
+
+from datetime import datetime, timezone
+from os.path import splitext
+from unittest import mock
+
+from django.core.management import call_command
+from django.test import TestCase
+
+from botocore.stub import Stubber
+
+from marsha import settings
+from marsha.bbb.factories import ClassroomDocumentFactory, ClassroomFactory
+from marsha.bbb.management.commands import rename_classroom_documents
+from marsha.bbb.models import ClassroomDocument
+from marsha.core.utils import time_utils
+
+
+class RenameClassroomDocumentsTestCase(TestCase):
+    """
+    Test the ``rename_classroom_documents`` command.
+    """
+
+    @mock.patch("marsha.core.storage.storage_class.file_storage.exists")
+    def test_rename_classroom_documents(self, mock_exists):
+        """Command should rename document S3 objects to their filename."""
+
+        mock_exists.return_value = False
+
+        now = datetime(2018, 8, 8, tzinfo=timezone.utc)
+
+        with Stubber(rename_classroom_documents.s3_client) as s3_client_stubber:
+            # Generate some classroom documents
+            # (<original filename>, <expected and cleaned>)
+            filenames = [
+                ("normal_filename.pdf", "normal_filename.pdf"),
+                ("weird\\file/name.pdf", "weird_file_name.pdf"),
+                (".hidden_file", "hidden_file"),
+            ]
+
+            documents = []
+            for filename_src, _ in filenames:
+                document = ClassroomDocumentFactory(
+                    classroom=ClassroomFactory(),
+                    filename=filename_src,
+                    uploaded_on=now,
+                )
+
+                documents.append(document)
+
+            # Create mocks for copy_objects with Stubber
+            # Note: Stubber requires that its mocks are called in the exact order they
+            # were created, so we must iterate over objects.all() in the same sequence
+            for document in ClassroomDocument.objects.all():
+                stamp = time_utils.to_timestamp(document.uploaded_on)
+                extension = ""
+                if "." in document.filename:
+                    extension = splitext(document.filename)[1]
+
+                file_key_src = (
+                    f"aws/{document.classroom.id}/classroomdocument/"
+                    f"{document.id}/{stamp}{extension}"
+                )
+
+                sanitized_filename = (
+                    rename_classroom_documents.Command().validate_filename(
+                        document.filename
+                    )
+                )
+                file_key_dest = (
+                    f"aws/{document.classroom.id}/classroomdocument/"
+                    f"{document.id}/{sanitized_filename}"
+                )
+
+                expected_params = {
+                    "Bucket": settings.STORAGE_S3_BUCKET_NAME,
+                    "CopySource": {
+                        "Bucket": settings.STORAGE_S3_BUCKET_NAME,
+                        "Key": file_key_src,
+                    },
+                    "Key": file_key_dest,
+                }
+                s3_client_stubber.add_response("copy_object", {}, expected_params)
+
+            call_command("rename_classroom_documents")
+
+            s3_client_stubber.assert_no_pending_responses()
+
+            # Check that each document.filename has been updated with the clean
+            # S3-compatible filename
+            for document, (_, expected_filename) in zip(documents, filenames):
+                document.refresh_from_db()
+                assert document.filename == expected_filename
+
+    @mock.patch("marsha.core.storage.storage_class.file_storage.exists")
+    def test_rename_classroom_documents_file_exists(self, mock_exists):
+        """Command should not copy document if file already exists."""
+
+        mock_exists.return_value = True
+
+        now = datetime(2018, 8, 8, tzinfo=timezone.utc)
+
+        ClassroomDocumentFactory(
+            classroom=ClassroomFactory(),
+            filename="filename.pdf",
+            uploaded_on=now,
+        )
+
+        with Stubber(rename_classroom_documents.s3_client) as s3_client_stubber:
+            call_command("rename_classroom_documents")
+            s3_client_stubber.assert_no_pending_responses()
diff --git a/src/backend/marsha/bbb/utils/bbb_utils.py b/src/backend/marsha/bbb/utils/bbb_utils.py
@@ -5,7 +5,6 @@
 import json
 from json import JSONDecodeError
 import logging
-from os.path import splitext
 
 from django.conf import settings
 from django.utils.timezone import now
@@ -16,7 +15,7 @@
 import xmltodict
 
 from marsha.bbb.models import Classroom, ClassroomRecording, ClassroomSession
-from marsha.core.defaults import SCW_S3
+from marsha.core.defaults import AWS_STORAGE_BASE_DIRECTORY, SCW_S3
 from marsha.core.storage.storage_class import file_storage
 from marsha.core.utils import time_utils
 
@@ -106,7 +105,7 @@ def get_url(obj):
     Returns
     -------
     String or None
-        the url to fetch the classroom document on CloudFront or Edge Service
+        the url to fetch the classroom document on Edge Service
         None if the classroom document is still not uploaded to S3 with success
 
     """
@@ -117,19 +116,12 @@ def get_url(obj):
         file_key = obj.get_storage_key(obj.filename)
         return file_storage.url(file_key)
 
-    extension = ""
-    if "." in obj.filename:
-        extension = splitext(obj.filename)[1]
-
-    stamp = time_utils.to_timestamp(obj.uploaded_on)
-
-    # Default AWS fallback
-    url = (
-        f"{settings.AWS_S3_URL_PROTOCOL}://{settings.CLOUDFRONT_DOMAIN}/"
-        f"{obj.classroom.pk}/classroomdocument/{obj.pk}/{stamp}{extension}"
+    # Default fallback to location under "aws" directory
+    file_key = obj.get_storage_key(
+        filename=f"{obj.filename}", base_dir=AWS_STORAGE_BASE_DIRECTORY
     )
 
-    return url
+    return file_storage.url(file_key)
 
 
 def create(classroom: Classroom, recording_ready_callback_url: str, attempt=0):
