✨(back) add support for storing classroom documents on Scaleway S3

Leverage the existing django-storages backend to enable storing classroom
documents on Scaleway S3, alongside on the AWS S3.
For now we support both storage for a smooth transition away from AWS S3.

Author: wilbrdt

CHANGELOG.md

@@ -8,6 +8,10 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+### Added
+
+- Add support for storing classroom documents on Scaleway S3
+
 ### Fixed
 
 - Fix timeout increase on Nginx for peertube runner success request

src/backend/marsha/bbb/api.py

@@ -35,14 +35,13 @@
     process_recordings,
 )
 from marsha.bbb.utils.tokens import create_classroom_stable_invite_jwt
-from marsha.core import defaults, permissions as core_permissions
+from marsha.core import defaults, permissions as core_permissions, storage
 from marsha.core.api import APIViewMixin, BulkDestroyModelMixin, ObjectPkMixin
-from marsha.core.defaults import VOD_CONVERT
+from marsha.core.defaults import READY, VOD_CONVERT
 from marsha.core.models import ADMINISTRATOR, INSTRUCTOR, Video
 from marsha.core.tasks.recording import copy_video_recording
 from marsha.core.tasks.video import launch_video_transcoding
-from marsha.core.utils.s3_utils import create_presigned_post
-from marsha.core.utils.time_utils import to_timestamp
+from marsha.core.utils.time_utils import to_datetime, to_timestamp
 
 
 class ObjectClassroomRelatedMixin:
@@ -615,32 +614,27 @@ def initiate_upload(self, request, pk=None, classroom_id=None):
             HttpResponse carrying the AWS S3 upload policy as a JSON object.
 
         """
-        classroom = self.get_object()  # check permissions first
+        classroom_document = self.get_object()  # check permissions first
 
         serializer = serializers.ClassroomDocumentInitiateUploadSerializer(
             data=request.data
         )
         if serializer.is_valid() is not True:
             return Response(serializer.errors, status=400)
 
-        now = timezone.now()
-        stamp = to_timestamp(now)
-
-        key = classroom.get_source_s3_key(
-            stamp=stamp, extension=serializer.validated_data["extension"]
-        )
-
-        presigned_post = create_presigned_post(
-            [
-                ["eq", "$Content-Type", serializer.validated_data["mimetype"]],
+        presigned_post = (
+            storage.get_initiate_backend().initiate_classroom_document_storage_upload(
+                request,
+                classroom_document,
                 [
-                    "content-length-range",
-                    0,
-                    settings.CLASSROOM_DOCUMENT_SOURCE_MAX_SIZE,
+                    ["eq", "$Content-Type", serializer.validated_data["mimetype"]],
+                    [
+                        "content-length-range",
+                        0,
+                        settings.CLASSROOM_DOCUMENT_SOURCE_MAX_SIZE,
+                    ],
                 ],
-            ],
-            {},
-            key,
+            )
         )
 
         # Reset the upload state of the classroom document
@@ -651,6 +645,44 @@ def initiate_upload(self, request, pk=None, classroom_id=None):
 
         return Response(presigned_post)
 
+    @action(methods=["post"], detail=True, url_path="upload-ended")
+    # pylint: disable=unused-argument
+    def upload_ended(self, request, pk=None, classroom_id=None):
+        """Notify the API that the classroom document upload has ended.
+
+        Calling the endpoint will update the upload state of the classroom document.
+        The request should have a file_key in the body, which is the key of the
+        uploaded file.
+
+        Parameters
+        ----------
+        request : Type[django.http.request.HttpRequest]
+            The request on the API endpoint
+        pk: string
+            The primary key of the classroom document
+
+        Returns
+        -------
+        Type[rest_framework.response.Response]
+            HttpResponse with the serialized classroom document.
+        """
+        # Ensure object exists and user has access to it
+        classroom_document = self.get_object()
+
+        serializer = serializers.ClassroomDocumentUploadEndedSerializer(
+            data=request.data, context={"obj": classroom_document}
+        )
+        serializer.is_valid(raise_exception=True)
+
+        file_key = serializer.validated_data["file_key"]
+        # The file_key have the "classroom/{classroom_pk}/classroomdocument/{stamp}"
+        # format
+        stamp = file_key.split("/")[-1]
+
+        classroom_document.update_upload_state(READY, to_datetime(stamp))
+
+        return Response(serializer.data)
+
     def perform_destroy(self, instance):
         """Change the default document if the instance is the default one"""
 

src/backend/marsha/bbb/migrations/0024_classroomdocument_storage_location.py

@@ -0,0 +1,24 @@
+# Generated by Django 5.0.9 on 2025-04-02 15:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bbb", "0023_classroom_infos"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="classroomdocument",
+            name="storage_location",
+            field=models.CharField(
+                choices=[("AWS", "AWS"), ("SCW", "SCW")],
+                default="SCW",
+                help_text="Location used to store the classroom document",
+                max_length=255,
+                verbose_name="storage location",
+            ),
+        ),
+    ]

src/backend/marsha/bbb/models.py

@@ -16,6 +16,13 @@
 
 from safedelete.managers import SafeDeleteManager
 
+from marsha.core.defaults import (
+    CLASSROOM_STORAGE_BASE_DIRECTORY,
+    DELETED_STORAGE_BASE_DIRECTORY,
+    SCW_S3,
+    STORAGE_BASE_DIRECTORY,
+    STORAGE_LOCATION_CHOICES,
+)
 from marsha.core.models import (
     BaseModel,
     Playlist,
@@ -329,6 +336,14 @@ class ClassroomDocument(UploadableFileMixin, BaseModel):
         help_text=_("is displayed by default in the classroom"),
     )
 
+    storage_location = models.CharField(
+        max_length=255,
+        verbose_name=_("storage location"),
+        help_text=_("Location used to store the classroom document"),
+        choices=STORAGE_LOCATION_CHOICES,
+        default=SCW_S3,
+    )
+
     class Meta:
         """Options for the ``ClassroomDocument`` model."""
 
@@ -369,6 +384,37 @@ def get_source_s3_key(self, stamp=None, extension=None):
         stamp = stamp or to_timestamp(self.uploaded_on)
         return f"{self.classroom.pk}/classroomdocument/{self.pk}/{stamp}{extension}"
 
+    def get_storage_prefix(
+        self,
+        stamp=None,
+        base_dir: STORAGE_BASE_DIRECTORY = CLASSROOM_STORAGE_BASE_DIRECTORY,
+    ):
+        """Compute the storage prefix for the classroom document.
+
+        Parameters
+        ----------
+        stamp: Type[string]
+            Passing a value for this argument will return the storage prefix for the
+            classroom document assuming its active stamp is set to this value. This is
+            useful to create an upload policy for this prospective version of the
+            classroom, so that the client can upload the file to S3.
+
+        base: Type[STORAGE_BASE_DIRECTORY]
+            The storage base directory. Defaults to Classroom. It will be used to
+            compute the storage prefix.
+
+        Returns
+        -------
+        string
+            The storage prefix for the classroom document, depending on the base directory passed.
+        """
+        stamp = stamp or self.uploaded_on_stamp()
+        base = base_dir
+        if base == DELETED_STORAGE_BASE_DIRECTORY:
+            base = f"{base}/{CLASSROOM_STORAGE_BASE_DIRECTORY}"
+
+        return f"{base}/{self.classroom.pk}/classroomdocument/{self.pk}/{stamp}"
+
 
 class ClassroomRecording(BaseModel):
     """Model representing a recording in a classroom."""

src/backend/marsha/bbb/serializers.py

@@ -22,14 +22,19 @@
     ClassroomSession,
 )
 from marsha.bbb.utils.bbb_utils import get_recording_url, get_url as get_document_url
-from marsha.core.defaults import CLASSROOM_RECORDINGS_KEY_CACHE, VOD_CONVERT
+from marsha.core.defaults import (
+    CLASSROOM_RECORDINGS_KEY_CACHE,
+    CLASSROOM_STORAGE_BASE_DIRECTORY,
+    VOD_CONVERT,
+)
 from marsha.core.serializers import (
     BaseInitiateUploadSerializer,
     PlaylistLiteSerializer,
     ReadOnlyModelSerializer,
     UploadableFileWithExtensionSerializerMixin,
     VideoFromRecordingSerializer,
 )
+from marsha.core.utils import time_utils
 
 
 class ClassroomRecordingSerializer(ReadOnlyModelSerializer):
@@ -453,3 +458,28 @@ def validate(self, attrs):
             attrs["mimetype"] = mimetype
 
         return attrs
+
+
+class ClassroomDocumentUploadEndedSerializer(serializers.Serializer):
+    """A serializer to validate data submitted on the UploadEnded API endpoint."""
+
+    file_key = serializers.CharField()
+
+    def validate_file_key(self, value):
+        """Check if the file_key is valid."""
+
+        stamp = value.split("/")[-1]
+
+        try:
+            time_utils.to_datetime(stamp)
+        except serializers.ValidationError as error:
+            raise serializers.ValidationError("file_key is not valid") from error
+
+        if (
+            self.context["obj"].get_storage_prefix(
+                stamp, CLASSROOM_STORAGE_BASE_DIRECTORY
+            )
+            != value
+        ):
+            raise serializers.ValidationError("file_key is not valid")
+        return value