✨(back) add support for storing deposited files on Scaleway S3

Leverage the existing django-storages backend to enable storing deposited
files on Scaleway S3, alongside AWS S3.
This dual-storage setup mirrors the approach used for classroom documents,
allowing for a smooth transition away from AWS S3.

Author: wilbrdt

CHANGELOG.md

@@ -11,6 +11,7 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 ### Added
 
 - Add support for storing classroom documents on Scaleway S3
+- Add support for storing deposited files on Scaleway S3
 
 ### Fixed
 

src/backend/marsha/core/defaults.py

@@ -181,10 +181,12 @@
 DELETED_STORAGE_BASE_DIRECTORY = "deleted"
 VOD_STORAGE_BASE_DIRECTORY = "vod"
 CLASSROOM_STORAGE_BASE_DIRECTORY = "classroom"
+FILE_DEPOSITORY_STORAGE_BASE_DIRECTORY = "filedepository"
 
 STORAGE_BASE_DIRECTORY = (
     (TMP_STORAGE_BASE_DIRECTORY, _("tmp")),
     (VOD_STORAGE_BASE_DIRECTORY, _("VOD")),
     (CLASSROOM_STORAGE_BASE_DIRECTORY, _("Classroom")),
     (DELETED_STORAGE_BASE_DIRECTORY, _("deleted")),
+    (FILE_DEPOSITORY_STORAGE_BASE_DIRECTORY, _("File Depository")),
 )

src/backend/marsha/core/storage/filesystem.py

@@ -77,3 +77,38 @@ def initiate_classroom_document_storage_upload(request, obj, filename, condition
             )
         ),
     }
+
+
+# pylint: disable=unused-argument
+def initiate_deposited_file_storage_upload(request, obj, filename, conditions):
+    """Get an upload policy for a deposited file.
+
+    Returns an upload policy for the filesystem backend.
+
+    Parameters
+    ----------
+    request : Type[django.http.request.HttpRequest]
+        The request on the API endpoint
+    pk: string
+        The primary key of the video
+
+    Returns
+    -------
+    Dictionary
+        A dictionary with two elements: url and fields. Url is the url to post to. Fields is a
+        dictionary filled with the form fields and respective values to use when submitting
+        the post.
+
+    """
+    key = obj.get_storage_key(filename=filename)
+    return {
+        "fields": {
+            "key": key,
+        },
+        "url": request.build_absolute_uri(
+            reverse(
+                "local-deposited_file-upload",
+                args=[obj.pk],
+            )
+        ),
+    }

src/backend/marsha/core/storage/s3.py

@@ -138,3 +138,29 @@ def initiate_classroom_document_storage_upload(request, obj, filename, condition
         S3FileStorage.bucket_name,
         "STORAGE_S3",
     )
+
+
+# pylint: disable=unused-argument
+def initiate_deposited_file_storage_upload(request, obj, filename, conditions):
+    """Get an upload policy for a deposited file.
+
+    The object must implement the get_storage_key method.
+    Returns an upload policy to our storage S3 destination bucket.
+
+    Returns
+    -------
+    Dictionary
+        A dictionary with two elements: url and fields. Url is the url to post to. Fields is a
+        dictionary filled with the form fields and respective values to use when submitting
+        the post.
+
+    """
+    key = obj.get_storage_key(filename=filename)
+
+    return create_presigned_post(
+        conditions,
+        {},
+        key,
+        S3FileStorage.bucket_name,
+        "STORAGE_S3",
+    )

src/backend/marsha/deposit/api.py

@@ -9,11 +9,10 @@
 from rest_framework.decorators import action
 from rest_framework.response import Response
 
-from marsha.core import defaults, permissions as core_permissions
+from marsha.core import defaults, permissions as core_permissions, storage
 from marsha.core.api import APIViewMixin, ObjectPkMixin, ObjectRelatedMixin
 from marsha.core.models import ADMINISTRATOR, LTI_ROLES, STUDENT
-from marsha.core.utils.s3_utils import create_presigned_post
-from marsha.core.utils.time_utils import to_timestamp
+from marsha.core.utils.time_utils import to_datetime, to_timestamp
 from marsha.deposit import permissions, serializers
 from marsha.deposit.defaults import LTI_ROUTE
 from marsha.deposit.forms import FileDepositoryForm
@@ -301,20 +300,20 @@ def initiate_upload(
     ):
         """Get an upload policy for a deposited file.
 
-        Calling the endpoint resets the upload state to `pending` and returns an upload policy to
-        our AWS S3 source bucket.
+        Calling the endpoint resets the upload state to `pending` and returns an upload
+        policy to our S3 storage bucket.
 
         Parameters
         ----------
         request : Type[django.http.request.HttpRequest]
             The request on the API endpoint
         pk: string
-            The primary key of the shared live media
+            The primary key of the deposited file
 
         Returns
         -------
         Type[rest_framework.response.Response]
-            HttpResponse carrying the AWS S3 upload policy as a JSON object.
+            HttpResponse carrying the S3 storage upload policy as a JSON object.
 
         """
         deposited_file = self.get_object()  # check permissions first
@@ -326,20 +325,26 @@ def initiate_upload(
         if serializer.is_valid() is not True:
             return Response(serializer.errors, status=400)
 
-        now = timezone.now()
-        stamp = to_timestamp(now)
-
-        key = deposited_file.get_source_s3_key(
-            stamp=stamp, extension=serializer.validated_data["extension"]
-        )
-
-        presigned_post = create_presigned_post(
-            [
-                ["eq", "$Content-Type", serializer.validated_data["mimetype"]],
-                ["content-length-range", 0, settings.DEPOSITED_FILE_SOURCE_MAX_SIZE],
-            ],
-            {},
-            key,
+        filename = serializer.validated_data["filename"]
+        extension = serializer.validated_data["extension"]
+
+        if not filename.endswith(extension):
+            filename = f"{filename}{extension}"
+
+        presigned_post = (
+            storage.get_initiate_backend().initiate_deposited_file_storage_upload(
+                request,
+                deposited_file,
+                filename,
+                [
+                    ["eq", "$Content-Type", serializer.validated_data["mimetype"]],
+                    [
+                        "content-length-range",
+                        0,
+                        settings.DEPOSITED_FILE_SOURCE_MAX_SIZE,
+                    ],
+                ],
+            )
         )
 
         # Reset the upload state of the deposited file
@@ -349,3 +354,44 @@ def initiate_upload(
         )
 
         return Response(presigned_post)
+
+    @action(methods=["post"], detail=True, url_path="upload-ended")
+    # pylint: disable=unused-argument
+    def upload_ended(
+        self,
+        request,
+        pk=None,
+        filedepository_id=None,
+    ):
+        """Notify the API that the deposited file upload has ended.
+
+        Calling the endpoint will update the upload state of the deposited file.
+        The request should have a file_key in the body, which is the key of the
+        uploaded file.
+
+        Parameters
+        ----------
+        request : Type[django.http.request.HttpRequest]
+            The request on the API endpoint
+        pk: string
+            The primary key of the deposited file
+
+        Returns
+        -------
+        Type[rest_framework.response.Response]
+            HttpResponse with the serialized deposited file.
+        """
+        deposited_file = self.get_object()  # check permissions first
+
+        serializer = serializers.DepositedFileUploadEndedSerializer(
+            data=request.data, context={"obj": deposited_file}
+        )
+
+        serializer.is_valid(raise_exception=True)
+
+        now = timezone.now()
+        stamp = to_timestamp(now)
+
+        deposited_file.update_upload_state(defaults.READY, to_datetime(stamp))
+
+        return Response(serializer.data)

src/backend/marsha/deposit/migrations/0006_depositedfile_storage_location.py

@@ -0,0 +1,24 @@
+# Generated by Django 5.0.9 on 2025-04-30 07:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("deposit", "0005_alter_depositedfile_options"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="depositedfile",
+            name="storage_location",
+            field=models.CharField(
+                choices=[("AWS", "AWS"), ("SCW", "SCW")],
+                default="SCW",
+                help_text="Location used to store the deposited file",
+                max_length=255,
+                verbose_name="storage location",
+            ),
+        ),
+    ]

src/backend/marsha/deposit/models.py

@@ -10,8 +10,14 @@
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 
+from marsha.core.defaults import (
+    DELETED_STORAGE_BASE_DIRECTORY,
+    FILE_DEPOSITORY_STORAGE_BASE_DIRECTORY,
+    SCW_S3,
+    STORAGE_BASE_DIRECTORY,
+    STORAGE_LOCATION_CHOICES,
+)
 from marsha.core.models import BaseModel, Playlist, UploadableFileMixin
-from marsha.core.utils.time_utils import to_timestamp
 
 
 logger = logging.getLogger(__name__)
@@ -152,6 +158,14 @@ class DepositedFile(UploadableFileMixin, BaseModel):
         verbose_name=_("read"),
     )
 
+    storage_location = models.CharField(
+        max_length=255,
+        verbose_name=_("storage location"),
+        help_text=_("Location used to store the deposited file"),
+        choices=STORAGE_LOCATION_CHOICES,
+        default=SCW_S3,
+    )
+
     class Meta:
         """Options for the ``DepositedFile`` model."""
 
@@ -160,43 +174,33 @@ class Meta:
         verbose_name = _("Deposited file")
         verbose_name_plural = _("Deposited files")
 
-    def get_source_s3_key(self, stamp=None, extension=None):
-        """Compute the S3 key in the source bucket.
-
-        It is built from the file deposit ID + ID of the deposited file + version stamp.
+    def get_storage_key(
+        self,
+        filename,
+        base_dir: STORAGE_BASE_DIRECTORY = FILE_DEPOSITORY_STORAGE_BASE_DIRECTORY,
+    ):
+        """Compute the storage key for the classroom document.
 
         Parameters
         ----------
-        stamp: Type[string]
-            Passing a value for this argument will return the source S3 key for the deposited file
-            assuming its active stamp is set to this value. This is useful to create an
-            upload policy for this prospective version of the track, so that the client can
-            upload the file to S3 and the confirmation lambda can set the `uploaded_on` field
-            to this value only after the file upload and processing is successful.
-
-
-        extension: Type[string]
-            The extension used by the uploaded media. This extension is added at the end of the key
-            to keep a record of the extension. We will use it in the update-state endpoint to
-            record it in the database.
+        filename: Type[string]
+            The filename of the uploaded media. For classroom documents, the filename is
+            directly set into the key.
+        base: Type[STORAGE_BASE_DIRECTORY]
+            The storage base directory. Defaults to Classroom. It will be used to
+            compute the storage key.
 
         Returns
         -------
         string
-            The S3 key for the deposited file in the source bucket, where uploaded files are
-            stored before they are converted and copied to the destination bucket.
-
+            The storage key for the classroom document, depending on the base directory
+            passed.
         """
-        # We don't want to deal with None value so we set it with an empty string
-        extension = extension or ""
-
-        # We check if the extension starts with a leading dot or not. If it's not the case we add
-        # it at the beginning of the string
-        if extension and not extension.startswith("."):
-            extension = "." + extension
+        base = base_dir
+        if base == DELETED_STORAGE_BASE_DIRECTORY:
+            base = f"{base}/{FILE_DEPOSITORY_STORAGE_BASE_DIRECTORY}"
 
-        stamp = stamp or to_timestamp(self.uploaded_on)
-        return f"{self.file_depository.pk}/depositedfile/{self.pk}/{stamp}{extension}"
+        return f"{base}/{self.file_depository.pk}/depositedfile/{self.pk}/{filename}"
 
     def update_upload_state(self, upload_state, uploaded_on, **extra_parameters):
         """Manage upload state.