✨(back) add support for storing markdown images on Scaleway S3

Leverage the existing django-storages backend to enable storing markdown images
on Scaleway S3, alongside AWS S3.

Author: wilbrdt

CHANGELOG.md

@@ -10,8 +10,9 @@ Versioning](https://semver.org/spec/v2.0.0.html).
 
 ### Added
 
-- Add support for storing classroom documents on Scaleway S3
+- Add support for storing markdown images on Scaleway S3
 - Add support for storing deposited files on Scaleway S3
+- Add support for storing classroom documents on Scaleway S3
 
 ### Fixed
 

src/backend/marsha/core/defaults.py

@@ -182,11 +182,13 @@
 VOD_STORAGE_BASE_DIRECTORY = "vod"
 CLASSROOM_STORAGE_BASE_DIRECTORY = "classroom"
 FILE_DEPOSITORY_STORAGE_BASE_DIRECTORY = "filedepository"
+MARKDOWN_DOCUMENT_STORAGE_BASE_DIRECTORY = "markdowndocument"
 
 STORAGE_BASE_DIRECTORY = (
     (TMP_STORAGE_BASE_DIRECTORY, _("tmp")),
     (VOD_STORAGE_BASE_DIRECTORY, _("VOD")),
     (CLASSROOM_STORAGE_BASE_DIRECTORY, _("Classroom")),
     (DELETED_STORAGE_BASE_DIRECTORY, _("deleted")),
     (FILE_DEPOSITORY_STORAGE_BASE_DIRECTORY, _("File Depository")),
+    (MARKDOWN_DOCUMENT_STORAGE_BASE_DIRECTORY, _("Markdown Document")),
 )

src/backend/marsha/core/storage/filesystem.py

@@ -3,7 +3,10 @@
 from django.urls import reverse
 from django.utils import timezone
 
-from marsha.core.defaults import TMP_STORAGE_BASE_DIRECTORY
+from marsha.core.defaults import (
+    MARKDOWN_DOCUMENT_STORAGE_BASE_DIRECTORY,
+    TMP_STORAGE_BASE_DIRECTORY,
+)
 from marsha.core.utils.time_utils import to_timestamp
 
 
@@ -55,7 +58,7 @@ def initiate_classroom_document_storage_upload(request, obj, filename, condition
     request : Type[django.http.request.HttpRequest]
         The request on the API endpoint
     pk: string
-        The primary key of the video
+        The primary key of the classroom document
 
     Returns
     -------
@@ -90,7 +93,7 @@ def initiate_deposited_file_storage_upload(request, obj, filename, conditions):
     request : Type[django.http.request.HttpRequest]
         The request on the API endpoint
     pk: string
-        The primary key of the video
+        The primary key of the deposited file
 
     Returns
     -------
@@ -112,3 +115,42 @@ def initiate_deposited_file_storage_upload(request, obj, filename, conditions):
             )
         ),
     }
+
+
+# pylint: disable=unused-argument
+def initiate_markdown_image_storage_upload(request, obj, conditions):
+    """Get an upload policy for a markdown image.
+
+    Returns an upload policy for the filesystem backend.
+
+    Parameters
+    ----------
+    request : Type[django.http.request.HttpRequest]
+        The request on the API endpoint
+    pk: string
+        The primary key of the markdown image
+
+    Returns
+    -------
+    Dictionary
+        A dictionary with two elements: url and fields. Url is the url to post to. Fields is a
+        dictionary filled with the form fields and respective values to use when submitting
+        the post.
+
+    """
+    now = timezone.now()
+    stamp = to_timestamp(now)
+    key = obj.get_storage_key(
+        stamp=stamp, base_dir=MARKDOWN_DOCUMENT_STORAGE_BASE_DIRECTORY
+    )
+    return {
+        "fields": {
+            "key": key,
+        },
+        "url": request.build_absolute_uri(
+            reverse(
+                "local-markdown-image-upload",
+                args=[obj.pk],
+            )
+        ),
+    }

src/backend/marsha/core/storage/s3.py

@@ -7,7 +7,10 @@
 
 from storages.backends.s3 import S3Storage
 
-from marsha.core.defaults import TMP_STORAGE_BASE_DIRECTORY
+from marsha.core.defaults import (
+    MARKDOWN_DOCUMENT_STORAGE_BASE_DIRECTORY,
+    TMP_STORAGE_BASE_DIRECTORY,
+)
 from marsha.core.models import Document
 from marsha.core.utils.cloudfront_utils import get_cloudfront_private_key
 from marsha.core.utils.s3_utils import create_presigned_post
@@ -164,3 +167,33 @@ def initiate_deposited_file_storage_upload(request, obj, filename, conditions):
         S3FileStorage.bucket_name,
         "STORAGE_S3",
     )
+
+
+# pylint: disable=unused-argument
+def initiate_markdown_image_storage_upload(request, obj, conditions):
+    """Get an upload policy for a markdown image.
+
+    The object must implement the get_storage_key method.
+    Returns an upload policy to our storage S3 destination bucket.
+
+    Returns
+    -------
+    Dictionary
+        A dictionary with two elements: url and fields. Url is the url to post to. Fields is a
+        dictionary filled with the form fields and respective values to use when submitting
+        the post.
+
+    """
+    now = timezone.now()
+    stamp = to_timestamp(now)
+    key = obj.get_storage_key(
+        stamp=stamp, base_dir=MARKDOWN_DOCUMENT_STORAGE_BASE_DIRECTORY
+    )
+
+    return create_presigned_post(
+        conditions,
+        {},
+        key,
+        S3FileStorage.bucket_name,
+        "STORAGE_S3",
+    )

src/backend/marsha/development/api.py

@@ -15,6 +15,7 @@
 from marsha.core.storage.storage_class import file_storage
 from marsha.core.utils import time_utils
 from marsha.deposit.models import DepositedFile
+from marsha.markdown.models import MarkdownImage
 
 
 logger = logging.getLogger(__name__)
@@ -103,3 +104,19 @@ def local_deposited_file_upload(request: HttpRequest, uuid=None):
 
     file_storage.save(destination, uploaded_deposited_file)
     return Response(status=204)
+
+
+@api_view(["POST"])
+def local_markdown_image_upload(request: HttpRequest, uuid=None):
+    """Endpoint to mock s3 markdown image upload."""
+    uploaded_markdown_image = request.FILES["file"]
+
+    try:
+        markdown_image = MarkdownImage.objects.get(id=uuid)
+    except MarkdownImage.DoesNotExist:
+        return Response({"success": False}, status=404)
+
+    destination = markdown_image.get_storage_key(filename=markdown_image.filename)
+
+    file_storage.save(destination, uploaded_markdown_image)
+    return Response(status=204)

src/backend/marsha/markdown/api.py

@@ -2,19 +2,17 @@
 
 from django.conf import settings
 from django.db.models import Q
-from django.utils import timezone
 
 import django_filters
 from rest_framework import filters, mixins, viewsets
 from rest_framework.decorators import action
 from rest_framework.response import Response
 from rest_framework.status import HTTP_200_OK, HTTP_400_BAD_REQUEST
 
-from marsha.core import defaults, permissions as core_permissions
+from marsha.core import defaults, permissions as core_permissions, storage
 from marsha.core.api import APIViewMixin, ObjectPkMixin, ObjectRelatedMixin
 from marsha.core.models import ADMINISTRATOR
-from marsha.core.utils.s3_utils import create_presigned_post
-from marsha.core.utils.time_utils import to_timestamp
+from marsha.core.utils.time_utils import to_datetime
 from marsha.markdown import permissions as markdown_permissions, serializers
 from marsha.markdown.defaults import LTI_ROUTE
 from marsha.markdown.forms import MarkdownDocumentForm
@@ -333,7 +331,7 @@ def initiate_upload(self, request, *args, pk=None, **kwargs):
         """Get an upload policy for a Markdown image.
 
         Calling the endpoint resets the upload state to `pending` and returns an upload policy to
-        our AWS S3 source bucket.
+        our S3 storage bucket.
 
         Parameters
         ----------
@@ -345,33 +343,31 @@ def initiate_upload(self, request, *args, pk=None, **kwargs):
         Returns
         -------
         Type[rest_framework.response.Response]
-            HttpResponse carrying the AWS S3 upload policy as a JSON object.
+            HttpResponse carrying the S3 storage upload policy as a JSON object.
 
         """
         markdown_image = self.get_object()  # check permissions first
 
-        serializer = serializers.MarkdownImageUploadSerializer(
+        serializer = serializers.MarkdownImageInitiateUploadSerializer(
             data=request.data,
         )
 
         if not serializer.is_valid():
             return Response(serializer.errors, status=HTTP_400_BAD_REQUEST)
 
-        now = timezone.now()
-        stamp = to_timestamp(now)
-
-        key = markdown_image.get_source_s3_key(
-            stamp=stamp,
-            extension=serializer.validated_data["extension"],
-        )
-
-        presigned_post = create_presigned_post(
-            [
-                ["starts-with", "$Content-Type", "image/"],
-                ["content-length-range", 0, settings.MARKDOWN_IMAGE_SOURCE_MAX_SIZE],
-            ],
-            {},
-            key,
+        presigned_post = (
+            storage.get_initiate_backend().initiate_markdown_image_storage_upload(
+                request,
+                markdown_image,
+                [
+                    ["starts-with", "$Content-Type", "image/"],
+                    [
+                        "content-length-range",
+                        0,
+                        settings.MARKDOWN_IMAGE_SOURCE_MAX_SIZE,
+                    ],
+                ],
+            )
         )
 
         # Reset the upload state of the Markdown image
@@ -381,3 +377,46 @@ def initiate_upload(self, request, *args, pk=None, **kwargs):
         )
 
         return Response(presigned_post)
+
+    @action(methods=["post"], detail=True, url_path="upload-ended")
+    # pylint: disable=unused-argument
+    def upload_ended(
+        self,
+        request,
+        pk=None,
+        markdown_document_id=None,
+    ):
+        """Notify the API that the markdown image upload has ended.
+
+        Calling the endpoint will update the upload state of the markdown image.
+        The request should have a file_key in the body, which is the key of the
+        uploaded file.
+
+        Parameters
+        ----------
+        request : Type[django.http.request.HttpRequest]
+            The request on the API endpoint
+        pk: string
+            The primary key of the markdown image
+
+        Returns
+        -------
+        Type[rest_framework.response.Response]
+            HttpResponse with the serialized markdown image.
+        """
+        markdown_image = self.get_object()  # check permissions first
+
+        serializer = serializers.MarkdownImageUploadEndedSerializer(
+            data=request.data, context={"obj": markdown_image}
+        )
+
+        serializer.is_valid(raise_exception=True)
+
+        file_key = serializer.validated_data["file_key"]
+        # The file_key have the "markdown/{markdown_pk}/markdownimage/{markdownimage}/
+        # {stamp}" format
+        stamp = file_key.split("/")[-1]
+
+        markdown_image.update_upload_state(defaults.READY, to_datetime(stamp))
+
+        return Response(serializer.data)

src/backend/marsha/markdown/migrations/0007_markdownimage_storage_location.py

@@ -0,0 +1,24 @@
+# Generated by Django 5.0.9 on 2025-05-09 07:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("markdown", "0006_alter_markdowndocumenttranslation_title"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="markdownimage",
+            name="storage_location",
+            field=models.CharField(
+                choices=[("AWS", "AWS"), ("SCW", "SCW")],
+                default="SCW",
+                help_text="Location used to store the Markdown image",
+                max_length=255,
+                verbose_name="storage location",
+            ),
+        ),
+    ]