🚚(backend) rename Django storage class for broader file support

Refactored the django-storages class to support various file types beyond
videos.
This change generalizes storage usage, making it applicable to different file
formats.

Author: wilbrdt

UPGRADE.md

@@ -7,6 +7,10 @@ not skip minor/major releases while upgrading (fix releases can be skipped).
 The format is inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+### 5.7.x to 5.8.0
+
+Environment variables previously prefixed with `DJANGO_VIDEOS_STORAGE_` are now prefixed with `DJANGO_STORAGE_`. Make sure to update your configuration accordingly.
+
 ### 4.2.x to 4.3.0
 
 `DJANGO_STATICFILES_STORAGE` environment variable is not used anymore. You have to replace it by `DJANGO_STORAGES_STATICFILES_BACKEND`.

docker-compose.yml

@@ -37,7 +37,7 @@ services:
       - .:/app
       - ./data/static:/data/static
       - ./data/media:/data/media
-      - ./data/videos:/data/videos
+      - ./data/files:/data/files
     depends_on:
       - "base"
       - "db"
@@ -57,7 +57,7 @@ services:
       - .:/app
       - ./data/static:/data/static
       - ./data/media:/data/media
-      - ./data/videos:/data/videos
+      - ./data/files:/data/files
     depends_on:
       - "base"
       - "db"

docs/peertube-runner.md

@@ -37,16 +37,16 @@ In Marsha app, anything that should be done through Peertube runners should have
 
 In our case we need to set the `STORAGES["videos"]["BACKEND"]` setting with the right storage class, and the `STORAGE_BACKEND` setting with the right value. See [storage](#storage), to understand why we need this second setting.
 
-#### S3VideoStorage
+#### S3FileStorage
 
-A storage used to store videos in a S3 like bucket. It can be used locally, and should be used in production.
+A storage used to store files in a S3 like bucket. It can be used locally, and should be used in production.
 
 ```Python
 STORAGES = {
     "videos": {
         "BACKEND": values.Value(
-            "marsha.core.storage.s3.S3VideoStorage",
-            environ_name="STORAGES_VIDEOS_BACKEND",
+            "marsha.core.storage.s3.S3FileStorage",
+            environ_name="STORAGES_BACKEND",
         ),
     },
 }
@@ -68,8 +68,8 @@ STORAGES = {
         ),
         "OPTIONS": values.DictValue(
             {
-                "location": str(Base.VIDEOS_ROOT),
-                "base_url": str(Base.VIDEOS_ROOT),
+                "location": str(Base.FILES_ROOT),
+                "base_url": str(Base.FILES_ROOT),
             },
             environ_name="STORAGES_VIDEOS_OPTIONS",
         ),
@@ -93,7 +93,7 @@ STORAGES = {
 
 #### URLS
 
-When a video object is returned to the client, it contains URLs to access the video playlist and mp4. These URLs are built in the video serializer. Before, they were manually created by combining the cloudfront domain and the video object key. It cannot work anymore because local files cannot be behind a cloudfront domain. BUT, we can simply make use `django-storages` and call the `video_storage.url(key)` method that will return the appropriate URL depending on the class used for videos.
+When a video object is returned to the client, it contains URLs to access the video playlist and mp4. These URLs are built in the video serializer. Before, they were manually created by combining the cloudfront domain and the video object key. It cannot work anymore because local files cannot be behind a cloudfront domain. BUT, we can simply make use `django-storages` and call the `storage.url(key)` method that will return the appropriate URL depending on the class used for videos.
 
 ### Callbacks
 

env.d/development.dist

@@ -15,11 +15,11 @@ DJANGO_AWS_ACCESS_KEY_ID=yourAwsAccesKey
 DJANGO_AWS_SECRET_ACCESS_KEY=YourAwsSecretAccessKeyId
 DJANGO_AWS_S3_REGION_NAME=eu-west-1
 
-# VIDEOS_STORAGE_S3
-DJANGO_VIDEOS_STORAGE_S3_ACCESS_KEY=yourS3AccesKey
-DJANGO_VIDEOS_STORAGE_S3_SECRET_KEY=yourS3AccesKey
-DJANGO_VIDEOS_STORAGE_S3_ENDPOINT_URL=https://s3.amazonaws.com
-DJANGO_VIDEOS_STORAGE_S3_BUCKET_NAME=yourS3BucketName
+# STORAGE_S3
+DJANGO_STORAGE_S3_ACCESS_KEY=yourS3AccesKey
+DJANGO_STORAGE_S3_SECRET_KEY=yourS3AccesKey
+DJANGO_STORAGE_S3_ENDPOINT_URL=https://s3.amazonaws.com
+DJANGO_STORAGE_S3_BUCKET_NAME=yourS3BucketName
 
 # LTI provider xml configuration (legacy)
 DJANGO_LTI_CONFIG_TITLE=Marsha Dev

src/backend/marsha/core/defaults.py

@@ -168,12 +168,12 @@
     (CELERY_PIPELINE, _("Celery")),
 )
 
-VOD_VIDEOS_STORAGE_BASE_DIRECTORY = "vod"
-TMP_VIDEOS_STORAGE_BASE_DIRECTORY = "tmp"
-DELETED_VIDEOS_STORAGE_BASE_DIRECTORY = "deleted"
-
-VIDEOS_STORAGE_BASE_DIRECTORY = (
-    (TMP_VIDEOS_STORAGE_BASE_DIRECTORY, _("tmp")),
-    (VOD_VIDEOS_STORAGE_BASE_DIRECTORY, _("VOD")),
-    (DELETED_VIDEOS_STORAGE_BASE_DIRECTORY, _("deleted")),
+VOD_STORAGE_BASE_DIRECTORY = "vod"
+TMP_STORAGE_BASE_DIRECTORY = "tmp"
+DELETED_STORAGE_BASE_DIRECTORY = "deleted"
+
+STORAGE_BASE_DIRECTORY = (
+    (TMP_STORAGE_BASE_DIRECTORY, _("tmp")),
+    (VOD_STORAGE_BASE_DIRECTORY, _("VOD")),
+    (DELETED_STORAGE_BASE_DIRECTORY, _("deleted")),
 )

src/backend/marsha/core/models/video.py

@@ -22,7 +22,7 @@
     APPROVAL,
     CELERY_PIPELINE,
     DELETED,
-    DELETED_VIDEOS_STORAGE_BASE_DIRECTORY,
+    DELETED_STORAGE_BASE_DIRECTORY,
     ENDED,
     ERROR,
     HARVESTED,
@@ -37,8 +37,8 @@
     STOPPING,
     TRANSCODE_PIPELINE_CHOICES,
     UPLOAD_ERROR_REASON_CHOICES,
-    VIDEOS_STORAGE_BASE_DIRECTORY,
-    VOD_VIDEOS_STORAGE_BASE_DIRECTORY,
+    STORAGE_BASE_DIRECTORY,
+    VOD_STORAGE_BASE_DIRECTORY,
 )
 from marsha.core.models.account import ADMINISTRATOR, INSTRUCTOR, OrganizationAccess
 from marsha.core.models.base import BaseModel
@@ -314,34 +314,34 @@ def get_source_s3_key(self, stamp=None):
         stamp = stamp or self.uploaded_on_stamp()
         return f"{self.pk}/video/{self.pk}/{stamp}"
 
-    def get_videos_storage_prefix(
+    def get_storage_prefix(
         self,
         stamp=None,
-        base_dir: VIDEOS_STORAGE_BASE_DIRECTORY = VOD_VIDEOS_STORAGE_BASE_DIRECTORY,
+        base_dir: STORAGE_BASE_DIRECTORY = VOD_STORAGE_BASE_DIRECTORY,
     ):
-        """Compute the videos storage prefix for the video.
+        """Compute the storage prefix for the video.
 
         Parameters
         ----------
         stamp: Type[string]
-            Passing a value for this argument will return the videos storage prefix for the video
+            Passing a value for this argument will return the storage prefix for the video
             assuming its active stamp is set to this value. This is useful to create an upload
             policy for this prospective version of the video, so that the client can upload the
             file to S3 and the transcodings job can set the `uploaded_on` field to this value.
 
-        base: Type[VIDEOS_STORAGE_BASE_DIRECTORY]
-            The videos storage base directory. Defaults to VOD. It will be used to compute the
-            videos storage prefix.
+        base: Type[STORAGE_BASE_DIRECTORY]
+            The storage base directory. Defaults to VOD. It will be used to compute the
+            storage prefix.
 
         Returns
         -------
         string
-            The videos storage prefix for the video, depending on the base directory passed.
+            The storage prefix for the video, depending on the base directory passed.
         """
         stamp = stamp or self.uploaded_on_stamp()
         base = base_dir
-        if base == DELETED_VIDEOS_STORAGE_BASE_DIRECTORY:
-            base = f"{base}/{VOD_VIDEOS_STORAGE_BASE_DIRECTORY}"
+        if base == DELETED_STORAGE_BASE_DIRECTORY:
+            base = f"{base}/{VOD_STORAGE_BASE_DIRECTORY}"
 
         return f"{base}/{self.pk}/video/{stamp}"
 
@@ -666,34 +666,34 @@ def get_source_s3_key(self, stamp=None):
         mode = f"_{self.mode}" if self.mode else ""
         return f"{self.video.pk}/timedtexttrack/{self.pk}/{stamp}_{self.language}{mode}"
 
-    def get_videos_storage_prefix(
+    def get_storage_prefix(
         self,
         stamp=None,
-        base_dir: VIDEOS_STORAGE_BASE_DIRECTORY = VOD_VIDEOS_STORAGE_BASE_DIRECTORY,
+        base_dir: STORAGE_BASE_DIRECTORY = VOD_STORAGE_BASE_DIRECTORY,
     ):
-        """Compute the videos storage prefix for the video.
+        """Compute the storage prefix for the video.
 
         Parameters
         ----------
         stamp: Type[string]
-            Passing a value for this argument will return the videos storage prefix for the video
+            Passing a value for this argument will return the storage prefix for the video
             assuming its active stamp is set to this value. This is useful to create an upload
             policy for this prospective version of the video, so that the client can upload the
             file to S3 and the transcodings job can set the `uploaded_on` field to this value.
 
-        base_dir: Type[VIDEOS_STORAGE_BASE_DIRECTORY]
-            The videos storage base directory. Defaults to VOD. It will be used to compute the
-            videos storage prefix.
+        base_dir: Type[STORAGE_BASE_DIRECTORY]
+            The storage base directory. Defaults to VOD. It will be used to compute the
+            storage prefix.
 
         Returns
         -------
         string
-            The videos storage prefix for the video, depending on the base directory passed.
+            The storage prefix for the video, depending on the base directory passed.
         """
         stamp = stamp or self.uploaded_on_stamp()
         base = base_dir
-        if base_dir == DELETED_VIDEOS_STORAGE_BASE_DIRECTORY:
-            base = f"{DELETED_VIDEOS_STORAGE_BASE_DIRECTORY}/{VOD_VIDEOS_STORAGE_BASE_DIRECTORY}"
+        if base_dir == DELETED_STORAGE_BASE_DIRECTORY:
+            base = f"{DELETED_STORAGE_BASE_DIRECTORY}/{VOD_STORAGE_BASE_DIRECTORY}"
 
         return f"{base}/{self.video.pk}/timedtext/{self.pk}/{stamp}"
 
@@ -817,35 +817,35 @@ def get_source_s3_key(self, stamp=None):
         stamp = stamp or self.uploaded_on_stamp()
         return f"{self.video.pk}/thumbnail/{self.pk}/{stamp}"
 
-    def get_videos_storage_prefix(
+    def get_storage_prefix(
         self,
         stamp=None,
-        base_dir: VIDEOS_STORAGE_BASE_DIRECTORY = VOD_VIDEOS_STORAGE_BASE_DIRECTORY,
+        base_dir: STORAGE_BASE_DIRECTORY = VOD_STORAGE_BASE_DIRECTORY,
     ):
-        """Compute the videos storage prefix for the thumbnail.
+        """Compute the storage prefix for the thumbnail.
 
         Parameters
         ----------
         stamp: Type[string]
-            Passing a value for this argument will return the videos storage prefix for the shared
+            Passing a value for this argument will return the storage prefix for the shared
             live media assuming its active stamp is set to this value. This is useful to create
             an upload policy for this prospective version of the thumbnail, so that the
             client can upload the file and celery task can set the `uploaded_on` field to this
             value.
 
-        base_dir: Type[VIDEOS_STORAGE_BASE_DIRECTORY]
-            The videos storage base directory. Defaults to VOD. It will be used to compute the
-            videos storage prefix.
+        base_dir: Type[STORAGE_BASE_DIRECTORY]
+            The storage base directory. Defaults to VOD. It will be used to compute the
+            storage prefix.
 
         Returns
         -------
         string
-            The videos storage prefix for the thumbnail.
+            The storage prefix for the thumbnail.
         """
         stamp = stamp or self.uploaded_on_stamp()
         base = base_dir
-        if base_dir == DELETED_VIDEOS_STORAGE_BASE_DIRECTORY:
-            base = f"{DELETED_VIDEOS_STORAGE_BASE_DIRECTORY}/{VOD_VIDEOS_STORAGE_BASE_DIRECTORY}"
+        if base_dir == DELETED_STORAGE_BASE_DIRECTORY:
+            base = f"{DELETED_STORAGE_BASE_DIRECTORY}/{VOD_STORAGE_BASE_DIRECTORY}"
 
         return f"{base}/{self.video.pk}/thumbnail/{stamp}"
 
@@ -1299,35 +1299,35 @@ def get_source_s3_key(self, stamp=None, extension=None):
         stamp = stamp or self.uploaded_on_stamp()
         return f"{self.video.pk}/sharedlivemedia/{self.pk}/{stamp}{extension}"
 
-    def get_videos_storage_prefix(
+    def get_storage_prefix(
         self,
         stamp=None,
-        base_dir: VIDEOS_STORAGE_BASE_DIRECTORY = VOD_VIDEOS_STORAGE_BASE_DIRECTORY,
+        base_dir: STORAGE_BASE_DIRECTORY = VOD_STORAGE_BASE_DIRECTORY,
     ):
-        """Compute the videos storage prefix for the shared live media.
+        """Compute the storage prefix for the shared live media.
 
         Parameters
         ----------
         stamp: Type[string]
-            Passing a value for this argument will return the videos storage prefix for the shared
+            Passing a value for this argument will return the storage prefix for the shared
             live media assuming its active stamp is set to this value. This is useful to create
             an upload policy for this prospective version of the shared live media, so that the
             client can upload the file and celery task can set the `uploaded_on` field to this
             value.
 
-        base_dir: Type[VIDEOS_STORAGE_BASE_DIRECTORY]
-            The videos storage base directory. Defaults to VOD. It will be used to compute the
-            videos storage prefix.
+        base_dir: Type[STORAGE_BASE_DIRECTORY]
+            The storage base directory. Defaults to VOD. It will be used to compute the
+            storage prefix.
 
         Returns
         -------
         string
-            The videos storage prefix for the shared live media.
+            The storage prefix for the shared live media.
         """
         stamp = stamp or self.uploaded_on_stamp()
         base = base_dir
-        if base_dir == DELETED_VIDEOS_STORAGE_BASE_DIRECTORY:
-            base = f"{DELETED_VIDEOS_STORAGE_BASE_DIRECTORY}/{VOD_VIDEOS_STORAGE_BASE_DIRECTORY}"
+        if base_dir == DELETED_STORAGE_BASE_DIRECTORY:
+            base = f"{DELETED_STORAGE_BASE_DIRECTORY}/{VOD_STORAGE_BASE_DIRECTORY}"
 
         return f"{base}/{self.video.pk}/sharedlivemedia/{self.pk}/{stamp}"
 

src/backend/marsha/core/serializers/shared_live_media.py

@@ -13,7 +13,7 @@
     UploadableFileWithExtensionSerializerMixin,
     get_video_cloudfront_url_params,
 )
-from marsha.core.storage.storage_class import video_storage
+from marsha.core.storage.storage_class import file_storage
 from marsha.core.utils import cloudfront_utils, time_utils
 
 
@@ -135,18 +135,18 @@ def get_celery_pipeline_url(self, obj):
         urls = {}
 
         stamp = time_utils.to_timestamp(obj.uploaded_on)
-        base = obj.get_videos_storage_prefix()
+        base = obj.get_storage_prefix()
 
         pages = {}
         for page_number in range(1, obj.nb_pages + 1):
             url = f"{base}/{stamp}_{page_number}.svg"
-            pages[page_number] = video_storage.url(url)
+            pages[page_number] = file_storage.url(url)
 
         urls["pages"] = pages
 
         if self.context.get("is_admin") or obj.show_download:
             url = f"{base}/{stamp}.pdf"
-            urls["media"] = video_storage.url(url)
+            urls["media"] = file_storage.url(url)
 
         return urls
 

src/backend/marsha/core/serializers/thumbnail.py

@@ -8,7 +8,7 @@
 from marsha.core.defaults import CELERY_PIPELINE
 from marsha.core.models import Thumbnail
 from marsha.core.serializers.base import TimestampField
-from marsha.core.storage.storage_class import video_storage
+from marsha.core.storage.storage_class import file_storage
 from marsha.core.utils import time_utils
 
 
@@ -115,11 +115,11 @@ def get_urls(self, obj):
 
         if obj.process_pipeline == CELERY_PIPELINE:
             stamp = time_utils.to_timestamp(obj.uploaded_on)
-            base = obj.get_videos_storage_prefix(stamp=stamp)
+            base = obj.get_storage_prefix(stamp=stamp)
 
             urls = {}
             for resolution in settings.VIDEO_RESOLUTIONS:
-                urls[resolution] = video_storage.url(f"{base}/{resolution}.jpg")
+                urls[resolution] = file_storage.url(f"{base}/{resolution}.jpg")
             return urls
 
         # Default AWS fallback:

src/backend/marsha/core/serializers/timed_text_track.py

@@ -10,7 +10,7 @@
 from marsha.core.defaults import CELERY_PIPELINE
 from marsha.core.models import TimedTextTrack
 from marsha.core.serializers.base import TimestampField, get_video_cloudfront_url_params
-from marsha.core.storage.storage_class import video_storage
+from marsha.core.storage.storage_class import file_storage
 from marsha.core.utils import cloudfront_utils, time_utils
 
 
@@ -146,9 +146,9 @@ def get_source_url(self, obj):
         stamp = time_utils.to_timestamp(obj.uploaded_on)
 
         if obj.process_pipeline == CELERY_PIPELINE:
-            base = obj.get_videos_storage_prefix()
+            base = obj.get_storage_prefix()
 
-            return video_storage.url(f"{base}/source.{obj.extension}")
+            return file_storage.url(f"{base}/source.{obj.extension}")
 
         # Default AWS fallback
         filename = f"{slugify(obj.video.playlist.title)}_{stamp}.{obj.extension}"
@@ -183,9 +183,9 @@ def get_url(self, obj):
 
         if obj.process_pipeline == CELERY_PIPELINE:
             stamp = time_utils.to_timestamp(obj.uploaded_on)
-            base = obj.get_videos_storage_prefix()
+            base = obj.get_storage_prefix()
 
-            return video_storage.url(f"{base}/{stamp}.vtt")
+            return file_storage.url(f"{base}/{stamp}.vtt")
 
         # Default AWS fallback
         url = self._generate_url(obj, "timedtext", extension="vtt")