add support for multiple local and S3 storage backends

Author: jrief

demoapp/models.py

@@ -9,4 +9,5 @@ class DemoAppModel(models.Model):
         null=True,
         blank=True,
         accept_mime_types=['image/*'],
+        realm='admin',
     )

demoapp/settings.py

@@ -95,6 +95,50 @@
         },
     }
 
+STORAGES = {
+    'default': {
+        'BACKEND': 'django.core.files.storage.FileSystemStorage',
+    },
+    'staticfiles': {
+        'BACKEND': 'django.contrib.staticfiles.storage.StaticFilesStorage',
+    },
+    'finder_public': {
+        'BACKEND': 'finder.storage.FinderSystemStorage',
+        'OPTIONS': {
+            'location': BASE_DIR / 'workdir/media/filer_public',
+            'base_url': '/media/filer_public/',
+            'allow_overwrite': True,
+        },
+    },
+    'finder_public_samples': {
+        'BACKEND': 'finder.storage.FinderSystemStorage',
+        'OPTIONS': {
+            'location': BASE_DIR / 'workdir/media/filer_public_thumbnails',
+            'base_url': '/media/filer_public_thumbnails/',
+            'allow_overwrite': True,
+        },
+    },
+}
+
+if False:
+    STORAGES['finder_public'] = {
+        'BACKEND': 'storages.backends.s3.S3Storage',
+        'OPTIONS': {
+            'access_key': 'finder',
+            'secret_key': 'finderadmin',
+            'bucket_name': 'finder-public',
+            'endpoint_url': 'http://localhost:9000',
+        },
+    }
+    STORAGES['finder_public_samples'] = {
+        'BACKEND': 'storages.backends.s3.S3Storage',
+        'OPTIONS': {
+            'access_key': 'finder',
+            'secret_key': 'finderadmin',
+            'bucket_name': 'finder-public-samples',
+            'endpoint_url': 'http://localhost:9000',
+        },
+    }
 
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 

finder/admin/file.py

@@ -75,13 +75,14 @@ def replace_file(self, request, file_id):
             return HttpResponseNotFound(f"File {file_id} not found.")
         if request.content_type != 'multipart/form-data' or 'upload_file' not in request.FILES:
             return HttpResponseBadRequest(f"Content-Type {request.content_type} invalid for file upload.")
+        realm = self.get_realm(request)
         uploaded_file = request.FILES['upload_file']
         if uploaded_file.content_type != file_obj.mime_type:
             return HttpResponseBadRequest(f"Can not replace file {file_obj.name} with different mime type.")
         # the payload of the file_obj is not replaced and remains orphaned, it may be deleted
         file_obj.file_name = file_obj.generate_filename(uploaded_file.name)
         file_obj.file_size = uploaded_file.size
-        file_obj.receive_file(uploaded_file)
+        file_obj.receive_file(realm, uploaded_file)
         file_obj.save()
         return HttpResponse(f"Replaced content of {file_obj.name} successfully.")
 
@@ -113,10 +114,11 @@ def render_change_form(self, request, context, add=False, change=False, form_url
 
     def get_editor_settings(self, request, inode):
         settings = super().get_editor_settings(request, inode)
+        realm = self.get_realm(request)
         settings.update(
             base_url=reverse('admin:finder_filemodel_changelist', current_app=self.admin_site.name),
-            download_url=inode.get_download_url(),
-            thumbnail_url=inode.get_thumbnail_url(),
+            download_url=inode.get_download_url(realm),
+            thumbnail_url=inode.get_thumbnail_url(realm),
             file_id=inode.id,
             filename=inode.file_name,
             file_mime_type=inode.mime_type,

finder/admin/folder.py

@@ -219,17 +219,20 @@ def upload_file(self, request, folder_id):
             return HttpResponseNotFound(f"FolderModel<{folder_id}> not found.")
         if request.content_type != 'multipart/form-data' or 'upload_file' not in request.FILES:
             return HttpResponse("Bad encoding type or missing payload.", status=415)
+        realm = self.get_realm(request)
         model = FileModel.objects.get_model_for(request.FILES['upload_file'].content_type)
         new_file = model.objects.create_from_upload(
+            realm,
             request.FILES['upload_file'],
             folder=folder,
             owner=request.user,
         )
-        return JsonResponse({'file_info': new_file.as_dict})
+        return JsonResponse({'file_info': new_file.as_dict(realm)})
 
     def update_inode(self, request, folder_id):
         if response := self.check_for_valid_post_request(request, folder_id):
             return response
+        realm = self.get_realm(request)
         body = json.loads(request.body)
         try:
             inode_obj = self.get_object(request, body['id'])
@@ -252,7 +255,7 @@ def update_inode(self, request, folder_id):
         if update_values:
             inode_obj.save(update_fields=list(update_values.keys()))
         return JsonResponse({
-            'new_inode': self.serialize_inode(inode_obj),
+            'new_inode': self.serialize_inode(realm, inode_obj),
             'favorite_folders': self.get_favorite_folders(request, current_folder),
         })
 
@@ -280,7 +283,10 @@ def move_inodes(self, request, folder_id):
         body = json.loads(request.body)
         current_folder = self.get_object(request, folder_id)
         if 'target_id' in body:
-            target_folder = self.get_object(request, body['target_id'])
+            if body['target_id'] == 'parent':
+                target_folder = current_folder.parent
+            else:
+                target_folder = self.get_object(request, body['target_id'])
             if not target_folder:
                 msg = gettext("Folder named “{folder}” not found.")
                 return HttpResponseNotFound(msg.format(folder=body['target_id']))
@@ -315,23 +321,25 @@ def delete_inodes(self, request, folder_id):
         trash_folder = self.get_trash_folder(request)
         if current_folder.id == trash_folder.id:
             return HttpResponse("Cannot move inodes from trash folder into itself.", status=409)
+        realm = self.get_realm(request)
         trash_ordering = trash_folder.get_max_ordering()
         inode_ids = body.get('inode_ids', [])
         for entry in FolderModel.objects.filter_unified(id__in=inode_ids):
             inode = FolderModel.objects.get_inode(id=entry['id'])
+            update_fields = ['parent', 'ordering']
             if entry['is_folder']:
                 PinnedFolder.objects.filter(folder=inode).delete()
                 while trash_folder.listdir(name=inode.name, is_folder=True).exists():
                     inode.name = f"{inode.name}.renamed"
-                inode.save(update_fields=['name'])
+                update_fields.append('name')
             DiscardedInode.objects.create(
                 inode=inode.id,
                 previous_parent=inode.parent,
             )
             trash_ordering += 1
             inode.ordering = trash_ordering
             inode.parent = trash_folder
-            inode.save(update_fields=['parent'])
+            inode.save(update_fields=update_fields)
         current_folder.reorder()
         return JsonResponse({
             'favorite_folders': self.get_favorite_folders(request, current_folder),
@@ -354,12 +362,16 @@ def undo_discarded_inodes(self, request):
     def erase_trash_folder(self, request):
         if request.method != 'DELETE':
             return HttpResponseNotAllowed(f"Method {request.method} not allowed. Only DELETE requests are allowed.")
+        realm = self.get_realm(request)
         trash_folder_entries = self.get_trash_folder(request).listdir()
         DiscardedInode.objects.filter(inode__in=list(trash_folder_entries.values_list('id', flat=True))).delete()
         for entry in trash_folder_entries:
-            # bulk delete does not work here because file must be erased from disk
+            # bulk delete does not work here because each file must be erased from disk
             proxy_obj = InodeManager.get_proxy_object(entry)
-            proxy_obj.delete()
+            if proxy_obj.is_folder:
+                proxy_obj.delete()
+            else:
+                proxy_obj.erase_and_delete(realm)
         fallback_folder = self.get_fallback_folder(request)
         return JsonResponse({
             'success_url': reverse(
@@ -373,6 +385,7 @@ def add_folder(self, request, folder_id):
             return response
         parent_folder = self.get_object(request, folder_id)
         assert parent_folder.is_folder
+        realm = self.get_realm(request)
         body = json.loads(request.body)
         if parent_folder.listdir(name=body['name'], is_folder=True).exists():
             msg = gettext("A folder named “{name}” already exists.")
@@ -383,13 +396,14 @@ def add_folder(self, request, folder_id):
             owner=request.user,
             ordering=parent_folder.get_max_ordering() + 1,
         )
-        return JsonResponse({'new_folder': self.serialize_inode(new_folder)})
+        return JsonResponse({'new_folder': self.serialize_inode(realm, new_folder)})
 
     def get_or_create_folder(self, request, folder_id):
         if response := self.check_for_valid_post_request(request, folder_id):
             return response
         if not (folder := self.get_object(request, folder_id)):
             return HttpResponseNotFound(f"FolderModel<{folder_id}> not found.")
+        realm = self.get_realm(request)
         ordering = folder.get_max_ordering() + 1
         body = json.loads(request.body)
         for folder_name in body['relative_path'].split('/'):
@@ -400,4 +414,4 @@ def get_or_create_folder(self, request, folder_id):
             )
             if created:
                 ordering += 1
-        return JsonResponse({'folder': self.serialize_inode(folder)})
+        return JsonResponse({'folder': self.serialize_inode(realm, folder)})

finder/admin/inode.py

@@ -78,7 +78,7 @@ def toggle_pin(self, request, folder_id):
             'favorite_folders': self.get_favorite_folders(request, current_folder),
         })
 
-    def serialize_inode(self, inode):
+    def serialize_inode(self, realm, inode):
         data = {field: inode.serializable_value(field) for field in inode.data_fields}
         data.update(
             owner_name=inode.owner.username if inode.owner else None,
@@ -88,8 +88,8 @@ def serialize_inode(self, inode):
                 args=(inode.id,),
                 current_app=self.admin_site.name,
             ),
-            download_url=inode.get_download_url(),
-            thumbnail_url=inode.get_thumbnail_url(),
+            download_url=inode.get_download_url(realm),
+            thumbnail_url=inode.get_thumbnail_url(realm),
             summary=inode.summary,
         )
         if (inode.is_folder):
@@ -129,9 +129,10 @@ def get_inodes(self, request, **lookup):
         Return a serialized list of file/folder-s for the given folder.
         """
         lookup = dict(lookup_by_label(request), **lookup)
+        realm = self.get_realm(request)
         unified_queryset = FolderModel.objects.filter_unified(**lookup)
         unified_queryset = sort_by_attribute(request, unified_queryset)
-        self.annotate_unified_queryset(unified_queryset)
+        self.annotate_unified_queryset(realm, unified_queryset)
         return unified_queryset
 
     def get_breadcrumbs(self, obj):
@@ -147,9 +148,9 @@ def get_breadcrumbs(self, obj):
         return breadcrumbs
 
     def get_favorite_folders(self, request, current_folder):
-        site = get_current_site(request)
+        realm = self.get_realm(request)
         folders = PinnedFolder.objects.filter(
-            realm__site=site,
+            realm__site=realm.site,
             realm__slug=self.admin_site.name,
             owner=request.user,
         ).values(
@@ -174,15 +175,15 @@ def get_favorite_folders(self, request, current_folder):
         for folder in folders:
             if folder['id'] == current_folder.id:
                 if len(folders) == 0:
-                    folders.append(self.serialize_inode(fallback_folder))
+                    folders.append(self.serialize_inode(realm, fallback_folder))
                 break
         else:
             if current_folder.id == root_folder.id:
-                folders.insert(0, self.serialize_inode(current_folder))
+                folders.insert(0, self.serialize_inode(realm, current_folder))
             elif current_folder.id != trash_folder.id:
-                folders.append(self.serialize_inode(current_folder))
+                folders.append(self.serialize_inode(realm, current_folder))
         if trash_folder.num_children > 0:
-            inode_data = self.serialize_inode(trash_folder)
+            inode_data = self.serialize_inode(realm, trash_folder)
             inode_data.update(is_trash=True)
             folders.append(inode_data)
         return folders
@@ -239,8 +240,8 @@ def get_menu_extension_settings(self, request):
         """
         return {}
 
-    def annotate_unified_queryset(self, queryset):
-        annotate_unified_queryset(queryset)
+    def annotate_unified_queryset(self, realm, queryset):
+        annotate_unified_queryset(realm, queryset)
         for entry in queryset:
             entry.update(
                 change_url=reverse(

finder/browser/views.py

@@ -1,6 +1,5 @@
 from django.contrib.sites.shortcuts import get_current_site
 from django.core.exceptions import BadRequest, ObjectDoesNotExist
-from django.core.files.storage import default_storage
 from django.db.models import QuerySet, Subquery
 from django.forms.renderers import DjangoTemplates
 from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotFound
@@ -68,7 +67,7 @@ def _get_realm(self, request, slug):
             raise ObjectDoesNotExist(f"Realm named {slug} not found for {site.domain}.")
 
     @method_decorator(require_GET)
-    def structure(self, request, slug):
+    def structure(self, request, slug=None):
         realm = self._get_realm(request, slug)
         root_folder_id = str(realm.root_folder.id)
         request.session.setdefault('finder.open_folders', [])
@@ -126,7 +125,8 @@ def fetch(self, request, inode_id):
                 'has_subfolders': inode.subfolders.exists(),
             }
         else:
-            return inode.as_dict
+            realm = inode.folder.get_realm()
+            return inode.as_dict(realm)
 
     @method_decorator(require_GET)
     def open(self, request, folder_id):
@@ -164,20 +164,22 @@ def list(self, request, folder_id):
         recursive = 'recursive' in request.GET
         lookup = lookup_by_label(request)
         lookup['mime_types'] = request.GET.getlist('mimetypes')
+        current_folder = FolderModel.objects.get(id=folder_id)
         if recursive:
-            descendants = FolderModel.objects.get(id=folder_id).descendants
-            if isinstance(descendants, QuerySet):
-                parent_ids = Subquery(descendants.values('id'))
+            if isinstance(current_folder.descendants, QuerySet):
+                parent_ids = Subquery(current_folder.descendants.values('id'))
             else:
-                parent_ids = [descendant.id for descendant in descendants]
+                parent_ids = [descendant.id for descendant in current_folder.descendants]
             unified_queryset = FileModel.objects.filter_unified(parent_id__in=parent_ids, is_folder=False, **lookup)
         else:
             unified_queryset = FileModel.objects.filter_unified(parent_id=folder_id, is_folder=False, **lookup)
         next_offset = offset + self.limit
         if next_offset >= unified_queryset.count():
             next_offset = None
+
+        realm = current_folder.get_realm()
         unified_queryset = sort_by_attribute(request, unified_queryset)
-        annotate_unified_queryset(unified_queryset)
+        annotate_unified_queryset(realm, unified_queryset)
         return {
             'files': list(unified_queryset[offset:offset + self.limit]),
             'offset': next_offset,
@@ -203,6 +205,7 @@ def search(self, request, folder_id):
         else:  # django-cte not installed (slow)
             parent_ids = [descendant.id for descendant in starting_folder.descendants]
 
+        realm = starting_folder.get_realm()
         lookup = {
             'parent_id__in': parent_ids,
             'name_lower__icontains': search_query,
@@ -212,7 +215,7 @@ def search(self, request, folder_id):
             next_offset = offset + self.limit
         else:
             next_offset = None
-        annotate_unified_queryset(unified_queryset)
+        annotate_unified_queryset(realm, unified_queryset)
         return {
             'files': unified_queryset[offset:next_offset],
             'offset': next_offset,
@@ -226,16 +229,18 @@ def upload(self, request, folder_id):
         if request.content_type != 'multipart/form-data' or 'upload_file' not in request.FILES:
             raise BadRequest("Bad form encoding or missing payload.")
         model = FileModel.objects.get_model_for(request.FILES['upload_file'].content_type)
-        folder = FolderModel.objects.get(id=folder_id)
+        current_folder = FolderModel.objects.get(id=folder_id)
+        realm = current_folder.get_realm()
         file = model.objects.create_from_upload(
+            realm,
             request.FILES['upload_file'],
-            folder=folder,
+            folder=current_folder,
             owner=request.user,
         )
         form_class = file.get_form_class()
         form = form_class(instance=file, renderer=FormRenderer())
         response = {
-            'file_info': file.as_dict,
+            'file_info': file.as_dict(realm),
             'form_html': mark_safe(strip_spaces_between_tags(form.as_div())),
         }
         return response
@@ -251,11 +256,12 @@ def change(self, request, file_id):
             return {'file_info': None}
         if request.content_type != 'multipart/form-data':
             raise BadRequest("Bad form encoding or missing payload.")
+        realm = file.folder.get_realm()
         form_class = file.get_form_class()
         form = form_class(instance=file, data=request.POST, renderer=FormRenderer())
         if form.is_valid():
             file = form.save()
-            return {'file_info': file.as_dict}
+            file.as_dict.cache_clear()
+            return {'file_info': file.as_dict(realm)}
         else:
             return {'form_html': mark_safe(strip_spaces_between_tags(form.as_div()))}
-