Implement media archive upload endpoint (#391)

* Implement media archiv upload endpoint

* Fix tests

* Fix tests

* Another try

* Version -> 1.1.0

* Fix tests

Author: DavidMStraub

gramps_webapi/_version.py

@@ -18,4 +18,4 @@
 #
 
 # make sure to match this version with the one in apispec.yaml
-__version__ = "1.0.0"
+__version__ = "1.1.0"

gramps_webapi/api/__init__.py

@@ -46,6 +46,7 @@
 from .resources.file import MediaFileResource
 from .resources.filters import FilterResource, FiltersResource, FiltersResources
 from .resources.holidays import HolidayResource, HolidaysResource
+from .resources.import_media import MediaUploadZipResource
 from .resources.importers import (
     ImporterFileResource,
     ImporterResource,
@@ -344,6 +345,13 @@ def register_endpt(resource: Type[Resource], url: str, name: str):
     "media_archive_filename",
 )
 
+# Media import
+register_endpt(
+    MediaUploadZipResource,
+    "/media/archive/upload/zip",
+    "media_archive_upload_zip",
+)
+
 
 # Thumbnails
 @api_blueprint.route("/media/<string:handle>/thumbnail/<int:size>")

gramps_webapi/api/file.py

@@ -132,7 +132,10 @@ def _check_path(self) -> None:
 
     def file_exists(self) -> bool:
         """Check if the file exists."""
-        self._check_path()
+        try:
+            self._check_path()
+        except ValueError:
+            return False
         return Path(self.path_abs).is_file()
 
     def get_file_object(self) -> BinaryIO:

gramps_webapi/api/media.py

@@ -88,7 +88,7 @@ def filter_existing_files(
         """Given a list of media objects, return the ones with existing files."""
         raise NotImplementedError
 
-    def get_media_size(self) -> int:
+    def get_media_size(self, db_handle: Optional[DbReadBase] = None) -> int:
         """Return the total disk space used by all existing media objects."""
         raise NotImplementedError
 
@@ -136,16 +136,17 @@ def filter_existing_files(
             if self.get_file_handler(obj.handle, db_handle=db_handle).file_exists()
         ]
 
-    def get_media_size(self) -> int:
+    def get_media_size(self, db_handle: Optional[DbReadBase] = None) -> int:
         """Return the total disk space used by all existing media objects.
 
         Only works with a request context.
         """
+        if not db_handle:
+            db_handle = get_db_handle()
         if not os.path.isdir(self.base_dir):
             raise ValueError(f"Directory {self.base_dir} does not exist")
         size = 0
         paths_seen = set()
-        db_handle = get_db_handle()
         for obj in db_handle.iter_media():
             path = obj.path
             if os.path.isabs(path):
@@ -257,9 +258,10 @@ def filter_existing_files(
         remote_keys = self.get_remote_keys()
         return [obj for obj in objects if obj.checksum in remote_keys]
 
-    def get_media_size(self) -> int:
+    def get_media_size(self, db_handle: Optional[DbReadBase] = None) -> int:
         """Return the total disk space used by all existing media objects."""
-        db_handle = get_db_handle()
+        if not db_handle:
+            db_handle = get_db_handle()
         keys = set(obj.checksum for obj in db_handle.iter_media())
         keys_size = get_object_keys_size(
             bucket_name=self.bucket_name,

gramps_webapi/api/resources/export_media.py

@@ -17,7 +17,7 @@
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
 
-"""Endpoint for up- and downloading media files."""
+"""Endpoint for creating and downloading media archives."""
 
 import os
 import re

gramps_webapi/api/resources/import_media.py

@@ -0,0 +1,70 @@
+#
+# Gramps Web API - A RESTful API for the Gramps genealogy program
+#
+# Copyright (C) 2023    David Straub
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+"""Endpoint for importing a media archive."""
+
+import os
+import uuid
+import zipfile
+
+from flask import Response, current_app, jsonify, request
+
+from ...auth.const import PERM_IMPORT_FILE
+from ..auth import require_permissions
+from ..tasks import AsyncResult, import_media_archive, make_task_response, run_task
+from ..util import abort_with_message, get_tree_from_jwt
+from . import ProtectedResource
+
+
+class MediaUploadZipResource(ProtectedResource):
+    """Resource for uploading an archive of media files."""
+
+    def post(self) -> Response:
+        """Upload an archive of media files."""
+        require_permissions([PERM_IMPORT_FILE])
+        request_stream = request.stream
+
+        # we use EXPORT_DIR as location to store the temporary file
+        export_path = current_app.config["EXPORT_DIR"]
+        os.makedirs(export_path, exist_ok=True)
+        file_name = f"{uuid.uuid4()}.zip"
+        file_path = os.path.join(export_path, file_name)
+
+        with open(file_path, "w+b") as ftmp:
+            ftmp.write(request_stream.read())
+
+        if os.path.getsize(file_path) == 0:
+            abort_with_message(400, "Imported file is empty")
+
+        try:
+            with zipfile.ZipFile(file_path) as zip_file:
+                zip_file.namelist()
+        except zipfile.BadZipFile:
+            abort_with_message(400, "The uploaded file is not a valid ZIP file.")
+
+        tree = get_tree_from_jwt()
+        task = run_task(
+            import_media_archive,
+            tree=tree,
+            file_name=file_path,
+            delete=True,
+        )
+        if isinstance(task, AsyncResult):
+            return make_task_response(task)
+        return jsonify(task), 201

gramps_webapi/api/resources/util.py

@@ -23,6 +23,9 @@
 
 import json
 import os
+import shutil
+import tempfile
+import zipfile
 from hashlib import sha256
 from http import HTTPStatus
 from typing import Any, Dict, List, Optional, Tuple, Union
@@ -42,19 +45,15 @@
     Citation,
     Event,
     Family,
-    GrampsType,
     Media,
-    Note,
     Person,
     Place,
     PlaceType,
-    Repository,
     Source,
     Span,
-    Tag,
 )
 from gramps.gen.lib.primaryobj import BasicPrimaryObject as GrampsObject
-from gramps.gen.lib.serialize import from_json, to_json
+from gramps.gen.lib.serialize import to_json
 from gramps.gen.plug import BasePluginManager
 from gramps.gen.relationship import get_relationship_calculator
 from gramps.gen.soundex import soundex
@@ -70,9 +69,11 @@
 from gramps.gen.utils.id import create_id
 from gramps.gen.utils.place import conv_lat_lon
 
+from ...auth import set_tree_usage
 from ...const import DISABLED_IMPORTERS, SEX_FEMALE, SEX_MALE, SEX_UNKNOWN
 from ...types import FilenameOrPath, Handle, TransactionJson
-from ..media import get_media_handler
+from ..file import get_checksum
+from ..media import check_quota_media, get_media_handler
 from ..util import abort_with_message, get_db_handle, get_tree_from_jwt
 
 pd = PlaceDisplay()
@@ -1209,3 +1210,96 @@ def dry_run_import(
         "notes": db_handle.get_number_of_notes(),
         "tags": db_handle.get_number_of_tags(),
     }
+
+
+def run_import_media_archive(
+    tree: str,
+    db_handle: DbReadBase,
+    file_name: FilenameOrPath,
+    delete: bool = True,
+) -> Dict[str, int]:
+    """Import a media archive file."""
+    media_handler = get_media_handler(db_handle, tree=tree)
+
+    # create a dict {checksum: [(handle1, path), (handle2, path2), ...], ...}
+    # of missing files
+    handles = db_handle.get_media_handles()
+    objects = [db_handle.get_media_from_handle(handle) for handle in handles]
+    objects_existing = media_handler.filter_existing_files(objects, db_handle=db_handle)
+    handles_existing = set(obj.handle for obj in objects_existing)
+    objects_missing = [obj for obj in objects if obj.handle not in handles_existing]
+
+    checksums_handles: Dict[str, List[Tuple[str, str, str]]] = {}
+    for obj in objects_missing:
+        if obj.checksum not in checksums_handles:
+            checksums_handles[obj.checksum] = []
+        obj_details = (obj.handle, obj.get_path(), obj.get_mime_type())
+        checksums_handles[obj.checksum].append(obj_details)
+    if len(checksums_handles) == 0:
+        # no missing files
+        # delete ZIP file
+        if delete:
+            os.remove(file_name)
+        return {"missing": 0, "uploaded": 0, "failures": 0}
+
+    total_size = 0
+    with zipfile.ZipFile(file_name, "r") as zip_file:
+        # compute file size
+        for file_info in zip_file.infolist():
+            total_size += file_info.file_size
+
+        # check disk usage
+        disk_usage = shutil.disk_usage(file_name)
+        if total_size > disk_usage.free:
+            raise ValueError("Not enough free space on disk")
+
+        # extract
+        temp_dir = tempfile.mkdtemp()
+        zip_file.extractall(temp_dir)
+
+    # delete ZIP file
+    if delete:
+        os.remove(file_name)
+
+    to_upload = {}
+    # walk extracted files
+    for root, _, files in os.walk(temp_dir):
+        for name in files:
+            file_path = os.path.join(root, name)
+            with open(file_path, "rb") as f:
+                checksum = get_checksum(f)
+                if checksum in checksums_handles and checksum not in to_upload:
+                    to_upload[checksum] = (file_path, os.path.getsize(file_path))
+
+    if len(to_upload) == 0:
+        # no files to upload
+
+        # delete extracted temp files
+        shutil.rmtree(temp_dir)
+
+        return {"missing": len(checksums_handles), "uploaded": 0, "failures": 0}
+
+    upload_size = sum([file_size for (file_path, file_size) in to_upload.values()])
+    check_quota_media(to_add=upload_size, tree=tree)
+
+    num_failures = 0
+    for checksum, (file_path, file_size) in to_upload.items():
+        for handle, media_path, mime in checksums_handles[checksum]:
+            with open(file_path, "rb") as f:
+                try:
+                    media_handler.upload_file(f, checksum, mime, path=media_path)
+                except Exception:
+                    num_failures += 1
+
+    # delete extracted temp files
+    shutil.rmtree(temp_dir)
+
+    # update media usage
+    usage_media = media_handler.get_media_size(db_handle=db_handle)
+    set_tree_usage(tree, usage_media=usage_media)
+
+    return {
+        "missing": len(checksums_handles),
+        "uploaded": len(to_upload) - num_failures,
+        "failures": num_failures,
+    }

gramps_webapi/api/tasks.py

@@ -33,7 +33,7 @@
 from .export import prepare_options, run_export
 from .media import get_media_handler
 from .report import run_report
-from .resources.util import dry_run_import, run_import
+from .resources.util import dry_run_import, run_import, run_import_media_archive
 from .util import (
     check_quota_people,
     get_config,
@@ -201,3 +201,17 @@ def export_media(tree: str, view_private: bool) -> Dict[str, Union[str, int]]:
         "url": f"/api/media/archive/{file_name}",
         "file_size": file_size,
     }
+
+
+@shared_task()
+def import_media_archive(tree: str, file_name: str, delete: bool = True):
+    """Import a media archive."""
+    # check_quota_people(to_add=object_counts["people"], tree=tree)
+    db_handle = get_db_outside_request(tree=tree, view_private=True, readonly=True)
+    result = run_import_media_archive(
+        tree=tree,
+        db_handle=db_handle,
+        file_name=file_name,
+        delete=delete,
+    )
+    return result

gramps_webapi/data/apispec.yaml

@@ -8,7 +8,7 @@ info:
 
 
     * More about Gramps and the numerous features it provides for genealogists can be found at https://gramps-project.org
-  version: "1.0.0"  # make sure to match this version with the one in _version.py
+  version: "1.1.0"  # make sure to match this version with the one in _version.py
   license:
     name: "GNU Affero General Public License v3.0"
     url: "http://www.gnu.org/licenses/agpl-3.0.html"
@@ -3927,6 +3927,29 @@ paths:
         422:
           description: "Unprocessable Entity: Invalid or bad parameter provided."
 
+  /media/archive/upload/zip:
+    post:
+      tags:
+      - media
+      summary: "Upload a zipped media file archive."
+      operationId: uploadMediaFileArchive
+      security:
+        - Bearer: []
+      responses:
+        201:
+          description: "OK: Successful operation."
+        202:
+          description: "Accepted: import will be processed in the background."
+          schema:
+            type: object
+            properties:
+              task:
+                $ref: "#/definitions/TaskReference"
+        401:
+          description: "Unauthorized: Missing authorization header."
+        403:
+          description: "Unauthorized: insufficient permissions."
+
 ##############################################################################
 # Endpoint - Notes
 ##############################################################################