Merge pull request #200 from Geode-solutions/feat/save_and_load

feat(save_and_load): add import/export methods

Author: BotellaA

opengeodeweb_back_schemas.json

@@ -332,6 +332,17 @@
       ],
       "additionalProperties": false
     },
+    "import_project": {
+      "$id": "opengeodeweb_back/import_project",
+      "route": "/import_project",
+      "methods": [
+        "POST"
+      ],
+      "type": "object",
+      "properties": {},
+      "required": [],
+      "additionalProperties": false
+    },
     "geographic_coordinate_systems": {
       "$id": "opengeodeweb_back/geographic_coordinate_systems",
       "route": "/geographic_coordinate_systems",
@@ -373,6 +384,28 @@
       ],
       "additionalProperties": false
     },
+    "export_project": {
+      "$id": "opengeodeweb_back/export_project",
+      "route": "/export_project",
+      "methods": [
+        "POST"
+      ],
+      "type": "object",
+      "properties": {
+        "snapshot": {
+          "type": "object"
+        },
+        "filename": {
+          "type": "string",
+          "minLength": 1
+        }
+      },
+      "required": [
+        "snapshot",
+        "filename"
+      ],
+      "additionalProperties": false
+    },
     "allowed_objects": {
       "$id": "opengeodeweb_back/allowed_objects",
       "route": "/allowed_objects",

requirements.txt

@@ -60,4 +60,3 @@ werkzeug==3.1.2
     #   flask
     #   flask-cors
 
-opengeodeweb-microservice==1.*,>=1.0.8

src/opengeodeweb_back/routes/blueprint_routes.py

@@ -1,16 +1,21 @@
 # Standard library imports
 import os
 import time
+import shutil
 
 # Third party imports
 import flask
 import werkzeug
+import zipfile
 from opengeodeweb_microservice.schemas import get_schemas_dict
 
 # Local application imports
 from opengeodeweb_back import geode_functions, utils_functions
 from .models import blueprint_models
 from . import schemas
+from opengeodeweb_microservice.database.data import Data
+from opengeodeweb_microservice.database.connection import get_session
+from opengeodeweb_microservice.database import connection
 
 routes = flask.Blueprint("routes", __name__, url_prefix="/opengeodeweb_back")
 
@@ -268,3 +273,141 @@ def kill() -> flask.Response:
     print("Manual server kill, shutting down...", flush=True)
     os._exit(0)
     return flask.make_response({"message": "Flask server is dead"}, 200)
+
+
+@routes.route(
+    schemas_dict["export_project"]["route"],
+    methods=schemas_dict["export_project"]["methods"],
+)
+def export_project() -> flask.Response:
+    utils_functions.validate_request(flask.request, schemas_dict["export_project"])
+    params = schemas.ExportProject.from_dict(flask.request.get_json())
+
+    project_folder: str = flask.current_app.config["DATA_FOLDER_PATH"]
+    os.makedirs(project_folder, exist_ok=True)
+
+    filename: str = werkzeug.utils.secure_filename(os.path.basename(params.filename))
+    if not filename.lower().endswith(".vease"):
+        flask.abort(400, "Requested filename must end with .vease")
+    export_vease_path = os.path.join(project_folder, filename)
+
+    with get_session() as session:
+        rows = session.query(Data.id, Data.input_file, Data.additional_files).all()
+
+    with zipfile.ZipFile(
+        export_vease_path, "w", compression=zipfile.ZIP_DEFLATED
+    ) as zip_file:
+        database_root_path = os.path.join(project_folder, "project.db")
+        if os.path.isfile(database_root_path):
+            zip_file.write(database_root_path, "project.db")
+
+        for data_id, input_file, additional_files in rows:
+            base_dir = os.path.join(project_folder, data_id)
+
+            input_path = os.path.join(base_dir, str(input_file))
+            if os.path.isfile(input_path):
+                zip_file.write(input_path, os.path.join(data_id, str(input_file)))
+
+            for relative_path in (
+                additional_files if isinstance(additional_files, list) else []
+            ):
+                additional_path = os.path.join(base_dir, relative_path)
+                if os.path.isfile(additional_path):
+                    zip_file.write(
+                        additional_path, os.path.join(data_id, relative_path)
+                    )
+
+        zip_file.writestr("snapshot.json", flask.json.dumps(params.snapshot))
+
+    return utils_functions.send_file(project_folder, [export_vease_path], filename)
+
+
+@routes.route(
+    schemas_dict["import_project"]["route"],
+    methods=schemas_dict["import_project"]["methods"],
+)
+def import_project() -> flask.Response:
+    utils_functions.validate_request(flask.request, schemas_dict["import_project"])
+    if "file" not in flask.request.files:
+        flask.abort(400, "No .vease file provided under 'file'")
+    zip_file = flask.request.files["file"]
+    assert zip_file.filename is not None
+    filename = werkzeug.utils.secure_filename(os.path.basename(zip_file.filename))
+    if not filename.lower().endswith(".vease"):
+        flask.abort(400, "Uploaded file must be a .vease")
+
+    data_folder_path: str = flask.current_app.config["DATA_FOLDER_PATH"]
+
+    # 423 Locked bypass : remove stopped requests
+    if connection.scoped_session_registry:
+        connection.scoped_session_registry.remove()
+    if connection.engine:
+        connection.engine.dispose()
+    connection.engine = connection.session_factory = (
+        connection.scoped_session_registry
+    ) = None
+
+    try:
+        if os.path.exists(data_folder_path):
+            shutil.rmtree(data_folder_path)
+        os.makedirs(data_folder_path, exist_ok=True)
+    except PermissionError:
+        flask.abort(423, "Project files are locked; cannot overwrite")
+
+    zip_file.stream.seek(0)
+    with zipfile.ZipFile(zip_file.stream) as zip_archive:
+        project_folder = os.path.abspath(data_folder_path)
+        for member in zip_archive.namelist():
+            target = os.path.abspath(
+                os.path.normpath(os.path.join(project_folder, member))
+            )
+            if not (
+                target == project_folder or target.startswith(project_folder + os.sep)
+            ):
+                flask.abort(400, "Vease file contains unsafe paths")
+        zip_archive.extractall(project_folder)
+
+        database_root_path = os.path.join(project_folder, "project.db")
+        if not os.path.isfile(database_root_path):
+            flask.abort(400, "Missing project.db at project root")
+
+        connection.init_database(database_root_path, create_tables=False)
+
+        try:
+            with get_session() as session:
+                rows = session.query(Data).all()
+        except Exception:
+            connection.init_database(database_root_path, create_tables=True)
+            with get_session() as session:
+                rows = session.query(Data).all()
+
+        with get_session() as session:
+            for data_entry in rows:
+                data_path = geode_functions.data_file_path(data_entry.id)
+                viewable_name = data_entry.viewable_file_name
+                if viewable_name:
+                    vpath = geode_functions.data_file_path(data_entry.id, viewable_name)
+                    if os.path.isfile(vpath):
+                        continue
+
+                input_file = str(data_entry.input_file or "")
+                if not input_file:
+                    continue
+
+                input_full = geode_functions.data_file_path(data_entry.id, input_file)
+                if not os.path.isfile(input_full):
+                    continue
+
+                data_object = geode_functions.load(data_entry.geode_object, input_full)
+                utils_functions.save_all_viewables_and_return_info(
+                    data_entry.geode_object, data_object, data_entry, data_path
+                )
+            session.commit()
+
+        snapshot = {}
+        try:
+            raw = zip_archive.read("snapshot.json").decode("utf-8")
+            snapshot = flask.json.loads(raw)
+        except KeyError:
+            snapshot = {}
+    return flask.make_response({"snapshot": snapshot}, 200)

src/opengeodeweb_back/routes/schemas/__init__.py

@@ -8,7 +8,9 @@
 from .missing_files import *
 from .kill import *
 from .inspect_file import *
+from .import_project import *
 from .geographic_coordinate_systems import *
 from .geode_objects_and_output_extensions import *
+from .export_project import *
 from .allowed_objects import *
 from .allowed_files import *

src/opengeodeweb_back/routes/schemas/export_project.json

@@ -0,0 +1,21 @@
+{
+  "route": "/export_project",
+  "methods": [
+    "POST"
+  ],
+  "type": "object",
+  "properties": {
+    "snapshot": {
+      "type": "object"
+    },
+    "filename": {
+      "type": "string",
+      "minLength": 1
+    }
+  },
+  "required": [
+    "snapshot",
+    "filename"
+  ],
+  "additionalProperties": false
+}

src/opengeodeweb_back/routes/schemas/export_project.py

@@ -0,0 +1,9 @@
+from dataclasses_json import DataClassJsonMixin
+from dataclasses import dataclass
+from typing import Dict, Any
+
+
+@dataclass
+class ExportProject(DataClassJsonMixin):
+    filename: str
+    snapshot: Dict[str, Any]

src/opengeodeweb_back/routes/schemas/import_project.json

@@ -0,0 +1,10 @@
+{
+  "route": "/import_project",
+  "methods": [
+    "POST"
+  ],
+  "type": "object",
+  "properties": {},
+  "required": [],
+  "additionalProperties": false
+}

src/opengeodeweb_back/routes/schemas/import_project.py

@@ -0,0 +1,7 @@
+from dataclasses_json import DataClassJsonMixin
+from dataclasses import dataclass
+
+
+@dataclass
+class ImportProject(DataClassJsonMixin):
+    pass

src/opengeodeweb_back/utils_functions.py

@@ -136,15 +136,17 @@ def send_file(
     else:
         mimetype = "application/zip"
         new_file_name = os.path.splitext(new_file_name)[0] + ".zip"
-        with zipfile.ZipFile(os.path.join(upload_folder, new_file_name), "w") as zipObj:
+        with zipfile.ZipFile(
+            os.path.join(os.path.abspath(upload_folder), new_file_name), "w"
+        ) as zipObj:
             for saved_file_path in saved_files:
                 zipObj.write(
                     saved_file_path,
                     os.path.basename(saved_file_path),
                 )
 
     response = flask.send_from_directory(
-        directory=upload_folder,
+        directory=os.path.abspath(upload_folder),
         path=new_file_name,
         as_attachment=True,
         mimetype=mimetype,