diff --git a/opengeodeweb_back_schemas.json b/opengeodeweb_back_schemas.json index ee3d0db..e5c8d4c 100644 --- a/opengeodeweb_back_schemas.json +++ b/opengeodeweb_back_schemas.json @@ -296,6 +296,17 @@ ], "additionalProperties": false }, + "import_project": { + "$id": "opengeodeweb_back/import_project", + "route": "/import_project", + "methods": [ + "POST" + ], + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": false + }, "geographic_coordinate_systems": { "$id": "opengeodeweb_back/geographic_coordinate_systems", "route": "/geographic_coordinate_systems", @@ -337,6 +348,28 @@ ], "additionalProperties": false }, + "export_project": { + "$id": "opengeodeweb_back/export_project", + "route": "/export_project", + "methods": [ + "POST" + ], + "type": "object", + "properties": { + "snapshot": { + "type": "object" + }, + "filename": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "snapshot", + "filename" + ], + "additionalProperties": false + }, "allowed_objects": { "$id": "opengeodeweb_back/allowed_objects", "route": "/allowed_objects", diff --git a/requirements.txt b/requirements.txt index 89b35fa..4e85944 100644 --- a/requirements.txt +++ b/requirements.txt @@ -60,4 +60,3 @@ werkzeug==3.1.2 # flask # flask-cors -opengeodeweb-microservice==1.*,>=1.0.6rc1 diff --git a/src/opengeodeweb_back/routes/blueprint_routes.py b/src/opengeodeweb_back/routes/blueprint_routes.py index 1f17f82..3271686 100644 --- a/src/opengeodeweb_back/routes/blueprint_routes.py +++ b/src/opengeodeweb_back/routes/blueprint_routes.py @@ -1,16 +1,21 @@ # Standard library imports import os import time +import shutil # Third party imports import flask import werkzeug +import zipfile from opengeodeweb_microservice.schemas import get_schemas_dict # Local application imports from opengeodeweb_back import geode_functions, utils_functions from .models import blueprint_models from . import schemas +from opengeodeweb_microservice.database.data import Data +from opengeodeweb_microservice.database.connection import get_session +from opengeodeweb_microservice.database import connection routes = flask.Blueprint("routes", __name__, url_prefix="/opengeodeweb_back") @@ -267,3 +272,142 @@ def kill() -> flask.Response: print("Manual server kill, shutting down...", flush=True) os._exit(0) return flask.make_response({"message": "Flask server is dead"}, 200) + + +@routes.route( + schemas_dict["export_project"]["route"], + methods=schemas_dict["export_project"]["methods"], +) +def export_project() -> flask.Response: + utils_functions.validate_request(flask.request, schemas_dict["export_project"]) + params = schemas.ExportProject.from_dict(flask.request.get_json()) + + project_folder: str = flask.current_app.config["DATA_FOLDER_PATH"] + os.makedirs(project_folder, exist_ok=True) + + filename: str = werkzeug.utils.secure_filename(os.path.basename(params.filename)) + export_zip_path = os.path.join(project_folder, filename) + + with get_session() as session: + rows = session.query(Data.id, Data.input_file, Data.additional_files).all() + + with zipfile.ZipFile( + export_zip_path, "w", compression=zipfile.ZIP_DEFLATED + ) as zip_file: + database_root_path = os.path.join(project_folder, "project.db") + if os.path.isfile(database_root_path): + zip_file.write(database_root_path, "project.db") + + for data_id, input_file, additional_files in rows: + base_dir = os.path.join(project_folder, data_id) + + input_path = os.path.join(base_dir, str(input_file)) + if os.path.isfile(input_path): + zip_file.write(input_path, os.path.join(data_id, str(input_file))) + + for relative_path in ( + additional_files if isinstance(additional_files, list) else [] + ): + additional_path = os.path.join(base_dir, relative_path) + if os.path.isfile(additional_path): + zip_file.write( + additional_path, os.path.join(data_id, relative_path) + ) + + zip_file.writestr("snapshot.json", flask.json.dumps(params.snapshot)) + + return utils_functions.send_file(project_folder, [export_zip_path], filename) + + +@routes.route( + schemas_dict["import_project"]["route"], + methods=schemas_dict["import_project"]["methods"], +) +def import_project() -> flask.Response: + if flask.request.method == "OPTIONS": + return flask.make_response({}, 200) + utils_functions.validate_request(flask.request, schemas_dict["import_project"]) + if "file" not in flask.request.files: + flask.abort(400, "No zip file provided under 'file'") + + zip_file = flask.request.files["file"] + assert zip_file.filename is not None + filename = werkzeug.utils.secure_filename(os.path.basename(zip_file.filename)) + if not filename.lower().endswith(".zip"): + flask.abort(400, "Uploaded file must be a .zip") + + data_folder_path: str = flask.current_app.config["DATA_FOLDER_PATH"] + + # 423 Locked bypass : remove stopped requests + if connection.scoped_session_registry: + connection.scoped_session_registry.remove() + if connection.engine: + connection.engine.dispose() + connection.engine = connection.session_factory = ( + connection.scoped_session_registry + ) = None + + try: + if os.path.exists(data_folder_path): + shutil.rmtree(data_folder_path) + os.makedirs(data_folder_path, exist_ok=True) + except PermissionError: + flask.abort(423, "Project files are locked; cannot overwrite") + + zip_file.stream.seek(0) + with zipfile.ZipFile(zip_file.stream) as zip_archive: + project_folder = os.path.abspath(data_folder_path) + for member in zip_archive.namelist(): + target = os.path.abspath( + os.path.normpath(os.path.join(project_folder, member)) + ) + if not ( + target == project_folder or target.startswith(project_folder + os.sep) + ): + flask.abort(400, "Zip contains unsafe paths") + zip_archive.extractall(project_folder) + + database_root_path = os.path.join(project_folder, "project.db") + if not os.path.isfile(database_root_path): + flask.abort(400, "Missing project.db at project root") + + connection.init_database(database_root_path, create_tables=False) + + try: + with get_session() as session: + rows = session.query(Data).all() + except Exception: + connection.init_database(database_root_path, create_tables=True) + with get_session() as session: + rows = session.query(Data).all() + + with get_session() as session: + for data_entry in rows: + data_path = geode_functions.data_file_path(data_entry.id) + viewable_name = data_entry.viewable_file_name + if viewable_name: + vpath = geode_functions.data_file_path(data_entry.id, viewable_name) + if os.path.isfile(vpath): + continue + + input_file = str(data_entry.input_file or "") + if not input_file: + continue + + input_full = geode_functions.data_file_path(data_entry.id, input_file) + if not os.path.isfile(input_full): + continue + + data_object = geode_functions.load(data_entry.geode_object, input_full) + utils_functions.save_all_viewables_and_return_info( + data_entry.geode_object, data_object, data_entry, data_path + ) + session.commit() + + snapshot = {} + try: + raw = zip_archive.read("snapshot.json").decode("utf-8") + snapshot = flask.json.loads(raw) + except KeyError: + snapshot = {} + return flask.make_response({"snapshot": snapshot}, 200) diff --git a/src/opengeodeweb_back/routes/schemas/__init__.py b/src/opengeodeweb_back/routes/schemas/__init__.py index 406ed4c..339f896 100644 --- a/src/opengeodeweb_back/routes/schemas/__init__.py +++ b/src/opengeodeweb_back/routes/schemas/__init__.py @@ -8,7 +8,9 @@ from .missing_files import * from .kill import * from .inspect_file import * +from .import_project import * from .geographic_coordinate_systems import * from .geode_objects_and_output_extensions import * +from .export_project import * from .allowed_objects import * from .allowed_files import * diff --git a/src/opengeodeweb_back/routes/schemas/export_project.json b/src/opengeodeweb_back/routes/schemas/export_project.json new file mode 100644 index 0000000..154709d --- /dev/null +++ b/src/opengeodeweb_back/routes/schemas/export_project.json @@ -0,0 +1,21 @@ +{ + "route": "/export_project", + "methods": [ + "POST" + ], + "type": "object", + "properties": { + "snapshot": { + "type": "object" + }, + "filename": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "snapshot", + "filename" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/src/opengeodeweb_back/routes/schemas/export_project.py b/src/opengeodeweb_back/routes/schemas/export_project.py new file mode 100644 index 0000000..dcf3636 --- /dev/null +++ b/src/opengeodeweb_back/routes/schemas/export_project.py @@ -0,0 +1,9 @@ +from dataclasses_json import DataClassJsonMixin +from dataclasses import dataclass +from typing import Dict, Any + + +@dataclass +class ExportProject(DataClassJsonMixin): + filename: str + snapshot: Dict[str, Any] diff --git a/src/opengeodeweb_back/routes/schemas/import_project.json b/src/opengeodeweb_back/routes/schemas/import_project.json new file mode 100644 index 0000000..9ae597c --- /dev/null +++ b/src/opengeodeweb_back/routes/schemas/import_project.json @@ -0,0 +1,10 @@ +{ + "route": "/import_project", + "methods": [ + "POST" + ], + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": false +} \ No newline at end of file diff --git a/src/opengeodeweb_back/routes/schemas/import_project.py b/src/opengeodeweb_back/routes/schemas/import_project.py new file mode 100644 index 0000000..2fd045e --- /dev/null +++ b/src/opengeodeweb_back/routes/schemas/import_project.py @@ -0,0 +1,7 @@ +from dataclasses_json import DataClassJsonMixin +from dataclasses import dataclass + + +@dataclass +class ImportProject(DataClassJsonMixin): + pass diff --git a/src/opengeodeweb_back/utils_functions.py b/src/opengeodeweb_back/utils_functions.py index 1e81f7a..1f70ccd 100644 --- a/src/opengeodeweb_back/utils_functions.py +++ b/src/opengeodeweb_back/utils_functions.py @@ -136,7 +136,9 @@ def send_file( else: mimetype = "application/zip" new_file_name = os.path.splitext(new_file_name)[0] + ".zip" - with zipfile.ZipFile(os.path.join(upload_folder, new_file_name), "w") as zipObj: + with zipfile.ZipFile( + os.path.join(os.path.abspath(upload_folder), new_file_name), "w" + ) as zipObj: for saved_file_path in saved_files: zipObj.write( saved_file_path, @@ -144,7 +146,7 @@ def send_file( ) response = flask.send_from_directory( - directory=upload_folder, + directory=os.path.abspath(upload_folder), path=new_file_name, as_attachment=True, mimetype=mimetype, diff --git a/tests/test_models_routes.py b/tests/test_models_routes.py index e1f916b..8bfd015 100644 --- a/tests/test_models_routes.py +++ b/tests/test_models_routes.py @@ -5,6 +5,8 @@ from opengeodeweb_back import geode_functions from opengeodeweb_microservice.database.data import Data from opengeodeweb_microservice.database.connection import get_session +import zipfile +import json def test_model_mesh_components(client, test_id): @@ -55,3 +57,131 @@ def test_extract_brep_uuids(client, test_id): assert "uuid_dict" in response.json uuid_dict = response.json["uuid_dict"] assert isinstance(uuid_dict, dict) + + +def test_export_project_route(client, tmp_path): + route = "/opengeodeweb_back/export_project" + snapshot = { + "styles": {"1": {"visibility": True, "opacity": 1.0, "color": [0.2, 0.6, 0.9]}} + } + filename = "export_project_test.zip" + project_folder = client.application.config["DATA_FOLDER_PATH"] + os.makedirs(project_folder, exist_ok=True) + database_root_path = os.path.join(project_folder, "project.db") + with open(database_root_path, "wb") as f: + f.write(b"test_project_db") + response = client.post(route, json={"snapshot": snapshot, "filename": filename}) + assert response.status_code == 200 + assert response.headers.get("new-file-name") == filename + assert response.mimetype == "application/octet-binary" + response.direct_passthrough = False + zip_bytes = response.get_data() + tmp_zip_path = tmp_path / filename + tmp_zip_path.write_bytes(zip_bytes) + with zipfile.ZipFile(tmp_zip_path, "r") as zip_file: + names = zip_file.namelist() + assert "snapshot.json" in names + parsed = json.loads(zip_file.read("snapshot.json").decode("utf-8")) + assert parsed == snapshot + assert "project.db" in names + response.close() + export_path = os.path.join(project_folder, filename) + if os.path.exists(export_path): + os.remove(export_path) + + +def test_import_project_route(client, tmp_path): + route = "/opengeodeweb_back/import_project" + snapshot = { + "styles": {"1": {"visibility": True, "opacity": 1.0, "color": [0.2, 0.6, 0.9]}} + } + + original_data_folder = client.application.config["DATA_FOLDER_PATH"] + client.application.config["DATA_FOLDER_PATH"] = os.path.join( + str(tmp_path), "project_data" + ) + db_path = os.path.join(client.application.config["DATA_FOLDER_PATH"], "project.db") + + import sqlite3, zipfile, json + + temp_db = tmp_path / "temp_project.db" + conn = sqlite3.connect(str(temp_db)) + conn.execute( + "CREATE TABLE datas (id TEXT PRIMARY KEY, geode_object TEXT, viewer_object TEXT, native_file_name TEXT, " + "viewable_file_name TEXT, light_viewable TEXT, input_file TEXT, additional_files TEXT)" + ) + conn.commit() + conn.close() + + z = tmp_path / "import_project_test.zip" + with zipfile.ZipFile(z, "w", compression=zipfile.ZIP_DEFLATED) as zipf: + zipf.writestr("snapshot.json", json.dumps(snapshot)) + zipf.write(str(temp_db), "project.db") + + with open(z, "rb") as f: + resp = client.post( + route, + data={"file": (f, "import_project_test.zip")}, + content_type="multipart/form-data", + ) + + assert resp.status_code == 200 + assert resp.json.get("snapshot") == snapshot + assert os.path.exists(db_path) + + from opengeodeweb_microservice.database import connection + + client.application.config["DATA_FOLDER_PATH"] = original_data_folder + test_db_path = os.environ.get("TEST_DB_PATH") + if test_db_path: + connection.init_database(test_db_path, create_tables=True) + + client.application.config["DATA_FOLDER_PATH"] = original_data_folder + + +def test_save_viewable_workflow_from_file(client): + route = "/opengeodeweb_back/save_viewable_file" + payload = {"input_geode_object": "BRep", "filename": "cube.og_brep"} + + response = client.post(route, json=payload) + assert response.status_code == 200 + + data_id = response.json["id"] + assert isinstance(data_id, str) and len(data_id) > 0 + assert response.json["viewable_file_name"].endswith(".vtm") + + comp_resp = client.post( + "/opengeodeweb_back/models/vtm_component_indices", json={"id": data_id} + ) + assert comp_resp.status_code == 200 + + refreshed = Data.get(data_id) + assert refreshed is not None + + +def test_save_viewable_workflow_from_object(client): + route = "/opengeodeweb_back/create/create_aoi" + aoi_data = { + "name": "workflow_aoi", + "points": [ + {"x": 0.0, "y": 0.0}, + {"x": 1.0, "y": 0.0}, + {"x": 1.0, "y": 1.0}, + {"x": 0.0, "y": 1.0}, + ], + "z": 0.0, + } + + response = client.post(route, json=aoi_data) + assert response.status_code == 200 + + data_id = response.json["id"] + assert isinstance(data_id, str) and len(data_id) > 0 + assert response.json["geode_object"] == "EdgedCurve3D" + assert response.json["viewable_file_name"].endswith(".vtp") + + attr_resp = client.post( + "/opengeodeweb_back/vertex_attribute_names", json={"id": data_id} + ) + assert attr_resp.status_code == 200 + assert isinstance(attr_resp.json.get("vertex_attribute_names", []), list) diff --git a/tests/test_utils_functions.py b/tests/test_utils_functions.py index 162af7b..fcca322 100644 --- a/tests/test_utils_functions.py +++ b/tests/test_utils_functions.py @@ -6,6 +6,8 @@ import flask import shutil import uuid +import zipfile +import io # Local application imports from opengeodeweb_microservice.database.data import Data @@ -206,3 +208,49 @@ def test_generate_native_viewable_and_light_viewable_from_file(client): assert isinstance(result["object_type"], str) assert isinstance(result["binary_light_viewable"], str) assert isinstance(result["input_file"], str) + + +def test_send_file_multiple_returns_zip(client, tmp_path): + app = client.application + with app.app_context(): + app.config["UPLOAD_FOLDER"] = str(tmp_path) + file_paths = [] + for i, content in [(1, b"hello 1"), (2, b"hello 2")]: + file_path = tmp_path / f"tmp_send_file_{i}.txt" + file_path.write_bytes(content) + file_paths.append(str(file_path)) + with app.test_request_context(): + response = utils_functions.send_file( + app.config["UPLOAD_FOLDER"], file_paths, "bundle" + ) + assert response.status_code == 200 + assert response.mimetype == "application/zip" + new_file_name = response.headers.get("new-file-name") + assert new_file_name == "bundle.zip" + zip_path = os.path.join(app.config["UPLOAD_FOLDER"], new_file_name) + with zipfile.ZipFile(zip_path, "r") as zip_file: + zip_entries = zip_file.namelist() + assert "tmp_send_file_1.txt" in zip_entries + assert "tmp_send_file_2.txt" in zip_entries + response.close() + + +def test_send_file_single_returns_octet_binary(client, tmp_path): + app = client.application + with app.app_context(): + app.config["UPLOAD_FOLDER"] = str(tmp_path) + file_path = tmp_path / "tmp_send_file_1.txt" + file_path.write_bytes(b"hello 1") + with app.test_request_context(): + response = utils_functions.send_file( + app.config["UPLOAD_FOLDER"], [str(file_path)], "tmp_send_file_1.txt" + ) + assert response.status_code == 200 + assert response.mimetype == "application/octet-binary" + new_file_name = response.headers.get("new-file-name") + assert new_file_name == "tmp_send_file_1.txt" + zip_path = os.path.join(app.config["UPLOAD_FOLDER"], new_file_name) + with open(zip_path, "rb") as f: + file_bytes = f.read() + assert file_bytes == b"hello 1" + response.close()