Merge branch 'develop' into dev-r85-v2-pull

Author: varmar05

server/Pipfile

@@ -26,7 +26,7 @@ flask-migrate = "==2.6.0" # 3.1.0
 wtforms-json = "==0.3.5"
 pytz = "==2022.2.1"
 scikit-build = "==0.18.1"
-pygeodiff = "==1.0.6"
+pygeodiff = "==2.0.4"
 pathvalidate = "==3.2.0"
 celery= "==5.4.0"
 redis= "==5.0.1"
@@ -40,6 +40,7 @@ psycogreen = "==1.0.2"
 importlib-metadata = "==8.4.0"  # https://github.com/pallets/flask/issues/4502
 typing_extensions = "==4.12.2"
 python-magic = "==0.4.27"
+click = "==8.2.0"
 # requirements for development on windows
 colorama = "==0.4.5"
 

server/Pipfile.lock

@@ -1,6 +1,7 @@
 # Copyright (C) Lutra Consulting Limited
 #
 # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
+
 import re
 import safe
 from flask_wtf import FlaskForm
@@ -48,18 +49,22 @@ class ExtendedEmail(Email):
     1. spaces,
     2. special characters ,:;()<>[]\"
     3, multiple @ symbols,
-    4, leading, trailing, or consecutive dots in the local part
-    5, invalid domain part - missing top level domain (user@example), consecutive dots
-    Custom check for additional invalid characters disallows |'— because they make our email sending service to fail
+    4, leading, trailing, or consecutive dots in the local part,
+    5, invalid domain part - missing top level domain (user@example), consecutive dots,
+    The extended validation checks email addresses using the regex provided by Brevo,
+    so that we stay consistent with their validation rules and avoid API failures.
     """
 
     def __call__(self, form, field):
         super().__call__(form, field)
 
-        if re.search(r"[|'—]", field.data):
-            raise ValidationError(
-                f"Email address '{field.data}' contains an invalid character."
-            )
+        email = field.data.strip()
+
+        pattern = r"^[\x60#&*\/=?^{!}~'+\w-]+(\.[\x60#&*\/=?^{!}~'+\w-]+)*\.?@([_a-zA-Z0-9-]+(\.[_a-zA-Z0-9-]+)*\.)[a-zA-Z0-9-]*[a-zA-Z0-9]{2,}$"
+        email_regexp = re.compile(pattern, re.IGNORECASE)
+
+        if not email_regexp.match(email):
+            raise ValidationError(f"Email address '{email}' is invalid.")
 
 
 class PasswordValidator:

server/mergin/auth/forms.py

@@ -20,10 +20,10 @@
 
 from .utils import (
     is_file_name_blacklisted,
-    is_qgis,
     is_supported_extension,
     is_valid_path,
     is_versioned_file,
+    has_trailing_space,
 )
 from ..app import DateTimeWithZ, ma
 
@@ -219,14 +219,21 @@ def validate(self, data, **kwargs):
 
             if not is_valid_path(file_path):
                 raise ValidationError(
-                    f"Unsupported file name detected: {file_path}. Please remove the invalid characters."
+                    f"Unsupported file name detected: '{file_path}'. Please remove the invalid characters."
                 )
 
             if not is_supported_extension(file_path):
                 raise ValidationError(
-                    f"Unsupported file type detected: {file_path}. "
+                    f"Unsupported file type detected: '{file_path}'. "
                     f"Please remove the file or try compressing it into a ZIP file before uploading.",
                 )
+        # new checks must restrict only new files not to block existing projects
+        for file in data["added"]:
+            file_path = file["path"]
+            if has_trailing_space(file_path):
+                raise ValidationError(
+                    f"Folder name contains a trailing space. Please remove the space in: '{file_path}'."
+                )
 
 
 class ProjectFileSchema(FileSchema):
@@ -239,7 +246,7 @@ class ProjectFileSchema(FileSchema):
     def patch_field(self, data, **kwargs):
         # drop 'diff' key entirely if empty or None as clients would expect
         if not data.get("diff"):
-            data.pop("diff")
+            data.pop("diff", None)
         return data
 
 

server/mergin/sync/files.py

@@ -79,9 +79,10 @@
     get_user_agent,
     generate_location,
     is_valid_uuid,
-    gpkg_wkb_to_wkt,
     get_device_id,
     prepare_download_response,
+    get_device_id,
+    wkb2wkt,
 )
 from .errors import StorageLimitHit, ProjectLocked
 from ..utils import format_time_delta
@@ -944,7 +945,7 @@ def push_finish(transaction_id):
         if len(unsupported_files):
             abort(
                 400,
-                f"Unsupported file type detected: {unsupported_files[0]}. "
+                f"Unsupported file type detected: '{unsupported_files[0]}'. "
                 f"Please remove the file or try compressing it into a ZIP file before uploading.",
             )
 
@@ -1000,14 +1001,6 @@ def push_finish(transaction_id):
         # let's move uploaded files where they are expected to be
         os.renames(files_dir, version_dir)
 
-        # remove used chunks
-        for file in upload.changes["added"] + upload.changes["updated"]:
-            file_chunks = file.get("chunks", [])
-            for chunk_id in file_chunks:
-                chunk_file = os.path.join(upload.upload_dir, "chunks", chunk_id)
-                if os.path.exists(chunk_file):
-                    move_to_tmp(chunk_file)
-
         logging.info(
             f"Push finished for project: {project.id}, project version: {v_next_version}, transaction id: {transaction_id}."
         )
@@ -1300,7 +1293,10 @@ def get_resource_changeset(project_name, namespace, version_id, path):  # noqa:
                 if key not in geom_change:
                     continue
                 gpkg_wkb = base64.b64decode(geom_change[key], validate=True)
-                wkt = gpkg_wkb_to_wkt(gpkg_wkb)
+                wkb = version.project.storage.geodiff.create_wkb_from_gpkg_header(
+                    gpkg_wkb
+                )
+                wkt = wkb2wkt(wkb)
                 if wkt:
                     geom_change[key] = wkt
         except (binascii.Error, TypeError, ValueError):