[rel-db] Fix meeting.import (#3287)

* Fix meeting.import

* Fix meeting export

* Fix meeting.clone

Author: luisa-beerboom

openslides_backend/action/actions/meeting/clone.py

@@ -268,6 +268,7 @@ def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]:
             raise ActionException(
                 "Cannot create a non-template meeting without administrators"
             )
+        self.transform_json_fields(instance)
         return instance
 
     def _create_or_get_meeting_user(self, meeting_id: int, user_id: int) -> int:

openslides_backend/action/actions/meeting/import_.py

@@ -4,6 +4,8 @@
 from datetime import datetime
 from typing import Any
 
+from psycopg.types.json import Jsonb
+
 from openslides_backend.action.actions.meeting.mixins import MeetingPermissionMixin
 from openslides_backend.migrations.migration_helper import MigrationHelper
 from openslides_backend.models.base import model_registry
@@ -13,9 +15,11 @@
     BaseRelationField,
     GenericRelationField,
     GenericRelationListField,
+    JSONField,
     OnDelete,
     RelationField,
     RelationListField,
+    TimestampField,
 )
 from openslides_backend.models.models import Meeting
 from openslides_backend.permissions.management_levels import OrganizationManagementLevel
@@ -161,9 +165,42 @@ def preprocess_data(self, instance: dict[str, Any]) -> dict[str, Any]:
         self.remove_not_allowed_fields(instance)
         self.set_committee_and_orga_relation(instance)
         self.check_data_migration_index(instance)
+        self.transform_timestamps(instance)
         self.unset_committee_and_orga_relation(instance)
         return instance
 
+    def transform_timestamps(self, instance: dict[str, Any]) -> dict[str, Any]:
+        for collection, collection_data in instance["meeting"].items():
+            if model := model_registry.get(collection):
+                fields = list(model().get_fields())
+                timestamp_field_names = [
+                    field.own_field_name
+                    for field in fields
+                    if isinstance(field, TimestampField)
+                ]
+                if timestamp_field_names:
+                    for mod in collection_data.values():
+                        for field in timestamp_field_names:
+                            if (iso := mod.get(field)) and isinstance(iso, str):
+                                mod[field] = datetime.fromisoformat(iso)
+        return instance
+
+    def transform_json_fields(self, instance: dict[str, Any]) -> dict[str, Any]:
+        for collection, collection_data in instance["meeting"].items():
+            if model := model_registry.get(collection):
+                fields = list(model().get_fields())
+                json_field_names = [
+                    field.own_field_name
+                    for field in fields
+                    if isinstance(field, JSONField)
+                ]
+                if json_field_names:
+                    for mod in collection_data.values():
+                        for field in json_field_names:
+                            if field in mod:
+                                mod[field] = Jsonb(mod[field])
+        return instance
+
     def check_one_meeting(self, instance: dict[str, Any]) -> None:
         if len(instance["meeting"]["meeting"]) != 1:
             raise ActionException("Need exactly one meeting in meeting collection.")
@@ -276,6 +313,7 @@ def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]:
             raise ActionException(str(ce))
         self.allowed_collections = checker.allowed_collections
 
+        self.transform_json_fields(instance)
         self.check_limit_of_meetings()
         self.update_meeting_and_users(instance)
 
@@ -752,7 +790,7 @@ def check_data_migration_index(self, instance: dict[str, Any]) -> None:
         """
         Check for valid migration index.
         """
-        start_migration_index = instance["meeting"].pop("_migration_index")
+        start_migration_index = instance["meeting"].get("_migration_index")
         backend_migration_index = MigrationHelper.get_backend_migration_index()
         if backend_migration_index < start_migration_index:
             raise ActionException(

openslides_backend/models/fields.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from decimal import Decimal
+from decimal import Decimal, InvalidOperation
 from enum import StrEnum
 from typing import Any, cast
 
@@ -253,7 +253,12 @@ def validate(self, value: Any, payload: dict[str, Any] = {}) -> Any:
         if value is not None or self.required:
             if (min_ := self.constraints.get("minimum")) is not None:
                 if isinstance(value, str):
-                    value = Decimal(value)
+                    try:
+                        value = Decimal(value)
+                    except InvalidOperation:
+                        raise ActionException(
+                            f"{self.own_field_name}: value '{value}' couldn't be converted to decimal."
+                        )
                 elif not isinstance(value, Decimal | None):
                     raise NotImplementedError(
                         f"Unexpected type: {type(value)} (value: {value}) for field {self.get_own_field_name()}"

openslides_backend/presenter/export_meeting.py

@@ -40,7 +40,9 @@ def get_result(self) -> Any:
             msg = "You are not allowed to perform presenter export_meeting."
             msg += f" Missing permission: {OrganizationManagementLevel.SUPERADMIN}"
             raise PermissionDenied(msg)
-        export_data = export_meeting(self.datastore, self.data["meeting_id"])
+        export_data = export_meeting(
+            self.datastore, self.data["meeting_id"], datetime_decimal_to_string=True
+        )
         if id_ := next(
             (
                 id_

openslides_backend/shared/export_helper.py

@@ -1,4 +1,6 @@
+import datetime
 from collections.abc import Iterable
+from decimal import Decimal
 from typing import Any
 
 from openslides_backend.migrations.migration_helper import MigrationHelper
@@ -34,6 +36,7 @@ def export_meeting(
     meeting_id: int,
     internal_target: bool = False,
     update_mediafiles: bool = False,
+    datetime_decimal_to_string: bool = False,
 ) -> dict[str, Any]:
     export: dict[str, Any] = {}
 
@@ -212,6 +215,14 @@ def export_meeting(
         export[collection] = dict(
             sorted(instances.items(), key=lambda item: int(item[0]))
         )
+        if datetime_decimal_to_string and isinstance(instances, dict):
+            for data in instances.values():
+                for field, value in data.items():
+                    if isinstance(value, datetime.datetime):
+                        data[field] = value.isoformat()
+                    if isinstance(value, Decimal):
+                        data[field] = str(value)
+
     return export
 
 

tests/system/action/meeting/test_clone.py

@@ -1372,11 +1372,11 @@ def test_meeting_name_exact_fit(self) -> None:
         self.assert_model_exists("meeting/2", {"name": long_name + " - Copy"})
 
     def test_meeting_name_too_long(self) -> None:
-        self.meeting_data["name"] = "A" * 100
+        self.meeting_data["name"] = "A" * 200
         self.set_test_data_with_admin()
         response = self.request("meeting.clone", {"meeting_id": 1})
         self.assert_status_code(response, 200)
-        self.assert_model_exists("meeting/2", {"name": "A" * 90 + "... - Copy"})
+        self.assert_model_exists("meeting/2", {"name": "A" * 190 + "... - Copy"})
 
     def test_permissions_explicit_source_committee_permission(self) -> None:
         self.set_test_data()
@@ -1841,6 +1841,21 @@ def test_clone_amendment_paragraphs(self) -> None:
             response.json["message"],
         )
 
+    def test_clone_amendment_paragraphs_regular(self) -> None:
+        self.set_test_data()
+        self.set_user_groups(1, [1])
+        self.create_motion(
+            1, 1, motion_data={"amendment_paragraphs": Jsonb({"1": "<p>test</p>"})}
+        )
+        response = self.request(
+            "meeting.clone",
+            {
+                "meeting_id": 1,
+                "admin_ids": [1],
+            },
+        )
+        self.assert_status_code(response, 200)
+
     def test_permissions_oml_locked_meeting(self) -> None:
         self.create_meeting(
             meeting_data={"locked_from_inside": True, "template_for_organization_id": 1}
@@ -2134,3 +2149,15 @@ def test_clone_with_structured_published_orga_files(self) -> None:
         for fqid, model in models.items():
             self.assert_model_exists(fqid, model)
         self.media.duplicate_mediafile.assert_not_called()
+
+    def test_clone_require_duplicate_from_allowed(self) -> None:
+        self.set_test_data_with_admin()
+        self.set_models(
+            {
+                "meeting/1": {"template_for_organization_id": 1, "name": "m1"},
+            }
+        )
+        self.set_committee_management_level([60])
+        self.set_organization_management_level(None)
+        response = self.request("meeting.clone", {"meeting_id": 1})
+        self.assert_status_code(response, 200)