diff --git a/openslides_backend/action/actions/meeting/clone.py b/openslides_backend/action/actions/meeting/clone.py index 4059f7d8e8..222ef4bf20 100644 --- a/openslides_backend/action/actions/meeting/clone.py +++ b/openslides_backend/action/actions/meeting/clone.py @@ -94,7 +94,9 @@ def check_permissions(self, instance: dict[str, Any]) -> None: MeetingPermissionMixin.check_permissions(self, instance) def update_instance(self, instance: dict[str, Any]) -> dict[str, Any]: - meeting_json = export_meeting(self.datastore, instance["meeting_id"], True) + meeting_json = export_meeting( + self.datastore, instance["meeting_id"], True, True + ) instance["meeting"] = meeting_json additional_user_ids = instance.pop("user_ids", None) or [] additional_admin_ids = instance.pop("admin_ids", None) or [] diff --git a/openslides_backend/models/checker.py b/openslides_backend/models/checker.py index badfc3115a..477e79bb2a 100644 --- a/openslides_backend/models/checker.py +++ b/openslides_backend/models/checker.py @@ -1,7 +1,6 @@ -import re from collections.abc import Callable, Iterable from datetime import datetime -from decimal import Decimal, InvalidOperation +from decimal import Decimal from math import floor from typing import Any, cast @@ -38,6 +37,7 @@ DECIMAL_PATTERN, EXTENSION_REFERENCE_IDS_PATTERN, collection_and_id_from_fqid, + is_fqid, ) from openslides_backend.shared.schema import ( models_map_object, @@ -83,6 +83,15 @@ def check_string(value: Any) -> bool: return value is None or isinstance(value, str) +def check_fqid(value: Any) -> bool: + if value is None: + return True + if not is_fqid(value): + return False + collection, _id = collection_and_id_from_fqid(value) + return collection in set(model_registry.keys()) + + def check_color(value: Any) -> bool: return value is None or bool(isinstance(value, str) and COLOR_PATTERN.match(value)) @@ -103,6 +112,10 @@ def check_string_list(value: Any) -> bool: return check_x_list(value, check_string) +def check_fqid_list(value: Any) -> bool: + return check_x_list(value, check_fqid) + + def check_number_list(value: Any) -> bool: return check_x_list(value, check_number) @@ -150,14 +163,14 @@ def check_timestamp(value: Any) -> bool: CharField: check_string, HTMLStrictField: check_string, HTMLPermissiveField: check_string, - GenericRelationField: check_string, + GenericRelationField: check_fqid, IntegerField: check_number, TimestampField: check_timestamp, RelationField: check_number, FloatField: check_float, BooleanField: check_boolean, CharArrayField: check_string_list, - GenericRelationListField: check_string_list, + GenericRelationListField: check_fqid_list, NumberArrayField: check_number_list, RelationListField: check_number_list, DecimalField: check_decimal, @@ -255,8 +268,7 @@ def get_fields(self, collection: str) -> Iterable[Field]: def run_check(self) -> None: self.check_json() - # TODO reenable when import migration works - # self.check_migration_index() + self.check_migration_index() self.check_collections() for collection, models in self.data.items(): if collection.startswith("_"): @@ -304,7 +316,6 @@ def check_normal_fields(self, model: dict[str, Any], collection: str) -> bool: all_collection_fields = { field.get_own_field_name() for field in self.get_fields(collection) } - # TODO: remove duplication: required is also checked in check_types required_or_default_collection_fields = { field.get_own_field_name() for field in self.get_fields(collection) @@ -333,9 +344,6 @@ def check_normal_fields(self, model: dict[str, Any], collection: str) -> bool: error = f"{collection}/{model['id']}/{fieldname}: {str(e)}" self.errors.append(error) errors = True - except InvalidOperation: - # invalide decimal json, will be checked at check_types - pass return errors def fix_missing_default_values( @@ -365,6 +373,7 @@ def check_types(self, model: dict[str, Any], collection: str) -> None: f"TODO implement check for field type {field_type}" ) + # TODO: move the validation logic to `field.validate` methods. Merge with the check from check_normal_fields(). if not checker(model[field]): error = f"{collection}/{model['id']}/{field}: Type error: Type is not {field_type}" self.errors.append(error) @@ -412,31 +421,22 @@ def check_special_fields(self, model: dict[str, Any], collection: str) -> None: html, ALLOWED_HTML_TAGS_STRICT ): self.errors.append(msg + f"Invalid html in {key}") - if "recommendation_extension" in model: - basemsg = ( - f"{collection}/{model['id']}/recommendation_extension: Relation Error: " - ) - RECOMMENDATION_EXTENSION_REFERENCE_IDS_PATTERN = re.compile( - r"\[(?P\w+/\d+)\]" - ) - recommendation_extension = model["recommendation_extension"] - if recommendation_extension is None: - recommendation_extension = "" - possible_rerids = RECOMMENDATION_EXTENSION_REFERENCE_IDS_PATTERN.findall( - recommendation_extension - ) - for fqid_str in possible_rerids: - re_collection, re_id_ = collection_and_id_from_fqid(fqid_str) - if re_collection != "motion": - self.errors.append( - basemsg + f"Found {fqid_str} but only motion is allowed." - ) - if not self.find_model(re_collection, int(re_id_)): - self.errors.append( - basemsg - + f"Found {fqid_str} in recommendation_extension but not in models." - ) + for field_name in ["state_extension", "recommendation_extension"]: + basemsg = f"{collection}/{model['id']}/{field_name}: Relation Error:" + if value := model.get(field_name): + matches = EXTENSION_REFERENCE_IDS_PATTERN.findall(value) + for fqid in matches: + re_collection, re_id = collection_and_id_from_fqid(fqid) + if re_collection != "motion": + self.errors.append( + basemsg + f" Found {fqid} but only motion is allowed." + ) + if not self.find_model(re_collection, int(re_id)): + self.errors.append( + basemsg + + f" Found {fqid} in {field_name} but not in models." + ) def check_relations(self, model: dict[str, Any], collection: str) -> None: for field in model.keys(): @@ -451,7 +451,7 @@ def check_relation( self, model: dict[str, Any], collection: str, field: str ) -> None: field_type = self.get_type_from_collection(field, collection) - basemsg = f"{collection}/{model['id']}/{field}: Relation Error: " + basemsg = f"{collection}/{model['id']}/{field}: Relation Error:" if collection == "user" and field == "organization_id": return @@ -478,7 +478,7 @@ def check_relation( ) elif isinstance(field_type, RelationListField): foreign_ids = model[field] - if not foreign_ids: + if not foreign_ids or not isinstance(foreign_ids, list): return foreign_collection, foreign_field = self.get_to(field, collection) @@ -513,6 +513,7 @@ def check_relation( foreign_field, basemsg, ) + # TODO: cleanup. Unreachable code (mode and collection are checked in split_fqid), but error message there is not too useful elif self.mode == "external": self.errors.append( f"{basemsg} points to {foreign_collection}/{foreign_id}, which is not allowed in an external import." @@ -541,24 +542,6 @@ def check_relation( f"{basemsg} points to {foreign_collection}/{foreign_id}, which is not allowed in an external import." ) - elif collection == "motion": - for prefix in ("state", "recommendation"): - if field == f"{prefix}_extension" and ( - value := model.get(f"{prefix}_extension") - ): - matches = EXTENSION_REFERENCE_IDS_PATTERN.findall(value) - for fqid in matches: - re_collection, re_id = collection_and_id_from_fqid(fqid) - if re_collection != "motion": - self.errors.append( - basemsg + f"Found {fqid} but only motion is allowed." - ) - if not self.find_model(re_collection, int(re_id)): - self.errors.append( - basemsg - + f"Found {fqid} in {prefix}_extension but not in models." - ) - def get_to(self, field: str, collection: str) -> tuple[str, str | None]: field_type = cast( BaseRelationField, self.get_model(collection).get_field(field) @@ -586,8 +569,8 @@ def check_calculated_fields( source_parent = self.find_model("mediafile", source_model["parent_id"]) # relations are checked beforehand, so parent always exists assert source_parent - parent_ids = set(meeting.get("meeting_mediafile_ids", [])).intersection( - source_parent.get("meeting_mediafile_ids", []) + parent_ids = set(meeting.get("meeting_mediafile_ids") or []).intersection( + set(source_parent.get("meeting_mediafile_ids") or []) ) assert len(parent_ids) <= 1 if len(parent_ids): @@ -668,21 +651,21 @@ def check_reverse_relation( if error: self.errors.append( f"{basemsg} points to {foreign_collection}/{foreign_id}/{actual_foreign_field}," - " but the reverse relation for it is corrupt" + " but the reverse relation for it is corrupt." ) def split_fqid(self, fqid: str) -> tuple[str, int]: try: - collection, _id = fqid.split("/") - id = int(_id) + collection, _id = collection_and_id_from_fqid(fqid) + assert collection if self.mode == "external" and collection not in self.allowed_collections: - raise CheckException(f"Fqid {fqid} has an invalid collection") - return collection, id - except (ValueError, AttributeError): + raise CheckException(f"Fqid {fqid} has an invalid collection.") + return collection, _id + except (ValueError, AttributeError, AssertionError, IndexError): raise CheckException(f"Fqid {fqid} is malformed") - def split_collectionfield(self, collectionfield: str) -> tuple[str, str]: - collection, field = collectionfield.split("/") + def split_collectionfield(self, collectionfield: str) -> tuple[str, int]: + collection, field = collection_and_id_from_fqid(collectionfield) if collection not in self.allowed_collections: raise CheckException( f"Collectionfield {collectionfield} has an invalid collection" @@ -704,7 +687,7 @@ def get_to_generic_case( if foreign_collection not in to.keys(): raise CheckException( f"The collection {foreign_collection} is not supported " - "as a reverse relation in {collection}/{field}" + f"as a reverse relation in {collection}/{field}." ) return to[foreign_collection] @@ -714,5 +697,5 @@ def get_to_generic_case( return f raise CheckException( - f"The collection {foreign_collection} is not supported as a reverse relation in {collection}/{field}" + f"The collection {foreign_collection} is not supported as a reverse relation in {collection}/{field}." ) diff --git a/openslides_backend/presenter/check_database.py b/openslides_backend/presenter/check_database.py index 5e58c90cb2..9f2034edc6 100644 --- a/openslides_backend/presenter/check_database.py +++ b/openslides_backend/presenter/check_database.py @@ -36,7 +36,7 @@ def check_meetings(datastore: Database, meeting_id: int | None) -> dict[int, str errors: dict[int, str] = {} for meeting_id in meeting_ids: - export = export_meeting(datastore, meeting_id, True) + export = export_meeting(datastore, meeting_id, True, True) try: Checker( data=export, diff --git a/openslides_backend/presenter/check_database_all.py b/openslides_backend/presenter/check_database_all.py index a2a09485b3..9ed5144895 100644 --- a/openslides_backend/presenter/check_database_all.py +++ b/openslides_backend/presenter/check_database_all.py @@ -3,6 +3,7 @@ import fastjsonschema from openslides_backend.migrations import get_backend_migration_index +from openslides_backend.shared.export_helper import get_fields_for_export from openslides_backend.shared.patterns import is_reserved_field from ..models.checker import Checker, CheckException @@ -32,7 +33,11 @@ def check_everything(datastore: Database) -> None: str(id): { field: value for field, value in model.items() - if not is_reserved_field(field) + if ( + field in get_fields_for_export(collection) + and not is_reserved_field(field) + and value is not None + ) } for id, model in models.items() } diff --git a/openslides_backend/shared/export_helper.py b/openslides_backend/shared/export_helper.py index 84b542dea5..8fe494483d 100644 --- a/openslides_backend/shared/export_helper.py +++ b/openslides_backend/shared/export_helper.py @@ -3,6 +3,7 @@ from openslides_backend.migrations import get_backend_migration_index from openslides_backend.shared.patterns import is_reserved_field +from openslides_backend.shared.util import ONE_ORGANIZATION_FQID, ONE_ORGANIZATION_ID from ..models.base import model_registry from ..models.fields import ( @@ -23,7 +24,10 @@ def export_meeting( - datastore: Database, meeting_id: int, internal_target: bool = False + datastore: Database, + meeting_id: int, + internal_target: bool = False, + update_mediafiles: bool = False, ) -> dict[str, Any]: export: dict[str, Any] = {} @@ -56,6 +60,63 @@ def export_meeting( results = datastore.get_many( get_many_requests, lock_result=False, use_changed_models=False ) + # update_mediafiles_for_internal_calls + if update_mediafiles and len( + mediafile_ids := results.get("mediafile", {}).keys() + ) != len(meeting_mediafiles := results.get("meeting_mediafile", {})): + mm_with_unknown_mediafiles = { + mm_id: mm_data + for mm_id, mm_data in meeting_mediafiles.items() + if mm_data["mediafile_id"] not in mediafile_ids + } + unknown_mediafiles: dict[int, dict[str, Any]] = {} + next_file_ids = [ + mm["mediafile_id"] for mm in mm_with_unknown_mediafiles.values() + ] + while next_file_ids: + unknown_mediafiles.update( + datastore.get_many( + [ + GetManyRequest( + "mediafile", + next_file_ids, + [ + "id", + "owner_id", + "published_to_meetings_in_organization_id", + "parent_id", + "child_ids", + ], + ), + ], + use_changed_models=False, + )["mediafile"] + ) + next_file_ids = list( + { + parent_id + for m in unknown_mediafiles.values() + if (parent_id := m.get("parent_id")) + } + - set(unknown_mediafiles) + ) + for mm_id, mm_data in mm_with_unknown_mediafiles.items(): + mediafile_id = mm_data["mediafile_id"] + mediafile = unknown_mediafiles.get(mediafile_id) + if ( + mediafile + and mediafile["owner_id"] == ONE_ORGANIZATION_FQID + and mediafile["published_to_meetings_in_organization_id"] + == ONE_ORGANIZATION_ID + ): + mediafile["meeting_mediafile_ids"] = [mm_id] + results.setdefault("mediafile", {})[mediafile_id] = mediafile + while ( + parent_id := mediafile.get("parent_id") + ) and parent_id not in results["mediafile"]: + mediafile = unknown_mediafiles[parent_id] + results["mediafile"][parent_id] = mediafile + else: results = {} diff --git a/tests/system/action/organization/test_initial_import.py b/tests/system/action/organization/test_initial_import.py index a5129f3a95..907f90f9d9 100644 --- a/tests/system/action/organization/test_initial_import.py +++ b/tests/system/action/organization/test_initial_import.py @@ -174,7 +174,7 @@ def test_initial_import_wrong_type(self) -> None: response.json["message"], ) self.assertIn( - "organization/1/theme_id: Relation Error: points to theme/test/theme_for_organization_id, but the reverse relation for it is corrupt", + "organization/1/theme_id: Relation Error: points to theme/test/theme_for_organization_id, but the reverse relation for it is corrupt.", response.json["message"], ) @@ -186,11 +186,11 @@ def test_initial_import_wrong_relation(self) -> None: ) self.assert_status_code(response, 400) self.assertIn( - "Relation Error: points to theme/666/theme_for_organization_id, but the reverse relation for it is corrupt", + "Relation Error: points to theme/666/theme_for_organization_id, but the reverse relation for it is corrupt.", response.json["message"], ) self.assertIn( - "Relation Error: points to organization/1/theme_id, but the reverse relation for it is corrupt", + "Relation Error: points to organization/1/theme_id, but the reverse relation for it is corrupt.", response.json["message"], ) diff --git a/tests/system/presenter/test_check_database.py b/tests/system/presenter/test_check_database.py index 24995d6814..087f0120ce 100644 --- a/tests/system/presenter/test_check_database.py +++ b/tests/system/presenter/test_check_database.py @@ -1,44 +1,15 @@ from typing import Any -import pytest +from psycopg.types.json import Jsonb from openslides_backend.models.models import Meeting from openslides_backend.permissions.management_levels import OrganizationManagementLevel +from openslides_backend.shared.util import ONE_ORGANIZATION_FQID, ONE_ORGANIZATION_ID from .base import BasePresenterTestCase -@pytest.mark.skip(reason="During development of relational DB not necessary") class TestCheckDatabase(BasePresenterTestCase): - def test_found_errors(self) -> None: - self.set_models( - { - "meeting/1": {"name": "test_foo"}, - "meeting/2": {"name": "test_bar"}, - } - ) - status_code, data = self.request("check_database", {}) - assert status_code == 200 - assert data["ok"] is False - assert "Meeting 1" in data["errors"] - assert "meeting/1: Missing fields" in data["errors"] - assert "Meeting 2" in data["errors"] - assert "meeting/2: Missing fields" in data["errors"] - - def test_found_errors_one_meeting(self) -> None: - self.set_models( - { - "meeting/1": {"name": "test_foo"}, - "meeting/2": {"name": "test_bar"}, - } - ) - status_code, data = self.request("check_database", {"meeting_id": 2}) - assert status_code == 200 - assert data["ok"] is False - assert "Meeting 1" not in data["errors"] - assert "Meeting 2" in data["errors"] - assert "meeting/2: Missing fields" in data["errors"] - def get_meeting_defaults(self) -> dict[str, Any]: return { "motions_export_title": "Motions", @@ -325,9 +296,10 @@ def test_correct_relations(self) -> None: "user_ids": [1, 2, 3, 4, 5, 6], "present_user_ids": [2], "mediafile_ids": [1, 2], - "meeting_mediafile_ids": [1, 2], + "meeting_mediafile_ids": [1, 2, 4], "logo_web_header_id": 1, "font_bold_id": 2, + "font_regular_id": 4, "meeting_user_ids": [11, 12, 13, 14, 15, 16], **{field: [1] for field in Meeting.all_default_projectors()}, **self.get_meeting_defaults(), @@ -344,6 +316,7 @@ def test_correct_relations(self) -> None: "name": "admin group", "weight": 1, "admin_group_for_meeting_id": 1, + "meeting_mediafile_inherited_access_group_ids": [4], }, "user/1": { "username": "no", @@ -387,6 +360,11 @@ def test_correct_relations(self) -> None: "meeting_user_ids": [16], }, ), + "gender/2": { + "id": 1, + "organization_id": 1, + "name": "male", + }, "meeting_user/11": { "user_id": 1, "meeting_id": 1, @@ -469,6 +447,18 @@ def test_correct_relations(self) -> None: }, "mediafile/1": {"owner_id": "meeting/1", "meeting_mediafile_ids": [1]}, "mediafile/2": {"owner_id": "meeting/1", "meeting_mediafile_ids": [2]}, + "mediafile/3": { + "owner_id": ONE_ORGANIZATION_FQID, + "published_to_meetings_in_organization_id": ONE_ORGANIZATION_ID, + "is_directory": True, + "child_ids": [4], + }, + "mediafile/4": { + "owner_id": ONE_ORGANIZATION_FQID, + "published_to_meetings_in_organization_id": ONE_ORGANIZATION_ID, + "parent_id": 3, + "meeting_mediafile_ids": [4], + }, "meeting_mediafile/1": { "meeting_id": 1, "mediafile_id": 1, @@ -481,6 +471,13 @@ def test_correct_relations(self) -> None: "is_public": True, "used_as_font_bold_in_meeting_id": 1, }, + "meeting_mediafile/4": { + "meeting_id": 1, + "mediafile_id": 4, + "used_as_font_regular_in_meeting_id": 1, + "is_public": False, + "inherited_access_group_ids": [2], + }, "motion/1": { "submitter_ids": [5], "meeting_id": 1, @@ -756,15 +753,206 @@ def test_relation_2(self) -> None: assert not data["errors"] def test_no_permissions(self) -> None: + self.create_meeting() + self.set_organization_management_level( + OrganizationManagementLevel.CAN_MANAGE_ORGANIZATION + ) + status_code, data = self.request("check_database", {}) + assert status_code == 403 + assert "Missing permission: superadmin" in data["message"] + + def test_with_structured_published_orga_files(self) -> None: self.set_models( { - "meeting/1": {"name": "test_foo"}, - "user/1": { - "username": "no", - "organization_management_level": OrganizationManagementLevel.CAN_MANAGE_ORGANIZATION, + ONE_ORGANIZATION_FQID: { + "active_meeting_ids": [1], + "organization_tag_ids": [1], + "mediafile_ids": [1, 2, 3, 4, 5], + "published_mediafile_ids": [1, 2, 3, 4, 5], + }, + "committee/1": {"name": "!", "organization_id": 1}, + "organization_tag/1": { + "name": "TEST", + "color": "#eeeeee", + "organization_id": 1, + }, + "meeting/1": { + "committee_id": 1, + "language": "en", + "name": "Test", + "description": "blablabla", + "default_group_id": 1, + "admin_group_id": 2, + "motions_default_amendment_workflow_id": 1, + "motions_default_workflow_id": 1, + "reference_projector_id": 1, + "projector_countdown_default_time": 60, + "projector_countdown_warning_time": 5, + "projector_ids": [1], + "motion_state_ids": [1], + "motion_workflow_ids": [1], + "is_active_in_organization_id": 1, + **self.get_meeting_defaults(), + **{field: [1] for field in Meeting.all_default_projectors()}, + "meeting_mediafile_ids": [10, 20, 30, 40, 50], + "group_ids": [1, 2, 3], + }, + "group/1": { + "meeting_id": 1, + "name": "default group", + "weight": 1, + "default_group_for_meeting_id": 1, + "meeting_mediafile_access_group_ids": [10, 40], + "meeting_mediafile_inherited_access_group_ids": [10, 20, 30, 40], + }, + "group/2": { + "meeting_id": 1, + "name": "admin group", + "weight": 1, + "admin_group_for_meeting_id": 1, + "meeting_mediafile_access_group_ids": [10], + "meeting_mediafile_inherited_access_group_ids": [10, 20, 30], + }, + "group/3": { + "meeting_id": 1, + "weight": 1, + "name": "third group", + "meeting_mediafile_access_group_ids": [50], + }, + "motion_workflow/1": { + "meeting_id": 1, + "name": "blup", + "first_state_id": 1, + "default_amendment_workflow_meeting_id": 1, + "default_workflow_meeting_id": 1, + "state_ids": [1], + "sequential_number": 1, + }, + "motion_state/1": { + "css_class": "lightblue", + "meeting_id": 1, + "workflow_id": 1, + "name": "test", + "weight": 1, + "workflow_id": 1, + "first_state_of_workflow_id": 1, + "restrictions": [], + "allow_support": False, + "allow_create_poll": False, + "allow_submitter_edit": False, + "set_number": True, + "show_state_extension_field": False, + "merge_amendment_into_final": "undefined", + "show_recommendation_extension_field": False, + }, + "projector/1": { + "sequential_number": 1, + "meeting_id": 1, + "used_as_reference_projector_meeting_id": 1, + "name": "Default projector", + "scale": 0, + "scroll": 0, + "width": 1200, + "aspect_ratio_numerator": 16, + "aspect_ratio_denominator": 9, + "color": "#000000", + "background_color": "#ffffff", + "header_background_color": "#317796", + "header_font_color": "#ffffff", + "header_h1_color": "#ffffff", + "chyron_background_color": "#ffffff", + "chyron_font_color": "#ffffff", + "show_header_footer": True, + "show_title": True, + "show_logo": True, + "show_clock": True, + **{field: 1 for field in Meeting.reverse_default_projectors()}, + }, + "mediafile/1": { + "title": "Mother of all directories (MOAD)", + "owner_id": ONE_ORGANIZATION_FQID, + "child_ids": [2, 3, 4, 5], + "is_directory": True, + "meeting_mediafile_ids": [10], + "published_to_meetings_in_organization_id": 1, + }, + "meeting_mediafile/10": { + "is_public": False, + "meeting_id": 1, + "mediafile_id": 1, + "access_group_ids": [1, 2], + "inherited_access_group_ids": [1, 2], + }, + "mediafile/2": { + "title": "Child_of_mother_of_all_directories.xlsx", + "filename": "COMOAD.xlsx", + "filesize": 10000, + "mimetype": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "owner_id": ONE_ORGANIZATION_FQID, + "parent_id": 1, + "meeting_mediafile_ids": [20], + "published_to_meetings_in_organization_id": 1, + }, + "meeting_mediafile/20": { + "is_public": False, + "meeting_id": 1, + "mediafile_id": 2, + "inherited_access_group_ids": [1, 2], + }, + "mediafile/3": { + "title": "Child_of_mother_of_all_directories.pdf", + "filename": "COMOAD.pdf", + "filesize": 750000, + "mimetype": "application/pdf", + "owner_id": ONE_ORGANIZATION_FQID, + "parent_id": 1, + "pdf_information": Jsonb({"pages": 1}), + "meeting_mediafile_ids": [30], + "published_to_meetings_in_organization_id": 1, + }, + "meeting_mediafile/30": { + "is_public": False, + "meeting_id": 1, + "mediafile_id": 3, + "inherited_access_group_ids": [1, 2], + }, + "mediafile/4": { + "title": "Child_of_mother_of_all_directories_with_limited_access.txt", + "filename": "COMOADWLA.txt", + "filesize": 100, + "mimetype": "text/plain", + "owner_id": ONE_ORGANIZATION_FQID, + "parent_id": 1, + "meeting_mediafile_ids": [40], + "published_to_meetings_in_organization_id": 1, + }, + "meeting_mediafile/40": { + "is_public": False, + "meeting_id": 1, + "mediafile_id": 4, + "access_group_ids": [1], + "inherited_access_group_ids": [1], + }, + "mediafile/5": { + "title": "Hidden_child_of_mother_of_all_directories.csv", + "filename": "HCOMOAD.csv", + "filesize": 420, + "mimetype": "text/csv", + "owner_id": ONE_ORGANIZATION_FQID, + "parent_id": 1, + "meeting_mediafile_ids": [50], + "published_to_meetings_in_organization_id": 1, + }, + "meeting_mediafile/50": { + "is_public": False, + "meeting_id": 1, + "mediafile_id": 5, + "access_group_ids": [3], + "inherited_access_group_ids": [], }, } ) status_code, data = self.request("check_database", {}) - assert status_code == 403 - assert "Missing permission: superadmin" in data["message"] + assert status_code == 200 + assert data["ok"] is True + assert not data["errors"] diff --git a/tests/system/presenter/test_check_database_all.py b/tests/system/presenter/test_check_database_all.py index 5db363c6f1..f5a174e8fd 100644 --- a/tests/system/presenter/test_check_database_all.py +++ b/tests/system/presenter/test_check_database_all.py @@ -1,8 +1,6 @@ -from time import time +from datetime import datetime, timedelta from typing import Any -import pytest - from openslides_backend.action.action_worker import ActionWorkerState from openslides_backend.models.models import Meeting from openslides_backend.permissions.management_levels import OrganizationManagementLevel @@ -11,21 +9,7 @@ from .base import BasePresenterTestCase -@pytest.mark.skip(reason="During development of relational DB not necessary") class TestCheckDatabaseAll(BasePresenterTestCase): - def test_found_errors(self) -> None: - self.set_models( - { - "meeting/1": {"name": "test_foo"}, - "meeting/2": {"name": "test_bar"}, - } - ) - status_code, data = self.request("check_database_all", {}) - assert status_code == 200 - assert data["ok"] is False - assert "meeting/1: Missing fields" in data["errors"] - assert "meeting/2: Missing fields" in data["errors"] - def get_meeting_defaults(self) -> dict[str, Any]: return { "motions_export_title": "Motions", @@ -277,13 +261,14 @@ def test_correct(self) -> None: "action_worker/1": { "name": "testcase", "state": ActionWorkerState.END, - "created": round(time() - 3), - "timestamp": round(time()), + "created": datetime.now() - timedelta(minutes=30), + "timestamp": datetime.now(), + "user_id": 1, }, "import_preview/1": { "name": "topic", "state": "done", - "created": round(time() - 3), + "created": datetime.now() - timedelta(minutes=30), }, } ) @@ -511,7 +496,6 @@ def test_correct_relations(self) -> None: "weight": 1, "workflow_id": 1, "first_state_of_workflow_id": 1, - "restrictions": [], "allow_support": False, "allow_create_poll": False, "allow_submitter_edit": False, @@ -885,13 +869,9 @@ def test_relation_2(self) -> None: assert "errors" not in data def test_no_permissions(self) -> None: - self.set_models( - { - "meeting/1": {"name": "test_foo"}, - "user/1": { - "organization_management_level": OrganizationManagementLevel.CAN_MANAGE_ORGANIZATION - }, - } + self.create_meeting() + self.set_organization_management_level( + OrganizationManagementLevel.CAN_MANAGE_ORGANIZATION ) status_code, data = self.request("check_database_all", {}) assert status_code == 403 diff --git a/tests/unit/test_checker.py b/tests/unit/test_checker.py new file mode 100644 index 0000000000..6ae5a867b3 --- /dev/null +++ b/tests/unit/test_checker.py @@ -0,0 +1,863 @@ +from copy import deepcopy +from datetime import datetime +from decimal import Decimal +from typing import Any, Literal +from unittest import TestCase + +from psycopg.types.json import Jsonb + +from openslides_backend.migrations import get_backend_migration_index +from openslides_backend.models.base import model_registry +from openslides_backend.models.checker import Checker, CheckException +from openslides_backend.models.fields import ( + BooleanField, + CharArrayField, + CharField, + ColorField, + DecimalField, + FloatField, + GenericRelationField, + GenericRelationListField, + HTMLPermissiveField, + HTMLStrictField, + IntegerField, + RelationField, + RelationListField, + TextField, + TimestampField, +) +from openslides_backend.shared.util import ONE_ORGANIZATION_FQID + +BACKEND_MIGRATION_INDEX = get_backend_migration_index() + + +class TestCheckerCheckMigrationIndex(TestCase): + def check_migration_index( + self, + data: dict[str, Any], + expected_error: str | None = None, + migration_mode: Literal["strict", "permissive"] = "strict", + ) -> None: + try: + Checker( + data=data, + mode="internal", + migration_mode=migration_mode, + ).run_check() + self.assertIsNone(expected_error) + except CheckException as ce: + self.assertEqual(ce.args[0], expected_error) + + def test_migration_index_correct(self) -> None: + self.check_migration_index({"_migration_index": BACKEND_MIGRATION_INDEX}) + self.check_migration_index( + {"_migration_index": BACKEND_MIGRATION_INDEX - 1}, + migration_mode="permissive", + ) + + def test_migration_index_is_none_error(self) -> None: + self.check_migration_index( + data={"_migration_index": None}, + expected_error="JSON does not match schema: data._migration_index must be integer", + ) + + def test_no_migration_index_error(self) -> None: + self.check_migration_index( + data={}, + expected_error="JSON does not match schema: data must contain ['_migration_index'] properties", + ) + + def test_migration_index_too_small_error(self) -> None: + self.check_migration_index( + data={"_migration_index": 0}, + expected_error="JSON does not match schema: data._migration_index must be bigger than or equal to 1", + ) + + def test_migration_index_lower_than_backend_MI_permissive_mode(self) -> None: + migration_index = BACKEND_MIGRATION_INDEX - 1 + self.check_migration_index( + data={"_migration_index": migration_index}, + migration_mode="permissive", + ) + + def test_migration_index_higher_than_backend_MI_error(self) -> None: + migration_index = BACKEND_MIGRATION_INDEX + 1 + msg = f"\tThe given migration index ({migration_index}) is higher than the backend ({BACKEND_MIGRATION_INDEX})." + + self.check_migration_index( + data={"_migration_index": migration_index}, + expected_error=msg, + ) + self.check_migration_index( + data={"_migration_index": migration_index}, + expected_error=msg, + migration_mode="permissive", + ) + + def test_migration_index_lower_than_backend_MI_error(self) -> None: + migration_index = BACKEND_MIGRATION_INDEX - 1 + self.check_migration_index( + data={"_migration_index": migration_index}, + expected_error=f"\tThe given migration index ({migration_index}) is lower than the backend ({BACKEND_MIGRATION_INDEX}).", + ) + + +class TestCheckerCheckData(TestCase): + def setUp(self) -> None: + super().setUp() + self.migration_index: dict[str, Any] = { + "_migration_index": BACKEND_MIGRATION_INDEX + } + self.theme_data: dict[str, Any] = { + "organization": {"1": {"id": 1, "theme_id": 1, "theme_ids": [1]}}, + "theme": { + "1": { + "id": 1, + "name": "Theme 1", + "organization_id": 1, + "theme_for_organization_id": 1, + } + }, + } + self.meeting_data: dict[str, Any] = { + "organization": { + "1": { + "id": 1, + "theme_id": 1, + "theme_ids": [1], + "committee_ids": [1], + "active_meeting_ids": [1], + } + }, + "theme": { + "1": { + "id": 1, + "name": "Theme 1", + "organization_id": 1, + "theme_for_organization_id": 1, + } + }, + "committee": { + "1": { + "id": 1, + "name": "Committee 1", + "organization_id": 1, + "meeting_ids": [1], + } + }, + "meeting": { + "1": { + "id": 1, + "committee_id": 1, + "is_active_in_organization_id": 1, + "motions_default_amendment_workflow_id": 1, + "motions_default_workflow_id": 1, + "group_ids": [1], + "default_group_id": 1, + "projector_ids": [1], + "reference_projector_id": 1, + "default_projector_motion_poll_ids": [1], + "default_projector_message_ids": [1], + "default_projector_countdown_ids": [1], + "default_projector_agenda_item_list_ids": [1], + "default_projector_assignment_poll_ids": [1], + "default_projector_mediafile_ids": [1], + "default_projector_current_los_ids": [1], + "default_projector_motion_block_ids": [1], + "default_projector_topic_ids": [1], + "default_projector_list_of_speakers_ids": [1], + "default_projector_amendment_ids": [1], + "default_projector_assignment_ids": [1], + "default_projector_motion_ids": [1], + "default_projector_poll_ids": [1], + "motion_workflow_ids": [1], + "motion_state_ids": [1], + } + }, + "group": { + "1": { + "id": 1, + "meeting_id": 1, + "name": "default", + "default_group_for_meeting_id": 1, + } + }, + "projector": { + "1": { + "id": 1, + "meeting_id": 1, + "used_as_reference_projector_meeting_id": 1, + "used_as_default_projector_for_agenda_item_list_in_meeting_id": 1, + "used_as_default_projector_for_topic_in_meeting_id": 1, + "used_as_default_projector_for_list_of_speakers_in_meeting_id": 1, + "used_as_default_projector_for_current_los_in_meeting_id": 1, + "used_as_default_projector_for_motion_in_meeting_id": 1, + "used_as_default_projector_for_amendment_in_meeting_id": 1, + "used_as_default_projector_for_motion_block_in_meeting_id": 1, + "used_as_default_projector_for_assignment_in_meeting_id": 1, + "used_as_default_projector_for_mediafile_in_meeting_id": 1, + "used_as_default_projector_for_message_in_meeting_id": 1, + "used_as_default_projector_for_countdown_in_meeting_id": 1, + "used_as_default_projector_for_assignment_poll_in_meeting_id": 1, + "used_as_default_projector_for_motion_poll_in_meeting_id": 1, + "used_as_default_projector_for_poll_in_meeting_id": 1, + } + }, + "motion_workflow": { + "1": { + "id": 1, + "name": "flo", + "meeting_id": 1, + "first_state_id": 1, + "default_workflow_meeting_id": 1, + "default_amendment_workflow_meeting_id": 1, + "state_ids": [1], + } + }, + "motion_state": { + "1": { + "id": 1, + "name": "stasis", + "weight": 1, + "meeting_id": 1, + "workflow_id": 1, + "first_state_of_workflow_id": 1, + } + }, + } + self.organization_tag_data: dict[str, Any] = { + "organization_tag": { + "1": { + "id": 1, + "name": "test", + "color": "#11aaee", + "organization_id": 1, + } + } + } + self.extended_user_data: dict[str, Any] = { + "user": { + "1": { + "id": 1, + "organization_id": 1, + "username": "johndoe", + "default_vote_weight": Decimal("1.00000"), + "is_active": True, + "is_physical_person": True, + "can_change_own_password": False, + "committee_ids": [1], + } + } + } + + def check_data( + self, + data: dict[str, Any], + expected_error: str | list[str] | None = None, + mode: Literal["internal", "external", "all"] = "all", + repair: bool = True, + fields_to_remove: dict[str, list[str]] = {}, + ) -> None: + try: + Checker( + data=self.migration_index | data, + mode=mode, + repair=repair, + fields_to_remove=fields_to_remove, + ).run_check() + self.assertIsNone(expected_error) + except CheckException as ce: + error_message = ce.args[0] + if isinstance(expected_error, list): + for message_part in expected_error: + self.assertIn(message_part, error_message) + else: + self.assertEqual(error_message, expected_error) + + # check_collections() + def test_collections_do_not_match_models_error(self) -> None: + invalid_collections = [ + "invalid_regular_collection", + "another_one", + ] + self.check_data( + data={collection: {} for collection in invalid_collections}, + repair=False, + expected_error=[ + "Collections in file do not match with models.py. Invalid collections:", + ] + + invalid_collections, + ) + + def test_collections_do_not_match_meeting_models_error(self) -> None: + self.check_data( + data={"theme": {"1": {"id": 1}}}, + mode="internal", + repair=False, + expected_error="Collections in file do not match with models.py. Invalid collections: theme.", + ) + + # check id + def test_id_mismatch_error(self) -> None: + self.check_data( + data={"theme": {"1": {"id": 2, "name": "Theme 1", "organization_id": 1}}}, + expected_error=["\ttheme/1: Id must be the same as model['id']"], + ) + + # check_normal_fields() + def test_skipped_fields_error(self) -> None: + invalid_data = {"accent_501": "#123432", "primary_499": "#83aa87"} + self.theme_data["theme"]["1"].update(invalid_data) + self.check_data( + data=self.theme_data, + repair=False, + expected_error=["\ttheme/1: Invalid fields "] + list(invalid_data.keys()), + ) + + def test_fix_missing_default_values_repair_true(self) -> None: + self.check_data(self.theme_data) + + def test_fix_missing_default_values_repair_false_error(self) -> None: + self.check_data( + data=self.theme_data, + repair=False, + expected_error=[ + "\ttheme/1: Missing fields ", + "accent_500", + "primary_500", + "warn_500", + ], + ) + + def test_missing_required_field_error(self) -> None: + del self.theme_data["theme"]["1"]["name"] + self.check_data( + data=self.theme_data, + expected_error="\ttheme/1: Missing fields name", + ) + + # check_types() + def test_empty_required_field_error(self) -> None: + self.theme_data["theme"]["1"]["name"] = None + self.check_data( + data=self.theme_data, + expected_error="\ttheme/1/name: Field required but empty.", + ) + + def test_correct_meeting(self) -> None: + """Also checks that no errors are raised for missing sequential_numbers.""" + self.check_data(self.meeting_data) + + def test_required_skip_special_fields(self) -> None: + skip_fields = [ + { + "field_name": "committee_id", + "related_collection": "committee", + "back_relation": "meeting_ids", + }, + { + "field_name": "is_active_in_organization_id", + "related_collection": "organization", + "back_relation": "active_meeting_ids", + }, + ] + for relation in skip_fields: + data = deepcopy(self.meeting_data) + data["meeting"]["1"][relation["field_name"]] = None + del data[relation["related_collection"]]["1"][relation["back_relation"]] + self.check_data(data) + + def test_invalid_enum_error(self) -> None: + self.theme_data["organization"]["1"]["default_language"] = "1337" + self.check_data( + data=self.theme_data, + expected_error="\torganization/1/default_language: Value error: Value 1337 is not a valid enum value", + ) + + def test_correct_types(self) -> None: + self.meeting_data.update( + { + **self.organization_tag_data, + "projection": {"1": {"id": 1, "meeting_id": 1}}, + "gender": { + "1": { + "id": 1, + "organization_id": 1, + "name": "male", + "user_ids": [1], + } + }, + "user": {"1": {"id": 1, "organization_id": 1, "username": "johndoe"}}, + } + ) + self.meeting_data["meeting"]["1"].update( + { + "organization_tag_ids": [1], + "all_projection_ids": [1], + "projection_ids": [1], + } + ) + self.meeting_data["organization"]["1"].update( + {"organization_tag_ids": [1], "gender_ids": [1]} + ) + + correct_value_types: list[dict[str, Any]] = [ + { + "field_type": CharField, + "collection": "organization", + "field_name": "name", + "value": "OpenSlides", + }, + { + "field_type": HTMLStrictField, + "collection": "organization", + "field_name": "description", + "value": "Descriptive text", + }, + { + "field_type": HTMLPermissiveField, + "collection": "meeting", + "field_name": "welcome_text", + "value": "Frieldnly welcome text", + }, + { + "field_type": GenericRelationField, + "collection": "projection", + "field_name": "content_object_id", + "value": "meeting/1", + }, + { + "field_type": IntegerField, + "collection": "organization", + "field_name": "limit_of_users", + "value": 100, + }, + { + "field_type": TimestampField, + "collection": "meeting", + "field_name": "start_time", + "value": 123, + }, + { + "field_type": TimestampField, + "collection": "meeting", + "field_name": "end_time", + "value": datetime.fromtimestamp(124), + }, + { + "field_type": RelationField, + "collection": "user", + "field_name": "gender_id", + "value": 1, + }, + { + "field_type": FloatField, + "collection": "meeting", + "field_name": "export_pdf_line_height", + "value": 1.25, + }, + { + "field_type": BooleanField, + "collection": "organization", + "field_name": "enable_anonymous", + "value": False, + }, + { + "field_type": CharArrayField, + "collection": "group", + "field_name": "permissions", + "value": ["1", "2"], + }, + { + "field_type": GenericRelationListField, + "collection": "organization_tag", + "field_name": "tagged_ids", + "value": ["meeting/1"], + }, + { + "field_type": RelationListField, + "collection": "gender", + "field_name": "user_ids", + "value": [1], + }, + { + "field_type": DecimalField, + "collection": "user", + "field_name": "default_vote_weight", + "value": "1.0000", + }, + { + "field_type": DecimalField, + "collection": "user", + "field_name": "default_vote_weight", + "value": Decimal("1.0000"), + }, + { + "field_type": ColorField, + "collection": "theme", + "field_name": "accent_500", + "value": "#e412a3", + }, + { + "field_type": TextField, + "collection": "meeting", + "field_name": "motions_preamble", + "value": "The assembly may decide:", + }, + ] + + # Check correct values + for field in correct_value_types: + self.meeting_data[field["collection"]]["1"][field["field_name"]] = field[ + "value" + ] + self.check_data(self.meeting_data) + + # Check None + self.meeting_data["meeting"]["1"].update( + {"organization_tag_ids": None, "projection_ids": None} + ) + for field in correct_value_types: + if field["field_type"] != TimestampField: + self.meeting_data[field["collection"]]["1"][field["field_name"]] = None + self.check_data( + data=self.meeting_data, + expected_error="\tprojection/1/content_object_id: Field required but empty.", + ) + + def test_correct_types_json(self) -> None: + raw_values = [ + None, + [1, "2"], + {"first": 1, "second": "2", "third": False}, + ] + json_values = raw_values + [Jsonb(v) for v in raw_values] + for value in json_values: + self.theme_data["organization"]["1"]["saml_attr_mapping"] = value + self.check_data(data=self.theme_data) + + def test_incorrect_types_json(self) -> None: + raw_values = [1, "2", 3.0, {1, 2}, {"second": 2.0}, Decimal("4.567")] + json_values = raw_values + [Jsonb(v) for v in raw_values] + field_type = model_registry["organization"]().get_field("saml_attr_mapping") + error = [ + f"organization/1/saml_attr_mapping: Type error: Type is not {field_type}" + ] + + for value in json_values: + self.theme_data["organization"]["1"]["saml_attr_mapping"] = value + self.check_data( + data=self.theme_data, + expected_error=error, + ) + + def test_incorrect_fqid_error(self) -> None: + base_error = "projection/1/content_object_id: Type error: Type is not GenericRelationField(to={'projector_countdown': 'projection_ids', 'projector_message': 'projection_ids', 'poll': 'projection_ids', 'topic': 'projection_ids', 'agenda_item': 'projection_ids', 'assignment': 'projection_ids', 'motion_block': 'projection_ids', 'list_of_speakers': 'projection_ids', 'meeting_mediafile': 'projection_ids', 'motion': 'projection_ids', 'meeting': 'projection_ids'}, is_list_field=False, on_delete=SET_NULL, required=True, constraints={}, equal_fields=['meeting_id'])" + map_invalid_values_to_special_errors = { + "meetings/1": "projection/1/content_object_id error: The collection meetings is not supported as a reverse relation in projection/content_object_id.", + "meeting/a": None, + "just_a_string": None, + "no_id/": None, + "/1": None, + } + + self.meeting_data.update({"projection": {"1": {"id": 1, "meeting_id": 1}}}) + self.meeting_data["meeting"]["1"]["all_projection_ids"] = [1] + + for value, value_error in map_invalid_values_to_special_errors.items(): + self.meeting_data["projection"]["1"]["content_object_id"] = value + if not value_error: + value_error = ( + f"projection/1/content_object_id error: Fqid {value} is malformed" + ) + self.check_data( + data=self.meeting_data, + expected_error=[base_error, value_error], + ) + + def test_incorrect_fqid_list_error(self) -> None: + base_error = "organization_tag/1/tagged_ids: Type error: Type is not GenericRelationListField(to={'committee': 'organization_tag_ids', 'meeting': 'organization_tag_ids'}, is_list_field=True, on_delete=SET_NULL, required=False, constraints={}, equal_fields=[])" + map_invalid_values_to_special_errors = { + "meetings/1": "organization_tag/1/tagged_ids error: The collection meetings is not supported as a reverse relation in organization_tag/tagged_ids.", + "meeting/a": None, + "just_a_string": None, + "no_id/": None, + "/1": None, + } + + self.meeting_data.update(self.organization_tag_data) + self.meeting_data["organization"]["1"].update({"organization_tag_ids": [1]}) + + for value, value_error in map_invalid_values_to_special_errors.items(): + self.meeting_data["organization_tag"]["1"]["tagged_ids"] = [value] + if not value_error: + value_error = ( + f"organization_tag/1/tagged_ids error: Fqid {value} is malformed" + ) + self.check_data( + data=self.meeting_data, + expected_error=[base_error, value_error], + ) + + def test_incorrect_value_type_list_field_error(self) -> None: + list_fields: dict[str, Any] = { + "group": "permissions", + "organization_tag": "tagged_ids", + "organization": "gender_ids", + } + invalid_values = [set(), dict(), 1, "2", True, 4.0, Decimal("5.600")] + self.meeting_data.update(self.organization_tag_data) + for value in invalid_values: + for collection, field_name in list_fields.items(): + self.meeting_data[collection]["1"][field_name] = value + # TODO: after unifying type checking logic also check field type in the message + self.check_data( + data=self.meeting_data, + expected_error=[ + f"{collection}/1/{field_name}: Type error: Type is not" + for field in list_fields + ], + ) + + # check_special_fields() + def set_motion_data(self, motion_ids: list[int]) -> None: + self.meeting_data["meeting"]["1"]["list_of_speakers_ids"] = motion_ids + self.meeting_data["meeting"]["1"]["motion_ids"] = motion_ids + self.meeting_data["motion_state"]["1"]["motion_ids"] = motion_ids + self.meeting_data["motion"] = { + str(id_): { + "id": id_, + "meeting_id": 1, + "state_id": 1, + "title": f"motion {id_}", + "list_of_speakers_id": id_, + } + for id_ in motion_ids + } + self.meeting_data["list_of_speakers"] = { + str(id_): { + "id": id_, + "content_object_id": f"motion/{id_}", + "meeting_id": 1, + } + for id_ in motion_ids + } + + def test_amendment_paragraphs_error(self) -> None: + self.set_motion_data([1]) + self.meeting_data["motion"]["1"]["amendment_paragraphs"] = { + "1": "test", + "2": "broken", + "3": 'forbidden attribute', + } + self.check_data( + data=self.meeting_data, + expected_error="\tmotion/1/amendment_paragraphs error: Invalid html in 1\n\tmotion/1/amendment_paragraphs error: Invalid html in 2\n\tmotion/1/amendment_paragraphs error: Invalid html in 3", + ) + + def test_motion_extensions_error(self) -> None: + self.set_motion_data([1, 2]) + errors = [] + for field_name in ["recommendation_extension", "state_extension"]: + self.meeting_data["motion"]["1"][field_name] = "ext [motion/3] [theme/1]" + errors.append( + f"\tmotion/1/{field_name}: Relation Error: Found motion/3 in {field_name} but not in models.\n\tmotion/1/{field_name}: Relation Error: Found theme/1 but only motion is allowed." + ) + + self.check_data(data=self.meeting_data, expected_error=errors) + + def test_external_mode_forbidden_field_error(self) -> None: + for collection in ["theme", "committee", "organization"]: + del self.meeting_data[collection] + self.meeting_data.update( + {"mediafile": {"1": {"id": 1, "owner_id": ONE_ORGANIZATION_FQID}}} + ) + self.check_data( + data=self.meeting_data, + mode="external", + expected_error="\tmeeting/1/committee_id: Relation Error: points to committee/1, which is not allowed in an external import.\n\tmeeting/1/is_active_in_organization_id: Relation Error: points to organization/1, which is not allowed in an external import.\n\tmediafile/1/owner_id error: Fqid organization/1 has an invalid collection.", + ) + + def test_external_mode_forbidden_field_repair_false_error(self) -> None: + self.check_data( + data=self.extended_user_data, + mode="external", + repair=False, + fields_to_remove={"user": ["committee_ids"]}, + expected_error="\tuser/1/committee_ids: Relation Error: points to committee/user_ids, which is not allowed in an external import.", + ) + + def test_external_mode_forbidden_field_in_fields_to_remove_repair_true( + self, + ) -> None: + self.check_data( + data=self.extended_user_data, + mode="external", + fields_to_remove={"user": ["committee_ids"]}, + ) + + def test_reverse_relation_corrupt_error(self) -> None: + self.meeting_data["committee"]["1"]["meeting_ids"] = None + self.check_data( + data=self.meeting_data, + expected_error="\tmeeting/1/committee_id: Relation Error: points to committee/1/meeting_ids, but the reverse relation for it is corrupt.", + ) + + # check_calculated_fields() + def test_calculated_fields(self) -> None: + """ + Check that no errors are raised for: + * meeting-wide mediafiles with and without meeting_mediafiles + * orga-wide mediafiles: grand-parent and parent without meeting_mediafiles, + child with meeting_mediafile and used as font in the meeting + """ + mediafiles_data: dict[str, Any] = { + "mediafile": { + "1": { + "id": 1, + "owner_id": "meeting/1", + "child_ids": [2], + "meeting_mediafile_ids": [11], + }, + "2": { + "id": 2, + "owner_id": "meeting/1", + "parent_id": 1, + "meeting_mediafile_ids": [12], + }, + "3": { + "id": 3, + "owner_id": ONE_ORGANIZATION_FQID, + "is_directory": True, + "child_ids": [4], + }, + "4": { + "id": 4, + "owner_id": ONE_ORGANIZATION_FQID, + "is_directory": True, + "parent_id": 3, + "child_ids": [5], + }, + "5": { + "id": 5, + "owner_id": ONE_ORGANIZATION_FQID, + "parent_id": 4, + "meeting_mediafile_ids": [15], + }, + }, + "meeting_mediafile": { + "11": { + "id": 11, + "mediafile_id": 1, + "meeting_id": 1, + "is_public": False, + "access_group_ids": [1], + "inherited_access_group_ids": [1], + }, + "12": { + "id": 12, + "mediafile_id": 2, + "meeting_id": 1, + "is_public": False, + "inherited_access_group_ids": [1], + }, + "15": { + "id": 15, + "mediafile_id": 5, + "meeting_id": 1, + "is_public": False, + "access_group_ids": [2], + "inherited_access_group_ids": [2], + "used_as_font_regular_in_meeting_id": 1, + }, + }, + } + self.meeting_data["meeting"]["1"].update( + { + "admin_group_id": 2, + "group_ids": [1, 2], + "mediafile_ids": [1, 2], + "meeting_mediafile_ids": [11, 12, 15], + "font_regular_id": 15, + } + ) + self.meeting_data["group"]["1"].update( + { + "meeting_mediafile_access_group_ids": [11], + "meeting_mediafile_inherited_access_group_ids": [11, 12], + } + ) + self.meeting_data["group"]["2"] = { + "id": 2, + "meeting_id": 1, + "name": "admin", + "admin_group_for_meeting_id": 1, + "meeting_mediafile_access_group_ids": [15], + "meeting_mediafile_inherited_access_group_ids": [15], + } + self.meeting_data["organization"]["1"]["mediafile_ids"] = [3, 4, 5] + self.check_data( + data=self.meeting_data | mediafiles_data, + ) + + def test_calculated_fields_error(self) -> None: + mediafiles_data: dict[str, Any] = { + "mediafile": { + "1": { + "id": 1, + "owner_id": ONE_ORGANIZATION_FQID, + "child_ids": [2], + }, + "2": { + "id": 2, + "owner_id": ONE_ORGANIZATION_FQID, + "parent_id": 1, + "meeting_mediafile_ids": [12], + }, + }, + "meeting_mediafile": { + "12": { + "id": 12, + "mediafile_id": 2, + "meeting_id": 1, + "is_public": True, + }, + }, + } + self.meeting_data["meeting"]["1"].update( + { + "admin_group_id": 2, + "group_ids": [1, 2], + "meeting_mediafile_ids": [12], + } + ) + self.meeting_data["group"]["2"] = { + "id": 2, + "meeting_id": 1, + "name": "admin", + "admin_group_for_meeting_id": 1, + } + self.meeting_data["organization"]["1"]["mediafile_ids"] = [1, 2] + self.check_data( + data=self.meeting_data | mediafiles_data, + expected_error="\tmeeting_mediafile/12: is_public is wrong. False != True\n\tmeeting_mediafile/12: inherited_access_group_ids is wrong", + ) + + # get_to_generic_case() + def test_get_to_generic_case_error(self) -> None: + self.meeting_data.update( + { + "projection": { + "1": { + "id": 1, + "meeting_id": 1, + "content_object_id": "theme/1", + } + }, + } + ) + self.meeting_data["meeting"]["1"].update({"all_projection_ids": [1]}) + self.check_data( + data=self.meeting_data, + expected_error="\tprojection/1/content_object_id error: The collection theme is not supported as a reverse relation in projection/content_object_id.", + )