|
| 1 | +from collections import defaultdict |
| 2 | +from enum import Enum, auto |
| 3 | +from time import time |
| 4 | +from typing import Any, Literal, TypedDict |
| 5 | + |
| 6 | +from datastore.migrations import BaseModelMigration |
| 7 | +from datastore.reader.core import GetManyRequestPart |
| 8 | +from datastore.writer.core.write_request import ( |
| 9 | + BaseRequestEvent, |
| 10 | + RequestDeleteEvent, |
| 11 | + RequestUpdateEvent, |
| 12 | +) |
| 13 | + |
| 14 | +from ...shared.filters import And, FilterOperator, Or |
| 15 | +from ...shared.patterns import collection_and_id_from_fqid |
| 16 | + |
| 17 | + |
| 18 | +class CountdownCommand(Enum): |
| 19 | + START = auto() |
| 20 | + STOP = auto() |
| 21 | + RESET = auto() |
| 22 | + RESTART = auto() |
| 23 | + |
| 24 | + |
| 25 | +class MigrationDataField(TypedDict): |
| 26 | + to_collection: str |
| 27 | + to_field: str |
| 28 | + on_delete: Literal["cascade", "special"] | None |
| 29 | + |
| 30 | + |
| 31 | +SPEAKER_EXTRA_FIELDS: list[str] = [] |
| 32 | + |
| 33 | +COLLECTION_TO_MIGRATION_FIELDS: dict[str, dict[str, MigrationDataField]] = { |
| 34 | + "meeting_user": { |
| 35 | + "user_id": { |
| 36 | + "to_collection": "user", |
| 37 | + "to_field": "meeting_user_ids", |
| 38 | + "on_delete": None, |
| 39 | + }, |
| 40 | + "meeting_id": { |
| 41 | + "to_collection": "meeting", |
| 42 | + "to_field": "meeting_user_ids", |
| 43 | + "on_delete": None, |
| 44 | + }, |
| 45 | + "personal_note_ids": { |
| 46 | + "to_collection": "personal_note", |
| 47 | + "to_field": "meeting_user_id", |
| 48 | + "on_delete": "cascade", |
| 49 | + }, |
| 50 | + "speaker_ids": { |
| 51 | + "to_collection": "speaker", |
| 52 | + "to_field": "meeting_user_id", |
| 53 | + "on_delete": None, |
| 54 | + }, |
| 55 | + "motion_supporter_ids": { |
| 56 | + "to_collection": "motion_supporter", |
| 57 | + "to_field": "meeting_user_id", |
| 58 | + "on_delete": None, |
| 59 | + }, |
| 60 | + "motion_editor_ids": { |
| 61 | + "to_collection": "motion_editor", |
| 62 | + "to_field": "meeting_user_id", |
| 63 | + "on_delete": None, |
| 64 | + }, |
| 65 | + "motion_working_group_speaker_ids": { |
| 66 | + "to_collection": "motion_working_group_speaker", |
| 67 | + "to_field": "meeting_user_id", |
| 68 | + "on_delete": None, |
| 69 | + }, |
| 70 | + "motion_submitter_ids": { |
| 71 | + "to_collection": "motion_submitter", |
| 72 | + "to_field": "meeting_user_id", |
| 73 | + "on_delete": None, |
| 74 | + }, |
| 75 | + "assignment_candidate_ids": { |
| 76 | + "to_collection": "assignment_candidate", |
| 77 | + "to_field": "meeting_user_id", |
| 78 | + "on_delete": None, |
| 79 | + }, |
| 80 | + "vote_delegated_to_id": { |
| 81 | + "to_collection": "meeting_user", |
| 82 | + "to_field": "vote_delegations_from_ids", |
| 83 | + "on_delete": None, |
| 84 | + }, |
| 85 | + "vote_delegations_from_ids": { |
| 86 | + "to_collection": "meeting_user", |
| 87 | + "to_field": "vote_delegated_to_id", |
| 88 | + "on_delete": None, |
| 89 | + }, |
| 90 | + "chat_message_ids": { |
| 91 | + "to_collection": "chat_message", |
| 92 | + "to_field": "meeting_user_id", |
| 93 | + "on_delete": None, |
| 94 | + }, |
| 95 | + "structure_level_ids": { |
| 96 | + "to_collection": "structure_level", |
| 97 | + "to_field": "meeting_user_ids", |
| 98 | + "on_delete": None, |
| 99 | + }, |
| 100 | + }, |
| 101 | + "speaker": { |
| 102 | + "list_of_speakers_id": { |
| 103 | + "to_collection": "list_of_speakers", |
| 104 | + "to_field": "speaker_ids", |
| 105 | + "on_delete": None, |
| 106 | + }, |
| 107 | + "structure_level_list_of_speakers_id": { |
| 108 | + "to_collection": "structure_level_list_of_speakers", |
| 109 | + "to_field": "speaker_ids", |
| 110 | + "on_delete": None, |
| 111 | + }, |
| 112 | + "point_of_order_category_id": { |
| 113 | + "to_collection": "point_of_order_category", |
| 114 | + "to_field": "speaker_ids", |
| 115 | + "on_delete": None, |
| 116 | + }, |
| 117 | + "meeting_id": { |
| 118 | + "to_collection": "meeting", |
| 119 | + "to_field": "speaker_ids", |
| 120 | + "on_delete": None, |
| 121 | + }, |
| 122 | + }, |
| 123 | + "personal_note": { |
| 124 | + "content_object_id": { |
| 125 | + "to_collection": "motion", |
| 126 | + "to_field": "personal_note_ids", |
| 127 | + "on_delete": "special", # bc generic but only points to motion |
| 128 | + }, |
| 129 | + "meeting_id": { |
| 130 | + "to_collection": "meeting", |
| 131 | + "to_field": "personal_note_ids", |
| 132 | + "on_delete": None, |
| 133 | + }, |
| 134 | + }, |
| 135 | +} |
| 136 | + |
| 137 | + |
| 138 | +def is_list_field(field: str) -> bool: |
| 139 | + return field.endswith("_ids") |
| 140 | + |
| 141 | + |
| 142 | +class Migration(BaseModelMigration): |
| 143 | + """ |
| 144 | + This migration removes meeting_users without groups |
| 145 | + """ |
| 146 | + |
| 147 | + target_migration_index = 75 |
| 148 | + |
| 149 | + def migrate_models(self) -> list[BaseRequestEvent] | None: |
| 150 | + self.end_time = round(time()) |
| 151 | + filter_ = And( |
| 152 | + Or( |
| 153 | + FilterOperator("group_ids", "=", None), |
| 154 | + FilterOperator("group_ids", "=", "[]"), |
| 155 | + ), |
| 156 | + FilterOperator("meta_deleted", "!=", True), |
| 157 | + ) |
| 158 | + musers_to_delete = self.reader.filter( |
| 159 | + "meeting_user", |
| 160 | + filter_, |
| 161 | + list(COLLECTION_TO_MIGRATION_FIELDS["meeting_user"]), |
| 162 | + ) |
| 163 | + speaker_ids_to_delete = self.calculate_speakers_to_delete(musers_to_delete) |
| 164 | + self.collection_to_model_ids_to_delete: dict[str, set[int]] = defaultdict(set) |
| 165 | + self.collection_to_model_ids_to_delete["speaker"] = set(speaker_ids_to_delete) |
| 166 | + |
| 167 | + self.fqids_to_delete: set[str] = { |
| 168 | + f"meeting_user/{id_}" for id_ in musers_to_delete |
| 169 | + } |
| 170 | + self.fqids_to_delete.update({f"speaker/{id_}" for id_ in speaker_ids_to_delete}) |
| 171 | + |
| 172 | + self.fqid_to_list_removal: dict[str, dict[str, list[int]]] = defaultdict( |
| 173 | + lambda: defaultdict(list) |
| 174 | + ) |
| 175 | + self.fqid_to_empty_fields: dict[str, set[str]] = defaultdict(set) |
| 176 | + self.migrate_collection("meeting_user", musers_to_delete) |
| 177 | + while delete_collections := list(self.collection_to_model_ids_to_delete.keys()): |
| 178 | + delete_collection = delete_collections[0] |
| 179 | + delete_collection_ids = self.collection_to_model_ids_to_delete.pop( |
| 180 | + delete_collection, None |
| 181 | + ) |
| 182 | + if delete_collection_ids: |
| 183 | + data_to_delete = self.reader.get_many( |
| 184 | + [ |
| 185 | + GetManyRequestPart( |
| 186 | + delete_collection, |
| 187 | + list(delete_collection_ids), |
| 188 | + list(COLLECTION_TO_MIGRATION_FIELDS[delete_collection]), |
| 189 | + ) |
| 190 | + ] |
| 191 | + ).get(delete_collection, {}) |
| 192 | + self.migrate_collection(delete_collection, data_to_delete) |
| 193 | + events: list[BaseRequestEvent] = [ |
| 194 | + ( |
| 195 | + RequestDeleteEvent(fqid) |
| 196 | + if fqid in self.fqids_to_delete |
| 197 | + else RequestUpdateEvent( |
| 198 | + fqid, |
| 199 | + fields={ |
| 200 | + field: None for field in self.fqid_to_empty_fields.get(fqid, []) |
| 201 | + }, |
| 202 | + list_fields=( |
| 203 | + { |
| 204 | + "remove": { |
| 205 | + field: [val for val in lis] |
| 206 | + for field, lis in list_data.items() |
| 207 | + } |
| 208 | + } |
| 209 | + if (list_data := self.fqid_to_list_removal.get(fqid, {})) |
| 210 | + else {} |
| 211 | + ), |
| 212 | + ) |
| 213 | + ) |
| 214 | + for fqid in self.fqids_to_delete.union( |
| 215 | + self.fqid_to_empty_fields, self.fqid_to_list_removal |
| 216 | + ) |
| 217 | + ] |
| 218 | + return events |
| 219 | + |
| 220 | + def migrate_collection( |
| 221 | + self, collection: str, models_to_delete: dict[int, dict[str, Any]] |
| 222 | + ) -> None: |
| 223 | + self.load_data(collection, models_to_delete) |
| 224 | + for id_, model in models_to_delete.items(): |
| 225 | + for field, value in model.items(): |
| 226 | + if value and ( |
| 227 | + field_data := COLLECTION_TO_MIGRATION_FIELDS[collection].get(field) |
| 228 | + ): |
| 229 | + if is_list_field(field): |
| 230 | + for val_id in value: |
| 231 | + self.handle_id(collection, id_, val_id, field, field_data) |
| 232 | + else: |
| 233 | + self.handle_id(collection, id_, value, field, field_data) |
| 234 | + |
| 235 | + def load_data( |
| 236 | + self, collection: str, models_to_delete: dict[int, dict[str, Any]] |
| 237 | + ) -> None: |
| 238 | + fields = COLLECTION_TO_MIGRATION_FIELDS[collection] |
| 239 | + collection_to_target_model_ids: dict[str, set[int]] = defaultdict(set) |
| 240 | + for field, field_data in fields.items(): |
| 241 | + collection_to_target_model_ids[field_data["to_collection"]].update( |
| 242 | + [ |
| 243 | + id_ if isinstance(id_, int) else id_.split("/")[1] |
| 244 | + for mod in models_to_delete.values() |
| 245 | + for id_ in (mod.get(field) or []) |
| 246 | + ] |
| 247 | + if is_list_field(field) |
| 248 | + else [ |
| 249 | + id_ if isinstance(id_, int) else id_.split("/")[1] |
| 250 | + for mod in models_to_delete.values() |
| 251 | + if (id_ := mod.get(field)) |
| 252 | + ] |
| 253 | + ) |
| 254 | + self.existing_target_models = self.reader.get_many( |
| 255 | + [ |
| 256 | + GetManyRequestPart(coll, list(ids), ["id"]) |
| 257 | + for coll, ids in collection_to_target_model_ids.items() |
| 258 | + ] |
| 259 | + ) |
| 260 | + |
| 261 | + def exists(self, fqid: str) -> bool: |
| 262 | + collection, id_ = collection_and_id_from_fqid(fqid) |
| 263 | + return id_ in self.existing_target_models.get(collection, {}) |
| 264 | + |
| 265 | + def handle_id( |
| 266 | + self, |
| 267 | + base_collection: str, |
| 268 | + base_id: int, |
| 269 | + value_id: int | str, |
| 270 | + field: str, |
| 271 | + field_data: MigrationDataField, |
| 272 | + ) -> None: |
| 273 | + target_fqid = f"{field_data['to_collection']}/{value_id}" |
| 274 | + if target_fqid in self.fqids_to_delete: |
| 275 | + return |
| 276 | + match field_data["on_delete"]: |
| 277 | + case "special": |
| 278 | + if base_collection == "personal_note" and field == "content_object_id": |
| 279 | + # back relation is always list-field personal_note_ids |
| 280 | + assert isinstance(value_id, str) |
| 281 | + target_fqid = value_id |
| 282 | + if target_fqid in self.fqids_to_delete or not self.exists( |
| 283 | + target_fqid |
| 284 | + ): |
| 285 | + return |
| 286 | + self.fqid_to_list_removal[target_fqid]["personal_note_ids"].append( |
| 287 | + base_id |
| 288 | + ) |
| 289 | + else: |
| 290 | + raise Exception( |
| 291 | + f"Bad migration: Handling of {base_collection}/{field} not defined." |
| 292 | + ) |
| 293 | + case "cascade": |
| 294 | + if not self.exists(target_fqid): |
| 295 | + return |
| 296 | + assert isinstance(value_id, int) |
| 297 | + self.collection_to_model_ids_to_delete[field_data["to_collection"]].add( |
| 298 | + value_id |
| 299 | + ) |
| 300 | + self.fqids_to_delete.add(f"{field_data['to_collection']}/{value_id}") |
| 301 | + case _: |
| 302 | + if not self.exists(target_fqid): |
| 303 | + return |
| 304 | + if is_list_field(field_data["to_field"]): |
| 305 | + self.fqid_to_list_removal[target_fqid][ |
| 306 | + field_data["to_field"] |
| 307 | + ].append(base_id) |
| 308 | + else: |
| 309 | + self.fqid_to_empty_fields[target_fqid].add(field_data["to_field"]) |
| 310 | + |
| 311 | + def calculate_speakers_to_delete( |
| 312 | + self, musers_to_delete: dict[int, dict[str, Any]] |
| 313 | + ) -> list[int]: |
| 314 | + """ |
| 315 | + Returns the list of speaker_ids that should be deleted |
| 316 | + """ |
| 317 | + speaker_ids = [ |
| 318 | + speaker_id |
| 319 | + for muser in musers_to_delete.values() |
| 320 | + for speaker_id in (muser.get("speaker_ids") or []) |
| 321 | + ] |
| 322 | + speakers = self.reader.get_many( |
| 323 | + [ |
| 324 | + GetManyRequestPart( |
| 325 | + "speaker", |
| 326 | + speaker_ids, |
| 327 | + [ |
| 328 | + "meeting_id", |
| 329 | + "list_of_speakers_id", |
| 330 | + "structure_level_list_of_speakers_id", |
| 331 | + "speech_state", |
| 332 | + "begin_time", |
| 333 | + "end_time", |
| 334 | + "pause_time", |
| 335 | + "point_of_order", |
| 336 | + "unpause_time", |
| 337 | + "id", |
| 338 | + ], |
| 339 | + ) |
| 340 | + ] |
| 341 | + ).get("speaker", {}) |
| 342 | + delete_speaker_ids = [ |
| 343 | + id_ |
| 344 | + for id_, speaker in speakers.items() |
| 345 | + if speaker.get("begin_time") is None |
| 346 | + ] |
| 347 | + return delete_speaker_ids |
0 commit comments