[SDBELGA-1059] - Support deletes during init for items marked for deletion (#3178)

* Support tombstone deletes during init

* Fix tombstone deletes in init data

Author: BrianMwangi21

apps/prepopulate/app_initialize.py

@@ -353,9 +353,29 @@ def import_file(self, entity_name, path, file_name, index_params, do_patch=False
                 service = superdesk.get_resource_service(entity_name)
                 json_data = json.loads(app_prepopulation.read())
                 data = [fillEnvironmentVariables(item) for item in json_data]
-                data = [app.data.mongo._mongotize(item, service.datasource) for item in data if item]
-                existing_data = []
+                deleted_ids = [item["_id"] for item in data if item and item.get("_deleted") and item.get("_id")]
+                data = [item for item in data if item and not item.get("_deleted")]
+                data = [app.data.mongo._mongotize(item, service.datasource) for item in data]
+
+                if deleted_ids:
+                    delete_lookup = {"_id": {"$in": deleted_ids}}
+                    delete_lookup = app.data.mongo._mongotize(delete_lookup, service.datasource)
+                    deleted_result = service.delete(delete_lookup)
+                    deleted_id_set = set()
+                    for item in deleted_result or []:
+                        if isinstance(item, dict) and "_id" in item:
+                            deleted_id_set.add(str(item["_id"]))
+                        else:
+                            deleted_id_set.add(str(item))
+
+                    for deleted_id in deleted_ids:
+                        if str(deleted_id) in deleted_id_set:
+                            logger.info(" - deleted tombstone record: %s", deleted_id)
+                        else:
+                            logger.info(" - record marked for deletion already missing: %s", deleted_id)
+
                 existing = service.get_from_mongo(None, {})
+                existing_data = []
                 update_data = True
                 if not do_patch and existing.count() > 0:
                     logger.info(" - data already exists none will be loaded")

tests/prepopulate/app_initialization_test.py

@@ -101,6 +101,25 @@ def test_init_keeps_user_modifications(self):
         self.assertEqual("Urgency", urgency["display_name"])
         self.assertEqual("init", urgency["_etag"])
 
+    def test_init_deletes_tombstone_records(self):
+        self._run(["vocabularies"])
+        urgency = self.app.data.find_one("vocabularies", req=None, _id="urgency")
+        self.assertIsNotNone(urgency)
+
+        init_dir = tempfile.mkdtemp("init", "test")
+        self.app.config.update({"INIT_DATA_PATH": init_dir})
+        self.addCleanup(self.app.config.pop, "INIT_DATA_PATH", None)
+        self.addCleanup(shutil.rmtree, init_dir)
+
+        with open(os.path.join(init_dir, "vocabularies.json"), "w") as f:
+            f.write(json.dumps([{"_id": "urgency", "_deleted": True}]))
+            f.flush()
+
+        self._run(["vocabularies"])
+
+        urgency = self.app.data.find_one("vocabularies", req=None, _id="urgency")
+        self.assertIsNone(urgency)
+
     def test_init_can_combine_files_from_folders(self):
         init_dir = tempfile.mkdtemp("init", "test")
         self.app.config.update({"INIT_DATA_PATH": init_dir})