fix(history): ensure history key always exists in items

- Use `get` method to safely access history in hubs and items
- Prevent potential KeyError by ensuring history key is present

Author: heroheman

scp_crawler/postprocessing.py

@@ -112,7 +112,7 @@ def get_wiki_source(page_id, domain, attempts=5):
     hub_list,
 ):
     # Convert history dict to list and sort by date.
-    hub["history"] = process_history(hub["history"])
+    hub["history"] = process_history(hub.get("history"))
 
     if len(hub["history"]) > 0:
         hub["created_at"] = hub["history"][0]["date"]
@@ -161,7 +161,7 @@ def run_postproc_items():
         item["hubs"] = get_hubs(item["link"])
 
         # Convert history dict to list and sort by date.
-        item["history"] = process_history(item["history"])
+        item["history"] = process_history(item.get("history"))
 
         if len(item["history"]) > 0:
             item["created_at"] = item["history"][0]["date"]

scp_crawler/spiders/scp.py

@@ -29,7 +29,8 @@ def parse_history(self, response, item, history_page=1):
         self.logger.info(f"Reviewing Page {item['page_id']} history")
 
         page_id = item["page_id"]
-        changes = item["history"] if "history" in item else {}
+        changes = item.get("history", {})
+        item["history"] = changes  # Ensure history key always exists
 
         try:
             response_text = getattr(response, "text", "") or ""
@@ -116,11 +117,13 @@ def parse_history(self, response, item, history_page=1):
                 self.logger.exception("Could not process row.")
                 self.logger.error(row)
 
-            item["history"] = changes
-            # The "0" change is the first revision, and the last one that shows up.
-            # If we have it then we're done.
-            if "0" in changes:
-                return self.get_page_source_request(page_id, item)
+        # Update item history after processing all rows
+        item["history"] = changes
+        
+        # The "0" change is the first revision, and the last one that shows up.
+        # If we have it then we're done.
+        if "0" in changes:
+            return self.get_page_source_request(page_id, item)
 
         next_page = history_page + 1
         if next_page > MAX_HISTORY_PAGES: