track runtimes

Signed-off-by: John Seekins <[email protected]>

Author: johnseekins

enricher.py

@@ -4,7 +4,7 @@
 import time
 from urllib.parse import quote
 import urllib3
-from utils import logger
+from utils import logger, facilities_schema
 
 # ExternalDataEnricher class for enrichment logic
 
@@ -27,14 +27,14 @@ def __init__(self):
         self.session.headers.update({"User-Agent": "ICE-Facilities-Research/1.0 (Educational Research Purpose)"})
 
     def enrich_facility_data(self, facilities_data: dict) -> dict:
+        start_time = time.time()
         logger.info("Starting data enrichment with external sources...")
-        enriched_data = copy.deepcopy(facilities_data)
-        enriched_data["facilities"] = []
+        enriched_data = copy.deepcopy(facilities_schema)
         total = len(facilities_data["facilities"])
 
         for i, facility in enumerate(facilities_data["facilities"]):
             logger.info("Processing facility %s/%s: %s...", i + 1, total, facility["name"])
-            enriched_facility = facility.copy()
+            enriched_facility = copy.deepcopy(facility)
             base_enrichment = {
                 "wikipedia_page_url": "",
                 "wikipedia_search_query": "",
@@ -81,13 +81,11 @@ def enrich_facility_data(self, facilities_data: dict) -> dict:
                 enriched_facility["osm_result_url"] = ""
                 enriched_facility["osm_search_query"] = str(e)
 
-            enriched_data["facilities"].append(enriched_facility)
-
-            # do we need the "progress bar" if we show the count in the beginning message?
-            # if (i + 1) % 10 == 0:
-            #     logger.info(" Progress: %s/%s facilities processed", i + 1, total)
+            enriched_data["facilities"].append(enriched_facility)  # type: ignore [attr-defined]
 
         logger.info("Data enrichment completed!")
+        enriched_data["enrich_runtime"] = time.time() - start_time
+        logger.info(" Completed in %s seconds", enriched_data["enrich_runtime"])
         return enriched_data
 
     def _search_wikipedia(self, facility_name: str) -> dict:

scraper.py

@@ -33,6 +33,7 @@ def __init__(self):
 
     def scrape_facilities(self):
         """Scrape all ICE detention facility data from all 6 pages"""
+        start_time = time.time()
         logger.info("Starting to scrape ICE detention facilities...")
 
         self.facilities_data["scraped_date"] = datetime.datetime.utcnow()
@@ -51,7 +52,9 @@ def scrape_facilities(self):
             time.sleep(1)  # Be respectful to the server
 
         # self.facilities_data = all_facilities
+        self.facilities_data["scrape_runtime"] = time.time() - start_time
         logger.info("Total facilities scraped: %s", len(self.facilities_data["facilities"]))
+        logger.info(" Completed in %s seconds", self.facilities_data["scrape_runtime"])
         return self.facilities_data
 
     def _scrape_updated(self, url: str):

utils.py

@@ -9,6 +9,8 @@
 facilities_schema = {
     "scraped_date": datetime.datetime.utcnow(),
     "page_updated_date": datetime.datetime.utcnow(),
+    "scrape_runtime": 0,
+    "enrich_runtime": 0,
     "facilities": [],
 }