start actually matching records between excel and web page

Signed-off-by: John Seekins <[email protected]>

Author: johnseekins

.gitignore

@@ -2,3 +2,4 @@ __pycache__/
 *.csv
 *.json
 *.xlsx
+*.xlsx#

enricher.py

@@ -1,20 +1,16 @@
 import copy
-import os
-import polars
 from schemas import (
     facilities_schema,
     resp_info_schema,
 )
 import time
 from urllib.parse import quote
 from utils import (
-    facility_sheet_header,
     logger,
     session,
 )
 # ExternalDataEnricher class for enrichment logic
 
-SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 # Rate limiting for API calls
 NOMINATIM_DELAY = 1.0  # 1 second between requests as per OSM policy
 WIKIPEDIA_DELAY = 0.5  # Be respectful to Wikipedia
@@ -23,80 +19,19 @@
 
 class ExternalDataEnricher(object):
     def __init__(self):
-        self.sheet_url = "https://www.ice.gov/doclib/detention/FY25_detentionStats08292025.xlsx"
-        self.filename = f"{SCRIPT_DIR}{os.sep}detentionstats.xlsx"
-        self.adp_sheet_data = self._load_sheet()
-
-    def _download_sheet(self) -> None:
-        if not os.path.isfile(self.filename) or os.path.getsize(self.filename) < 1:
-            logger.info("Downloading sheet from %s", self.sheet_url)
-            resp = session.get(self.sheet_url, timeout=120)
-            with open(self.filename, "wb") as f:
-                for chunk in resp.iter_content(chunk_size=1024):
-                    if chunk:
-                        f.write(chunk)
-
-    def _load_sheet(self) -> dict:
-        """Convert the detentionstats sheet data into something we can update our facilities with"""
-        self._download_sheet()
-        df = polars.read_excel(
-            drop_empty_rows=True,
-            has_header=False,
-            # because we're manually defining the header...
-            read_options={"skip_rows": 7, "column_names": facility_sheet_header},
-            sheet_name="Facilities FY25",
-            source=open(self.filename, "rb"),
-        )
-        results: dict = {}
-        for row in df.iter_rows(named=True):
-            full_address = f"{row['Address']} {row['City']}, {row['State']} {row['Zip']}".upper()
-            results[full_address] = row
-        return results
-
-    def _update_from_sheet(self, base: dict, row: dict) -> dict:
-        base["population"]["male"]["criminal"] = row["Male Crim"]
-        base["population"]["male"]["non_criminal"] = row["Male Non-Crim"]
-        base["population"]["female"]["criminal"] = row["Female Crim"]
-        base["population"]["female"]["non_criminal"] = row["Female Non-Crim"]
-        if "/" in row["Male/Female"]:
-            base["population"]["female"]["allowed"] = True
-            base["population"]["male"]["allowed"] = True
-        elif "Female" in row["Male/Female"]:
-            base["population"]["female"]["allowed"] = True
-        else:
-            base["population"]["male"]["allowed"] = True
-
-        base["base_type"] = row["Type Detailed"]
-        base["avg_stay_length"] = row["FY25 ALOS"]
-        base["inspection_date"] = row["Last Inspection End Date"]
-        logger.debug("Updated facility: %s", base)
-        return base
+        pass
 
     def enrich_facility_data(self, facilities_data: dict) -> dict:
         start_time = time.time()
         logger.info("Starting data enrichment with external sources...")
         enriched_data = copy.deepcopy(facilities_schema)
         total = len(facilities_data["facilities"])
+        processed = 0
 
-        for index, facility in enumerate(facilities_data["facilities"]):
+        for facility_id, facility in enumerate(facilities_data["facilities"]):
             facility_name = facility["name"]
-            logger.info("Processing facility %s/%s: %s...", index + 1, total, facility_name)
+            logger.info("Processing facility %s/%s: %s...", processed + 1, total, facility_name)
             enriched_facility = copy.deepcopy(facility)
-            addr = facility["address"]
-            full_address = (
-                f"{addr['street']} {addr['locality']}, {addr['administrative_area']} {addr['postal_code']}".upper()
-            )
-            if full_address in self.adp_sheet_data:
-                row = self.adp_sheet_data[full_address]
-                logger.debug("Found additional data in the ADP sheet for %s", facility_name)
-                enriched_facility = self._update_from_sheet(facility, row)
-            else:
-                logger.debug("Just making sure no other facilities match...")
-                for sheet_row in self.adp_sheet_data.values():
-                    if facility_name.upper() == sheet_row["Name"].upper():
-                        logger.debug("Matching facility for %s", facility_name)
-                        enriched_facility = self._update_from_sheet(facility, sheet_row)
-                        break
 
             # Wikipedia search # todo refactor to method
             try:
@@ -131,7 +66,8 @@ def enrich_facility_data(self, facilities_data: dict) -> dict:
                 enriched_facility["osm_result_url"] = ""
                 enriched_facility["osm_search_query"] = str(e)
 
-            enriched_data["facilities"].append(enriched_facility)  # type: ignore [attr-defined]
+            enriched_data["facilities"][facility_id] = enriched_facility  # type: ignore [index]
+            processed += 1
 
         logger.info("Data enrichment completed!")
         enriched_data["enrich_runtime"] = time.time() - start_time

file_utils.py

@@ -22,7 +22,7 @@ def export_to_file(
     try:
         with open(full_name, "w", newline="", encoding="utf-8") as f_out:
             if file_type == "csv":
-                flatdata = [_flatdict(f) for f in facilities_data["facilities"]]
+                flatdata = [_flatdict(f) for _, f in facilities_data["facilities"].items()]
                 fieldnames = [k for k in flatdata[0].keys() if k not in csv_filtered_keys]
 
                 writer = csv.DictWriter(f_out, fieldnames=fieldnames)
@@ -60,7 +60,7 @@ def print_summary(facilities_data: dict) -> None:
 
     # Count by field office
     field_offices: dict = {}
-    for facility in facilities_data["facilities"]:
+    for facility_id, facility in facilities_data["facilities"].items():
         office = facility.get("field_office", "Unknown")
         field_offices[office] = field_offices.get(office, 0) + 1
 
@@ -70,9 +70,13 @@ def print_summary(facilities_data: dict) -> None:
 
     # Check enrichment data if available
     enrich_data = copy.deepcopy(enrichment_print_schema)
-    enrich_data["wiki_found"] = sum(1 for f in facilities_data["facilities"] if f.get("wikipedia_page_url", None))
-    enrich_data["wikidata_found"] = sum(1 for f in facilities_data["facilities"] if f.get("wikidata_page_url", None))
-    enrich_data["osm_found"] = sum(1 for f in facilities_data["facilities"] if f.get("osm_result_url", None))
+    enrich_data["wiki_found"] = sum(
+        1 for f in facilities_data["facilities"].values() if f.get("wikipedia_page_url", None)
+    )
+    enrich_data["wikidata_found"] = sum(
+        1 for f in facilities_data["facilities"].values() if f.get("wikidata_page_url", None)
+    )
+    enrich_data["osm_found"] = sum(1 for f in facilities_data["facilities"].values() if f.get("osm_result_url", None))
 
     if any(v > 0 for v in enrich_data.values()):
         logger.info("\n=== External Data Enrichment Results ===")
@@ -96,25 +100,17 @@ def print_summary(facilities_data: dict) -> None:
         )
 
         # Debug information if available
-        if facilities_data["facilities"][0].get("wikipedia_search_query", None):
-            logger.info("\n=== Wikipedia Debug Information ===")
-            false_positives = 0
-            errors = 0
-            for facility in facilities_data["facilities"]:
-                query = facility.get("wikipedia_search_query", "")
-                if "REJECTED" in query:
-                    false_positives += 1
-                elif "ERROR" in query:
-                    errors += 1
+        logger.info("\n=== Wikipedia Debug Information ===")
+        false_positives = 0
+        errors = 0
+        for facility in facilities_data["facilities"].values():
+            query = facility.get("wikipedia_search_query", "")
+            if "REJECTED" in query:
+                false_positives += 1
+            elif "ERROR" in query:
+                errors += 1
 
             logger.info("False positives detected and rejected: %s", false_positives)
             logger.info("Search errors encountered: %s", errors)
-            logger.info("Note: Review 'wikipedia_search_query' column for detailed search information")
-
-        if facilities_data["facilities"][0].get("wikidata_search_query", None):
-            logger.warning("Note: Review 'wikidata_search_query' column for detailed search information")
-
-        if facilities_data["facilities"][0].get("osm_search_query", None):
-            logger.warning("Note: Review 'osm_search_query' column for detailed search information")
 
     logger.info("\n=== ICE Detention Facilities Scraper: Run completed ===")

schemas.py

@@ -4,7 +4,7 @@
     "scraped_date": datetime.datetime.now(datetime.UTC),
     "scrape_runtime": 0,
     "enrich_runtime": 0,
-    "facilities": [],
+    "facilities": {},
 }
 
 # default keys to "false"-y values so we can merge easier
@@ -16,6 +16,7 @@
         "postal_code": "",
         "street": "",
     },
+    "_repaired_record": False,
     "facility_url": "",
     "field_office": "",
     "image_url": "",