shared function for special facilities

Signed-off-by: John Seekins <[email protected]>

Author: johnseekins

ice_scrapers/__init__.py

@@ -131,6 +131,7 @@
     repair_locality,  # noqa: F401
     repair_street,  # noqa: F401
     repair_zip,  # noqa: F401
+    special_facilities,  # noqa: F401
     update_facility,  # noqa: F401
 )
 from .facilities_scraper import scrape_facilities  # noqa: F401,E402

ice_scrapers/facilities_scraper.py

@@ -7,6 +7,7 @@
     repair_locality,
     repair_street,
     repair_zip,
+    special_facilities,
     update_facility,
 )
 from schemas import facility_schema
@@ -21,19 +22,6 @@
 base_scrape_url = "https://www.ice.gov/detention-facilities"
 
 
-def _special_facilities(facility: dict) -> dict:
-    match facility["name"]:
-        case "Naval Station Guantanamo Bay (JTF Camp Six and Migrant Ops Center Main A)":
-            facility["address"]["country"] = "Cuba"
-            facility["address"]["administrative_area"] = "FPO"
-            facility["address"]["locality"] = "FPO"
-            facility["address"]["postal_code"] = "34009"
-            facility["address"]["street"] = "AVENUE C PSC 1005 BOX 55"
-        case _:
-            pass
-    return facility
-
-
 def scrape_facilities(facilities_data: dict) -> dict:
     """Scrape all ICE detention facility data from all discovered pages"""
     start_time = time.time()
@@ -50,7 +38,7 @@ def scrape_facilities(facilities_data: dict) -> dict:
         logger.debug("Found %s facilities on page %s", len(facilities), page_num + 1)
         time.sleep(1)  # Be respectful to the server
         for facility in facilities:
-            facility = _special_facilities(facility)
+            facility = special_facilities(facility)
             addr = facility["address"]
             street, cleaned = repair_street(addr["street"], addr["locality"])
             if cleaned:

ice_scrapers/spreadsheet_load.py

@@ -15,6 +15,7 @@
     repair_locality,
     repair_street,
     repair_zip,
+    special_facilities,
 )
 from typing import Tuple
 from utils import (
@@ -72,17 +73,6 @@ def _download_sheet(keep_sheet: bool = True) -> Tuple[polars.DataFrame, str]:
     return df, actual_link
 
 
-def _special_rows(row: dict) -> dict:
-    match row["name"]:
-        case "JTF CAMP SIX":
-            row["address"]["country"] = "Cuba"
-            row["address"]["administrative_area"] = "FPO"
-            row["name"] = "Naval Station Guantanamo Bay (JTF Camp Six and Migrant Ops Center Main A)"
-        case _:
-            pass
-    return row
-
-
 def load_sheet(keep_sheet: bool = True) -> dict:
     df, sheet_url = _download_sheet(keep_sheet)
     """Convert the detentionstats sheet data into something we can update our facilities with"""
@@ -110,7 +100,7 @@ def load_sheet(keep_sheet: bool = True) -> dict:
         details["address"]["postal_code"] = zcode
         details["address"]["street"] = street
         details["name"] = row["Name"]
-        details = _special_rows(details)
+        details = special_facilities(details)
         full_address = ",".join(
             [
                 details["address"]["street"],

ice_scrapers/utils.py

@@ -7,6 +7,34 @@
 )
 
 
+def special_facilities(facility: dict) -> dict:
+    """
+    Some very specific facilities have unique fixes
+    that are hard to fit into our normal repair_* pattern.
+
+    Please don't expand this function unless it's necessary
+    """
+    match facility["name"]:
+        case "Naval Station Guantanamo Bay (JTF Camp Six and Migrant Ops Center Main A)":
+            """
+            First special case? JTF Camp Six is purely a mess.
+            While we work on getting a consistent address for this facility,
+            we'll need to make the two records converge.
+            """
+            facility["address"]["country"] = "Cuba"
+            facility["address"]["administrative_area"] = "FPO"
+            facility["address"]["locality"] = "FPO"
+            facility["address"]["postal_code"] = "34009"
+            facility["address"]["street"] = "AVENUE C PSC 1005 BOX 55"
+        case "JTF CAMP SIX":
+            facility["address"]["country"] = "Cuba"
+            facility["address"]["administrative_area"] = "FPO"
+            facility["name"] = "Naval Station Guantanamo Bay (JTF Camp Six and Migrant Ops Center Main A)"
+        case _:
+            pass
+    return facility
+
+
 def repair_street(street: str, locality: str = "") -> Tuple[str, bool]:
     """Generally, we'll let the spreadsheet win arguments just to be consistent"""
     street_filters = [