Merge pull request #48 from johnseekins/small-improvements

Add custom facilities and some small schema improvements

Author: HongPong

README.md

@@ -18,9 +18,6 @@ scrape of the data from ICE.gov. This is stored in `default_data.py` and include
 
 > Note ICE has been renaming known "detention center" sites to "processing center", and so on.
 
-The initial scrape data also keeps a `base64` ecoded string containing the original HTML that was scraped from ice.gov about the
-facility. Keeping this initial data allows us to verify the resulting extracted data if we need to.
-
 It also shows the ICE "field office" managing each detention facility.
 
 On the OpenStreetMap (OSM) CSV results, if the URL includes a "way" then it has probably identified the correctly tagged

default_data.py

@@ -36,3 +36,7 @@ and contact info for the field office.
 > The field-offices page shows information about a number of different offices. As we
 > are largely focused on detention, ERO (Enforcement and Removal Operations) centers
 > are the most interesting.
+
+## custom_facilities.py
+
+Some facilities we may discover manually. Or they may be "pending" classification, but we discover them early on. These facilities are defined here.

ice_scrapers/README.md

@@ -82,6 +82,14 @@
         "expanded_name": "Contract Detention Facility",
         "description": "Name derived from listing at https://www.vera.org/ice-detention-trends",
     },
+    "Staging": {
+        "description": "Some facilities in the ICE spreadsheet are marked 'Staging'. Hard to determine why.",
+        "expanded_name": "Staging",
+    },
+    "Pending": {
+        "expanded_name": "Pending Classication and Inclusion",
+        "description": "Facilities discovered through other means that may become ICE/DHS facilities",
+    },
 }
 
 # ICE AOR mappings
@@ -115,10 +123,10 @@
 field_office_to_aor = {v: k for k, v in area_of_responsibility.items()}
 
 from .utils import (  # noqa: E402
-    clean_street,  # noqa: F401
     get_ice_scrape_pages,  # noqa: F401
-    repair_zip,  # noqa: F401
     repair_locality,  # noqa: F401
+    repair_street,  # noqa: F401
+    repair_zip,  # noqa: F401
     update_facility,  # noqa: F401
 )
 from .facilities_scraper import scrape_facilities  # noqa: F401,E402
@@ -127,3 +135,5 @@
     merge_field_offices,  # noqa: F401
     scrape_field_offices,  # noqa: F401
 )
+from .custom_facilities import insert_additional_facilities  # noqa: F401,E402
+from .general import facilities_scrape_wrapper  # noqa: F401,E402

ice_scrapers/__init__.py

@@ -0,0 +1,85 @@
+import copy
+
+"""
+Handle manually discovered/managed facilities
+"""
+custom_facilities = {
+    "North Highway 83,McCook,NE,69001": {
+        "_repaired_record": False,
+        "address": {
+            "administrative_area": "NE",
+            "country": "United States",
+            "locality": "McCook",
+            "postal_code": "69001",
+            "street": "North Highway 83",
+        },
+        "address_str": "North Highway 83,McCook,NE,69001",
+        "facility_type": {
+            "expanded_name": "Pending Classication and Inclusion",
+            "description": "Facilities discovered through other means that may become ICE/DHS facilities",
+            "id": "Pending",
+        },
+        "field_office": {
+            "address": {
+                "administrative_area": "MN",
+                "country": "United States",
+                "locality": "Fort Snelling",
+                "postal_code": "55111",
+                "street": "1 Federal Drive Suite 1601",
+            },
+            "address_str": "1 Federal Drive Suite 1601 Fort Snelling, MN 55111",
+            "aor": "Iowa, Minnesota, Nebraska, North Dakota, South Dakota",
+            "email": "[email protected]",
+            "field_office": "St Paul Field Office",
+            "id": "SPM",
+            "name": "Fort Snelling - ERO",
+            "phone": "(612) 843-8600",
+            "source_urls": ["https://www.ice.gov/contact/field-offices?page=2&exposed_form_display=1"],
+        },
+        "image_url": "https://corrections.nebraska.gov/sites/default/files/2024-08/wec_thumbnail.jpg",
+        "inspection": {
+            "last_date": None,
+            "last_rating": "",
+            "last_type": "",
+        },
+        "name": "Work Ethic Camp",
+        "osm": {
+            "latitude": 40.22851,
+            "longitude": -100.548001,
+            "search_query": "",
+            "url": "https://www.openstreetmap.org/way/456014773#map=19/40.228251/-100.648001",
+        },
+        "page_updated_date": None,
+        "phone": "308-345-8405",
+        "population": {
+            "female": {"allowed": False, "criminal": 0, "non_criminal": 0},
+            "avg_stay_length": 0,
+            "ice_threat_level": {
+                "level_1": 0,
+                "level_2": 0,
+                "level_3": 0,
+                "none": 0,
+            },
+            "male": {"allowed": False, "criminal": 0, "non_criminal": 0},
+            "housing": {"guaranteed_min": 0, "mandatory": 0},
+            "security_threat": {
+                "high": 0,
+                "low": 0,
+                "medium_high": 0,
+                "medium_low": 0,
+            },
+            "total": 0,
+        },
+        "source_urls": [
+            "https://corrections.nebraska.gov/facilities/work-ethic-camp",
+        ],
+        "wikidata": {"page_url": "", "search_query": ""},
+        "wikipedia": {"page_url": "", "search_query": ""},
+    },
+}
+
+
+def insert_additional_facilities(facilities_data: dict) -> dict:
+    for facility_id, facility in custom_facilities.items():
+        facilities_data["facilities"][facility_id] = copy.deepcopy(facility)
+    return facilities_data

ice_scrapers/custom_facilities.py

@@ -3,10 +3,10 @@
 import datetime
 import re
 from ice_scrapers import (
-    clean_street,
     get_ice_scrape_pages,
-    repair_zip,
     repair_locality,
+    repair_street,
+    repair_zip,
     update_facility,
 )
 from schemas import facility_schema
@@ -38,7 +38,7 @@ def scrape_facilities(facilities_data: dict) -> dict:
         time.sleep(1)  # Be respectful to the server
         for facility in facilities:
             addr = facility["address"]
-            street, cleaned = clean_street(addr["street"], addr["locality"])
+            street, cleaned = repair_street(addr["street"], addr["locality"])
             if cleaned:
                 addr["street"] = street
                 facility["_repaired_record"] = True

ice_scrapers/facilities_scraper.py

@@ -0,0 +1,21 @@
+import copy
+from ice_scrapers import (
+    insert_additional_facilities,
+    load_sheet,
+    merge_field_offices,
+    scrape_facilities,
+    scrape_field_offices,
+)
+from schemas import facilities_schema
+
+
+def facilities_scrape_wrapper() -> dict:
+    facilities_data = copy.deepcopy(facilities_schema)
+    facilities = load_sheet()
+    facilities_data["facilities"] = copy.deepcopy(facilities)
+    facilities_data = scrape_facilities(facilities_data)
+    field_offices = scrape_field_offices()
+    facilities_data = merge_field_offices(facilities_data, field_offices)
+    facilities_data = insert_additional_facilities(facilities_data)
+
+    return facilities_data

ice_scrapers/general.py

@@ -9,12 +9,12 @@
     field_office_schema,
 )
 from ice_scrapers import (
-    clean_street,
     facility_sheet_header,
     ice_facility_types,
     ice_inspection_types,
-    repair_zip,
     repair_locality,
+    repair_street,
+    repair_zip,
 )
 from typing import Tuple
 from utils import (
@@ -84,7 +84,7 @@ def load_sheet(keep_sheet: bool = True) -> dict:
         zcode, cleaned = repair_zip(row["Zip"], row["City"])
         if cleaned:
             details["_repaired_record"] = True
-        street, cleaned = clean_street(row["Address"], row["City"])
+        street, cleaned = repair_street(row["Address"], row["City"])
         if cleaned:
             details["_repaired_record"] = True
         match = phone_re.search(row["Address"])
@@ -101,7 +101,9 @@ def load_sheet(keep_sheet: bool = True) -> dict:
         details["address"]["street"] = street
         details["name"] = row["Name"]
 
-        # population statistics
+        """
+        population statistics
+        """
         details["population"]["male"]["criminal"] = row["Male Crim"]
         details["population"]["male"]["non_criminal"] = row["Male Non-Crim"]
         details["population"]["female"]["criminal"] = row["Female Crim"]
@@ -123,27 +125,22 @@ def load_sheet(keep_sheet: bool = True) -> dict:
             "level_3": row["ICE Threat Level 3"],
             "none": row["No ICE Threat Level"],
         }
-        """
-        # extracted from https://www.ice.gov/doclib/detention/FY25_detentionStats09112025.xlsx 2025-09-22
-        Upon admission and periodically thereafter, detainees are categorized into a security level based on a variety of public safety factors, and are housed accordingly.  Factors include prior convictions, threat risk, disciplinary record, special vulnerabilities, and special management concerns.  Detainees are categorized into one of four classes of security risk: A/low, B/medium low, C/medium high, and D/high.
-        """
+        # Levels extracted from https://www.ice.gov/doclib/detention/FY25_detentionStats09112025.xlsx 2025-09-22
         details["population"]["security_threat"]["low"] = row["Level A"]
         details["population"]["security_threat"]["medium_low"] = row["Level B"]
         details["population"]["security_threat"]["medium_high"] = row["Level C"]
         details["population"]["security_threat"]["high"] = row["Level D"]
+        details["population"]["housing"]["mandatory"] = row["Mandatory"]
+        details["population"]["housing"]["guaranteed_min"] = row["Guaranteed Minimum"]
+        details["population"]["avg_stay_length"] = row["FY25 ALOS"]
 
         details["facility_type"] = {
             "id": row["Type Detailed"],
-            "housing": {
-                "mandatory": row["Mandatory"],
-                "guaranteed_min": row["Guaranteed Minimum"],
-            },
         }
         ft_details = ice_facility_types.get(row["Type Detailed"], {})
         if ft_details:
             details["facility_type"]["description"] = ft_details["description"]
             details["facility_type"]["expanded_name"] = ft_details["expanded_name"]
-        details["avg_stay_length"] = row["FY25 ALOS"]
         details["inspection"] = {
             # fall back to type code
             "last_type": ice_inspection_types.get(row["Last Inspection Type"], row["Last Inspection Type"]),

ice_scrapers/spreadsheet_load.py

@@ -7,7 +7,7 @@
 )
 
 
-def clean_street(street: str, locality: str = "") -> Tuple[str, bool]:
+def repair_street(street: str, locality: str = "") -> Tuple[str, bool]:
     """Generally, we'll let the spreadsheet win arguments just to be consistent"""
     street_filters = [
         # address mismatch between site and spreadsheet

ice_scrapers/utils.py

@@ -23,17 +23,9 @@
 import logging
 from file_utils import export_to_file, print_summary
 import default_data
-from ice_scrapers import (
-    load_sheet,
-    merge_field_offices,
-    scrape_facilities,
-    scrape_field_offices,
-)
+from ice_scrapers import facilities_scrape_wrapper
 from enrichers import enrich_facility_data
-from schemas import (
-    facilities_schema,
-    supported_output_types,
-)
+from schemas import supported_output_types
 from utils import logger
 # CLI, argument parsing, script orchestration
 
@@ -116,18 +108,12 @@ def main() -> None:
         logger.warning(
             "Warning: --debug-wikipedia, --debug-wikidata and --debug-osm are currently not implemented as command line options."
         )
-    facilities_data = copy.deepcopy(facilities_schema)
-
     if args.scrape and args.load_existing:
         logger.error("Can't scrape and load existing data!")
         exit(1)
 
     if args.scrape:
-        facilities = load_sheet()
-        facilities_data["facilities"] = copy.deepcopy(facilities)
-        facilities_data = scrape_facilities(facilities_data)
-        field_offices = scrape_field_offices()
-        facilities_data = merge_field_offices(facilities_data, field_offices)
+        facilities_data = facilities_scrape_wrapper()
     elif args.load_existing:
         facilities_data = copy.deepcopy(default_data.facilities_data)
         logger.info(