DRY and docs

Signed-off-by: John Seekins <[email protected]>

Author: johnseekins

ice_scrapers/README.md

@@ -0,0 +1,25 @@
+# ICE Facility scrapers
+
+These files maintain the code to collect (and collate) ICE facility data from a number of sources.
+
+## utils.py
+
+Contains most of our collating functions and shared functions that scrapers may need.
+
+## __init__.py
+
+Contains some static objects and import declarations (so we can `from ice_scrapers import` successfully)...
+
+## spreadsheet_load.py
+
+ICE is required by law to produce regular custody data. We can pull that data from here `https://www.ice.gov/detain/detention-management`. Because this spreadsheet is more "complete" than other sources we've found, we use it as our base scrape.
+
+## facilities_scraper.py
+
+Pulls information about ICE detention facilities from `https://www.ice.gov/detention-facilities`. This can add additional (or corrected) data about facilities locations, contact information, and provides facility images.
+
+## field_offices.py
+
+Collects additional data about ICE/DHS field offices from `https://www.ice.gov/contact/field-offices`. Largely basic areas of responsibility and contact info for the field office.
+
+> The field-offices page shows information about a number of different offices. As we are largely focused on detention, ERO (Eforcement and Removal Operations) centers are the most interesting.

ice_scrapers/__init__.py

@@ -77,13 +77,44 @@
     },
 }
 
+# ICE AOR mappings
+area_of_responsibility = {
+    "ATL": "Atlanta Field Office",
+    "BAL": "Baltimore Field Office",
+    "BOS": "Boston Field Office",
+    "BUF": "Buffalo Field Office",
+    "CHI": "Chicago Field Office",
+    "DAL": "Dallas Field Office",
+    "DEN": "Denver Field Office",
+    "DET": "Detroit Field Office",
+    "ELP": "El Paso Field Office",
+    "HLG": "Harlingen Field Office",
+    "HOU": "Houston Field Office",
+    "LOS": "Los Angeles Field Office",
+    "MIA": "Miami Field Office",
+    "NEW": "Newark Field Office",
+    "NOL": "New Orleans Field Office",
+    "NYC": "New York City Field Office",
+    "PHI": "Philadelphia Field Office",
+    "PHO": "Phoenix Field Office",
+    "SEA": "Seattle Field Office",
+    "SFR": "San Francisco Field Office",
+    "SLC": "Salt Lake City Field Office",
+    "SNA": "San Antonio Field Office",
+    "SND": "San Diego Field Office",
+    "SPM": "St Paul Field Office",
+    "WAS": "Washington Field Office",
+}
+field_office_to_aor = {v: k for k, v in area_of_responsibility.items()}
+
 from .utils import (  # noqa: E402
     clean_street,  # noqa: F401
+    get_ice_scrape_pages,  # noqa: F401
     repair_zip,  # noqa: F401
     repair_locality,  # noqa: F401
     update_facility,  # noqa: F401
 )
-from .page_load import scrape_facilities  # noqa: F401,E402
+from .facilities_scraper import scrape_facilities  # noqa: F401,E402
 from .spreadsheet_load import load_sheet  # noqa: F401,E402
 from .field_offices import (  # noqa: E402
     merge_field_offices,  # noqa: F401

ice_scrapers/page_load.py renamed to ice_scrapers/facilities_scraper.py

@@ -1,10 +1,10 @@
-# scraping-related code for ice.gov detention facility pages
 from bs4 import BeautifulSoup
 import copy
 import datetime
 import re
 from ice_scrapers import (
     clean_street,
+    get_ice_scrape_pages,
     repair_zip,
     repair_locality,
     update_facility,
@@ -21,29 +21,12 @@
 base_scrape_url = "https://www.ice.gov/detention-facilities"
 
 
-def _get_scrape_pages() -> list:
-    """Discover all facility pages"""
-    resp = session.get(base_scrape_url, timeout=30)
-    resp.raise_for_status()
-    soup = BeautifulSoup(resp.content, "html.parser")
-    links = soup.findAll("a", href=re.compile(r"\?page="))
-    if not links:
-        raise Exception(f"{base_scrape_url} contains *no* links?!")
-    pages = [
-        f"{base_scrape_url}{link['href']}&exposed_form_display=1"
-        for link in links
-        if not any(k in link["aria-label"] for k in ["Next", "Last"])
-    ]
-    logger.debug("Pages discovered: %s", pages)
-    return pages
-
-
 def scrape_facilities(facilities_data: dict) -> dict:
     """Scrape all ICE detention facility data from all discovered pages"""
     start_time = time.time()
     logger.info("Starting to scrape ICE.gov detention facilities...")
     facilities_data["scraped_date"] = datetime.datetime.now(datetime.UTC)
-    urls = _get_scrape_pages()
+    urls = get_ice_scrape_pages(base_scrape_url)
 
     for page_num, url in enumerate(urls):
         logger.info("Scraping page %s/%s...", page_num + 1, len(urls))

ice_scrapers/field_offices.py

@@ -2,6 +2,11 @@
 from bs4 import BeautifulSoup
 import copy
 import datetime
+from ice_scrapers import (
+    area_of_responsibility,
+    field_office_to_aor,
+    get_ice_scrape_pages,
+)
 import re
 from schemas import (
     field_offices_schema,
@@ -15,61 +20,14 @@
 
 base_scrape_url = "https://www.ice.gov/contact/field-offices"
 
-# ICE AOR mappings
-area_of_responsibility = {
-    "ATL": "Atlanta Field Office",
-    "BAL": "Baltimore Field Office",
-    "BOS": "Boston Field Office",
-    "BUF": "Buffalo Field Office",
-    "CHI": "Chicago Field Office",
-    "DAL": "Dallas Field Office",
-    "DEN": "Denver Field Office",
-    "DET": "Detroit Field Office",
-    "ELP": "El Paso Field Office",
-    "HLG": "Harlingen Field Office",
-    "HOU": "Houston Field Office",
-    "LOS": "Los Angeles Field Office",
-    "MIA": "Miami Field Office",
-    "NEW": "Newark Field Office",
-    "NOL": "New Orleans Field Office",
-    "NYC": "New York City Field Office",
-    "PHI": "Philadelphia Field Office",
-    "PHO": "Phoenix Field Office",
-    "SEA": "Seattle Field Office",
-    "SFR": "San Francisco Field Office",
-    "SLC": "Salt Lake City Field Office",
-    "SNA": "San Antonio Field Office",
-    "SND": "San Diego Field Office",
-    "SPM": "St Paul Field Office",
-    "WAS": "Washington Field Office",
-}
-field_office_to_aor = {v: k for k, v in area_of_responsibility.items()}
-
-
-def _get_scrape_pages() -> list:
-    """Discover all facility pages"""
-    resp = session.get(base_scrape_url, timeout=30)
-    resp.raise_for_status()
-    soup = BeautifulSoup(resp.content, "html.parser")
-    links = soup.findAll("a", href=re.compile(r"\?page="))
-    if not links:
-        raise Exception(f"{base_scrape_url} contains *no* links?!")
-    pages = [
-        f"{base_scrape_url}{link['href']}&exposed_form_display=1"
-        for link in links
-        if not any(k in link["aria-label"] for k in ["Next", "Last"])
-    ]
-    logger.debug("Pages discovered: %s", pages)
-    return pages
-
 
 def scrape_field_offices() -> dict:
     """Collect data on ICE field offices"""
     start_time = time.time()
     office_data = copy.deepcopy(field_offices_schema)
     office_data["scraped_date"] = datetime.datetime.now(datetime.UTC)
     logger.info("Starting to scrape ICE.gov field offices...")
-    urls = _get_scrape_pages()
+    urls = get_ice_scrape_pages(base_scrape_url)
     for page_num, url in enumerate(urls):
         logger.info("Scraping page %s/%s...", page_num + 1, len(urls))
         offices = _scrape_page(url)
@@ -138,11 +96,6 @@ def _scrape_page(page_url: str) -> list:
             )
             break
 
-    # if not office_elements:
-    #     # Fallback: look for any element containing office-like text patterns
-    #     logger.warning("  Using fallback: searching for office patterns in text")
-    #     office_elements = _find_office_patterns(content_container)
-
     # Extract data from each office element
     for element in office_elements:
         office_data = _extract_single_office(element, page_url)
@@ -208,6 +161,7 @@ def _extract_single_office(element: BeautifulSoup, page_url: str) -> dict:
 
 
 def merge_field_offices(facilities_data: dict, field_offices: dict) -> dict:
+    """Actually insert field office data into our facilities_data object"""
     final_facilities = copy.deepcopy(facilities_data["facilities"])
     for facility_id, facility in facilities_data["facilities"].items():
         office = field_offices["field_offices"].get(facility["field_office"]["field_office"], None)

ice_scrapers/utils.py

@@ -1,4 +1,10 @@
+from bs4 import BeautifulSoup
+import re
 from typing import Tuple
+from utils import (
+    logger,
+    session,
+)
 
 
 def clean_street(street: str, locality: str = "") -> Tuple[str, bool]:
@@ -179,3 +185,23 @@ def update_facility(old: dict, new: dict) -> dict:
         if not old.get(k, None):
             old[k] = v
     return old
+
+
+def get_ice_scrape_pages(url: str) -> list:
+    """
+    Discover all facility pages
+    This _may_ be generic to Drupal's pagination code...
+    """
+    resp = session.get(url, timeout=30)
+    resp.raise_for_status()
+    soup = BeautifulSoup(resp.content, "html.parser")
+    links = soup.findAll("a", href=re.compile(r"\?page="))
+    if not links:
+        raise Exception(f"{url} contains *no* links?!")
+    pages = [
+        f"{url}{link['href']}&exposed_form_display=1"
+        for link in links
+        if not any(k in link["aria-label"] for k in ["Next", "Last"])
+    ]
+    logger.debug("Pages discovered: %s", pages)
+    return pages