Merge pull request #28 from johnseekins/dynamic-scrape-page-count

find pages to scrape rather than hard-coding

Author: HongPong

README.md

@@ -109,7 +109,6 @@ You can change this in scraper.py and enricher.py.
 seems wrong.
 * The remote query rate limiting is (I think) done in series but would go faster with parallel/async processing.
 * This is only targeted at English (EN) Wikipedia currently, but  multi-lingual page checks would help a wider audience.
-* `uv run python main.py --load-existing` kicks errors. It doesn't know what to do.
 
 ## Contributing & Code Standards
 

enricher.py

@@ -28,7 +28,7 @@ def enrich_facility_data(self, facilities_data: dict) -> dict:
         total = len(facilities_data["facilities"])
         processed = 0
 
-        for facility_id, facility in enumerate(facilities_data["facilities"]):
+        for facility_id, facility in facilities_data["facilities"].items():
             facility_name = facility["name"]
             logger.info("Processing facility %s/%s: %s...", processed + 1, total, facility_name)
             enriched_facility = copy.deepcopy(facility)

schemas.py

@@ -43,6 +43,7 @@
         },
     },
     "facility_type": "",
+    "facility_type_detail": {},
     "inspection_date": None,
     "avg_stay_length": 0,
 }
@@ -70,9 +71,13 @@
         "description": "A facility primarily contracted with the USMS for housing of USMS detainees, in which ICE contracts with the USMS for bed space.",
     },
     "USMSIGA": {
-        "expanded_name": "USMS Intergovernmental Agreement",
+        "expanded_name": "United States Marshal Service Intergovernmental Agreement",
         "description": "A USMS Intergovernmental Agreement in which ICE agrees to utilize an already established US Marshal Service contract.",
     },
+    "USMS CDF": {
+        "expanded_name": "United States Marshal Service Central Detention Facility",
+        "description": "Name guessed at from searching",
+    },
 }
 
 # enrichment response object
@@ -90,3 +95,5 @@
     "wikidata_found": 0,
     "osm_found": 0,
 }
+
+default_field_office = "(Possibly) Not managed by DHS field office"

scraper.py

@@ -7,8 +7,10 @@
 import polars
 import re
 from schemas import (
+    default_field_office,
     facilities_schema,
     facility_schema,
+    ice_facility_types,
 )
 import time
 from typing import Tuple
@@ -34,6 +36,7 @@ def __init__(self):
 
     def _download_sheet(self) -> None:
         resp = session.get(self.base_xlsx_url, timeout=120)
+        resp.raise_for_status()
         soup = BeautifulSoup(resp.content, "html.parser")
         links = soup.findAll("a", href=re.compile("^https://www.ice.gov/doclib.*xlsx"))
         if not links:
@@ -260,9 +263,11 @@ def _load_sheet(self) -> dict:
                     details["population"]["male"]["allowed"] = True
 
             details["facility_type"] = row["Type Detailed"]
+            details["facility_type_detail"] = ice_facility_types.get(row["Type Detailed"], {})
             details["avg_stay_length"] = row["FY25 ALOS"]
             details["inspection_date"] = row["Last Inspection End Date"]
             details["source_urls"].append(self.sheet_url)
+            details["field_office"] = default_field_office
             results[full_address] = details
         return results
 
@@ -274,18 +279,32 @@ def _update_facility(self, old: dict, new: dict) -> dict:
                 old[k] = v
         return old
 
+    def _get_scrape_pages(self) -> list:
+        """Discover all facility pages"""
+        resp = session.get(self.base_scrape_url, timeout=30)
+        resp.raise_for_status()
+        soup = BeautifulSoup(resp.content, "html.parser")
+        links = soup.findAll("a", href=re.compile(r"\?page="))
+        if not links:
+            raise Exception(f"{self.base_scrape_url} contains *no* links?!")
+        pages = [
+            f"{self.base_scrape_url}{link['href']}&exposed_form_display=1"
+            for link in links
+            if not any(k in link["aria-label"] for k in ["Next", "Last"])
+        ]
+        logger.debug("Pages discovered: %s", pages)
+        return pages
+
     def scrape_facilities(self):
         """Scrape all ICE detention facility data from all 6 pages"""
         start_time = time.time()
         logger.info("Starting to scrape ICE.gov detention facilities...")
         self.facilities_data["scraped_date"] = datetime.datetime.now(datetime.UTC)
         self.facilities_data["facilities"] = self._load_sheet()
-
-        # URLs for all pages
-        urls = [f"{self.base_scrape_url}?exposed_form_display=1&page={i}" for i in range(6)]
+        urls = self._get_scrape_pages()
 
         for page_num, url in enumerate(urls):
-            logger.info("Scraping page %s/6...", page_num + 1)
+            logger.info("Scraping page %s/%s...", page_num + 1, len(urls))
             try:
                 facilities = self._scrape_page(url)
             except Exception as e:
@@ -308,6 +327,8 @@ def scrape_facilities(self):
                     self.facilities_data["facilities"][full_address] = self._update_facility(
                         self.facilities_data["facilities"][full_address], facility
                     )
+                    if facility["field_office"]:
+                        self.facilities_data["facilities"][full_address]["field_office"] = facility["field_office"]
                     # update to the frequently nicer address from ice.gov
                     self.facilities_data["facilities"][full_address]["address"] = addr
                     # add scraped urls