document enricment a bit and improve layout somewhat

Signed-off-by: John Seekins <[email protected]>

Author: johnseekins

enrichers/README.md

@@ -0,0 +1,30 @@
+# Facility enrichment scrapers
+
+These functions let us collect data about facilities from additional sources.
+
+## Enrichment class
+
+The base class we can build enrichment tools from. Largely ensures some consistent in functionality between enrichment tools.
+
+### Available functions
+
+Sub-classing `Enrichment` provides the following functions/objects:
+
+* `self.resp_info`
+    * Pre-created response object following our expected schema
+* `self._wait_time`
+    * simple rate-limiting through `time.sleep()` calls, `wait_time` tells us how long we should sleep between calls to an individual API/site.
+    * Defaults to `1` (seconds)
+* `self._req(...)`
+    * Wrapper function around a call to `requests.get` (using a properly configured `session` object)
+    * handles redirects
+    * supports most normal requests function calls (`params`, `timeout`, `stream`, custom headers)
+    * raises for non-2xx/3xx status
+    * returns the entire `requests.Response` object for manipulation
+* `_minimal_clean_facility_name(str)`
+    * standardizes facility name for searching
+* `_clean_facility_name(str)`
+    * standardizes facility name for searching
+    * more aggressive formatting than `_minimal_...` above
+
+> All child functions should implement the `search()` function, which should return a dictionary using the `enrich_resp_schema` schema.

enrichers/__init__.py

@@ -1,3 +1,9 @@
+"""
+Import order here is a touch weird, but we need it so
+types exist before attempting to import functions that
+may call them
+"""
+
 import copy
 import requests
 from schemas import enrich_resp_schema
@@ -7,39 +13,28 @@
     session,
 )
 
-OSM_DELAY = 1.0  # 1 second between requests as per OSM policy
-WIKIDATA_DELAY = 0.5  # Be respectful to Wikidata
-WIKIPEDIA_DELAY = 0.5  # Be respectful to Wikipedia
-
-# default to Washington, D.C.?
-default_coords: dict = {
-    "latitude": 38.89511000,
-    "longitude": -77.03637000,
-}
-
 
 class Enrichment(object):
-    required_keys = [
+    _required_keys = [
         "facility_name",
     ]
+    # in seconds
+    _wait_time: float = 1
 
     def __init__(self, **kwargs):
         self.resp_info = copy.deepcopy(enrich_resp_schema)
-        for k in self.required_keys:
+        for k in self._required_keys:
             if k not in kwargs.keys():
                 raise KeyError("Missing required key %s in %s", k, kwargs)
         self.search_args = copy.deepcopy(kwargs)
-        self.wait_time = int(kwargs.get("wait_time", 1))
 
     def search(self) -> dict:
         """Child objects should implement this"""
         return {}
 
-    def _req(
-        self, url: str, params: dict = {}, timeout: int = 10, stream: bool = False, headers: dict = default_headers
-    ) -> requests.Response:
+    def _req(self, url: str, **kwargs) -> requests.Response:
         """requests response wrapper to ensure we honor waits"""
-
+        headers = kwargs.get("headers", {})
         # ensure we get all headers configured correctly
         # but manually applied headers win the argument
         for k, v in default_headers.items():
@@ -48,10 +43,15 @@ def _req(
             headers[k] = v
 
         response = session.get(
-            url, allow_redirects=True, timeout=timeout, params=params, stream=stream, headers=headers
+            url,
+            allow_redirects=True,
+            timeout=kwargs.get("timeout", 10),
+            params=kwargs.get("params", {}),
+            stream=kwargs.get("stream", False),
+            headers=headers,
         )
         response.raise_for_status()
-        time.sleep(self.wait_time)
+        time.sleep(self._wait_time)
         return response
 
     def _minimal_clean_facility_name(self, name: str) -> str:

enrichers/general.py

@@ -1,13 +1,9 @@
 from concurrent.futures import ProcessPoolExecutor
 import copy
 from enrichers import (
-    default_coords,
-    OSM_DELAY,
     openstreetmap,
     wikidata,
-    WIKIDATA_DELAY,
     wikipedia,
-    WIKIPEDIA_DELAY,
 )
 from schemas import (
     facilities_schema,
@@ -43,17 +39,16 @@ def _enrich_facility(facility_data: tuple) -> tuple:
     logger.info("Enriching facility %s...", facility_name)
     enriched_facility = copy.deepcopy(facility)
 
-    wiki_res = wikipedia.Wikipedia(facility_name=facility_name, wait_time=WIKIPEDIA_DELAY).search()
-    wd_res = wikidata.Wikidata(facility_name=facility_name, wait_time=WIKIDATA_DELAY).search()
-    osm_res = openstreetmap.OpenStreetMap(
-        facility_name=facility_name, address=facility.get("address", {}), wait_time=OSM_DELAY
-    ).search()
+    wiki_res = wikipedia.Wikipedia(facility_name=facility_name).search()
+    wd_res = wikidata.Wikidata(facility_name=facility_name).search()
+    osm = openstreetmap.OpenStreetMap(facility_name=facility_name, address=facility.get("address", {}))
+    osm_res = osm.search()
     enriched_facility["wikipedia"]["page_url"] = wiki_res.get("url", "")
     enriched_facility["wikipedia"]["search_query"] = wiki_res.get("search_query_steps", "")
     enriched_facility["wikidata"]["page_url"] = wd_res.get("url", "")
     enriched_facility["wikidata"]["search_query"] = wd_res.get("search_query_steps", "")
-    enriched_facility["osm"]["latitude"] = osm_res.get("details", {}).get("latitude", default_coords["latitude"])
-    enriched_facility["osm"]["longitude"] = osm_res.get("details", {}).get("longitude", default_coords["longitude"])
+    enriched_facility["osm"]["latitude"] = osm_res.get("details", {}).get("latitude", osm.default_coords["latitude"])
+    enriched_facility["osm"]["longitude"] = osm_res.get("details", {}).get("longitude", osm.default_coords["longitude"])
     enriched_facility["osm"]["url"] = osm_res.get("url", "")
     enriched_facility["osm"]["search_query"] = osm_res.get("search_query_steps", "")
 

enrichers/openstreetmap.py

@@ -1,11 +1,14 @@
-from enrichers import (
-    default_coords,
-    Enrichment,
-)
+from enrichers import Enrichment
 from utils import logger
 
 
 class OpenStreetMap(Enrichment):
+    # default to Washington, D.C.?
+    default_coords: dict = {
+        "latitude": 38.89511000,
+        "longitude": -77.03637000,
+    }
+
     def search(self) -> dict:
         facility_name = self.search_args["facility_name"]
         address = self.search_args.get("address", {})
@@ -74,8 +77,8 @@ def search(self) -> dict:
         match_terms = ["prison", "detention", "correctional", "jail"]
         for result in data:
             osm_type = result.get("type", "").lower()
-            lat = result.get("lat", default_coords["latitude"])
-            lon = result.get("lon", default_coords["longitude"])
+            lat = result.get("lat", self.default_coords["latitude"])
+            lon = result.get("lon", self.default_coords["longitude"])
             display_name = result.get("display_name", "").lower()
             if any(term in osm_type for term in match_terms) or any(term in display_name for term in match_terms):
                 # todo courthouse could be added, or other tags such as "prison:for=migrant" as a clear positive search result.
@@ -92,8 +95,8 @@ def search(self) -> dict:
         first_result = data[0]
         logger.debug("Address searches didn't directly find anything, just using the first result: %s", first_result)
         title = first_result.get("display_name", "")
-        lat = first_result.get("lat", default_coords["latitude"])
-        lon = first_result.get("lon", default_coords["longitude"])
+        lat = first_result.get("lat", self.default_coords["latitude"])
+        lon = first_result.get("lon", self.default_coords["longitude"])
         self.resp_info["search_query_steps"].append(f"{lat}&{lon}")  # type: ignore [attr-defined]
         if lat and lon:
             self.resp_info["url"] = f"https://www.openstreetmap.org/?mlat={lat}&mlon={lon}&zoom=15"

enrichers/wikidata.py

@@ -3,6 +3,9 @@
 
 
 class Wikidata(Enrichment):
+    # wikidata can handle slightly more aggressive requests
+    _wait_time = 0.5
+
     def search(self) -> dict:
         facility_name = self.search_args["facility_name"]
         # Fetches 3 results based on _clean_facility_name (not exact name). todo: needs adjustment.

enrichers/wikipedia.py

@@ -4,6 +4,7 @@
 
 
 class Wikipedia(Enrichment):
+    _wait_time = 0.5
     static_search: str = "https://en.wikipedia.org/wiki/"
     api_search: str = "https://en.wikipedia.org/w/api.php"
     facility_terms: list = [