move all functions into folder

Signed-off-by: John Seekins <[email protected]>

Author: johnseekins

enrichers/__init__.py

@@ -7,6 +7,16 @@
     session,
 )
 
+OSM_DELAY = 1.0  # 1 second between requests as per OSM policy
+WIKIDATA_DELAY = 0.5  # Be respectful to Wikidata
+WIKIPEDIA_DELAY = 0.5  # Be respectful to Wikipedia
+
+# default to Washington, D.C.?
+default_coords: dict = {
+    "latitude": 38.89511000,
+    "longitude": -77.03637000,
+}
+
 
 class Enrichment(object):
     required_keys = [
@@ -109,3 +119,6 @@ def _clean_facility_name(self, name: str) -> str:
                 cleaned = cleaned[: -len(suffix)].strip()
                 break
         return cleaned
+
+
+from .general import enrich_facility_data  # noqa: F401,E402

enricher.py renamed to enrichers/general.py

@@ -1,16 +1,16 @@
 from concurrent.futures import ProcessPoolExecutor
 import copy
 from enrichers import (
+    default_coords,
+    OSM_DELAY,
     openstreetmap,
     wikidata,
+    WIKIDATA_DELAY,
     wikipedia,
+    WIKIPEDIA_DELAY,
 )
 from schemas import (
-    default_coords,
     facilities_schema,
-    OSM_DELAY,
-    WIKIDATA_DELAY,
-    WIKIPEDIA_DELAY,
 )
 import time
 from utils import logger
@@ -25,7 +25,7 @@ def enrich_facility_data(facilities_data: dict, workers: int = 3) -> dict:
     processed = 0
 
     with ProcessPoolExecutor(max_workers=workers) as pool:
-        for res in pool.map(enrich_facility, facilities_data["facilities"].items()):
+        for res in pool.map(_enrich_facility, facilities_data["facilities"].items()):
             enriched_data["facilities"][res[0]] = res[1]  # type: ignore [index]
             processed += 1
             logger.info("  -> Finished %s, %s/%s completed", res[1]["name"], processed, total)
@@ -36,7 +36,7 @@ def enrich_facility_data(facilities_data: dict, workers: int = 3) -> dict:
     return enriched_data
 
 
-def enrich_facility(facility_data: tuple) -> tuple:
+def _enrich_facility(facility_data: tuple) -> tuple:
     """enrich a single facility"""
     facility_id, facility = facility_data
     facility_name = facility["name"]

enrichers/openstreetmap.py

@@ -1,5 +1,7 @@
-from schemas import default_coords
-from enrichers import Enrichment
+from enrichers import (
+    default_coords,
+    Enrichment,
+)
 from utils import logger
 
 

main.py

@@ -23,7 +23,7 @@
 import logging
 from file_utils import export_to_file, print_summary
 import default_data
-from enricher import enrich_facility_data
+from enrichers import enrich_facility_data
 from schemas import supported_output_types
 from scraper import ICEGovFacilityScraper
 from field_offices import ICEFieldOfficeScraper

schemas.py

@@ -1,16 +1,6 @@
 import copy
 import datetime
 
-OSM_DELAY = 1.0  # 1 second between requests as per OSM policy
-WIKIDATA_DELAY = 0.5  # Be respectful to Wikidata
-WIKIPEDIA_DELAY = 0.5  # Be respectful to Wikipedia
-
-# default to Washington, D.C.?
-default_coords: dict = {
-    "latitude": 38.89511000,
-    "longitude": -77.03637000,
-}
-
 facilities_schema = {
     "scraped_date": datetime.datetime.now(datetime.UTC),
     "scrape_runtime": 0,