move rough schemas to a separate file

Signed-off-by: John Seekins <[email protected]>

Author: johnseekins

enricher.py

@@ -1,10 +1,11 @@
 import copy
 import requests
 from requests.adapters import HTTPAdapter
+from schemas import facilities_schema
 import time
 from urllib.parse import quote
 import urllib3
-from utils import logger, facilities_schema
+from utils import logger
 
 # ExternalDataEnricher class for enrichment logic
 

file_utils.py

@@ -1,11 +1,11 @@
 import csv
 import json
-from utils import (
+from schemas import (
     debug_schema,
     facility_schema,
     enrichment_schema,
-    logger,
 )
+from utils import logger
 
 
 def export_to_file(

schemas.py

@@ -0,0 +1,35 @@
+import datetime
+
+facilities_schema = {
+    "scraped_date": datetime.datetime.utcnow(),
+    "page_updated_date": datetime.datetime.utcnow(),
+    "scrape_runtime": 0,
+    "enrich_runtime": 0,
+    "facilities": [],
+}
+
+facility_schema = {
+    "address": "",
+    "administrative_area": "",
+    "country": "",
+    "facility_url": "",
+    "field_office": "",
+    "full_address": "",
+    "image_url": "",
+    "locality": "",
+    "name": "",
+    "phone": "",
+    "postal_code": "",
+    "raw_scrape": "",
+    "source_url": "",
+}
+enrichment_schema = [
+    "wikipedia_page_url",
+    "wikidata_page_url",
+    "osm_result_url",
+]
+debug_schema = [
+    "wikipedia_search_query",
+    "wikidata_search_query",
+    "osm_search_query",
+]

scraper.py

@@ -6,13 +6,13 @@
 import re
 import requests
 from requests.adapters import HTTPAdapter
-import time
-import urllib3
-from utils import (
+from schemas import (
     facilities_schema,
     facility_schema,
-    logger,
 )
+import time
+import urllib3
+from utils import logger
 
 
 class ICEFacilityScraper(object):

utils.py

@@ -1,41 +1,6 @@
 # For general helpers, regexes, or shared logic (e.g. phone/address parsing functions).
-import datetime
 import logging
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 logger.addHandler(logging.StreamHandler())
-
-facilities_schema = {
-    "scraped_date": datetime.datetime.utcnow(),
-    "page_updated_date": datetime.datetime.utcnow(),
-    "scrape_runtime": 0,
-    "enrich_runtime": 0,
-    "facilities": [],
-}
-
-facility_schema = {
-    "address": "",
-    "administrative_area": "",
-    "country": "",
-    "facility_url": "",
-    "field_office": "",
-    "full_address": "",
-    "image_url": "",
-    "locality": "",
-    "name": "",
-    "phone": "",
-    "postal_code": "",
-    "raw_scrape": "",
-    "source_url": "",
-}
-enrichment_schema = [
-    "wikipedia_page_url",
-    "wikidata_page_url",
-    "osm_result_url",
-]
-debug_schema = [
-    "wikipedia_search_query",
-    "wikidata_search_query",
-    "osm_search_query",
-]