fixed missing positional arguments in logging statement, added support for reusing the same session when storing all animals and events, restored various logging statements in ingest_sources_from_api.py

Author: sposerina

src/server/api/API_ingest/ingest_sources_from_api.py

@@ -1,22 +1,18 @@
-from api.API_ingest import shelterluv_people, salesforce_contacts
+from api.API_ingest import shelterluv_people, salesforce_contacts, sl_animal_events
 import structlog
 logger = structlog.get_logger()
 
 def start(session):
     logger.debug("Start Fetching raw data from different API sources")
-    #Run each source to store the output in dropbox and in the container as a CSV
-    shelterluv_people.store_shelterluv_people_all(session)
+    logger.debug("   Fetching Salesforce contacts")
     salesforce_contacts.store_contacts_all(session)
-    logger.debug("Finish Fetching raw data from different API sources")
-    slp_count = shelterluv_api_handler.store_shelterluv_people_all(conn)
-    logger.debug("   Finished fetching Shelterluv people - %d records" , slp_count)
-
+    logger.debug("   Finished fetching Salesforce contacts")
+    logger.debug("   Fetching Shelterluv people")
+    shelterluv_people.store_shelterluv_people_all(session)
+    logger.debug("   Finished fetching Shelterluv people")
     logger.debug("   Fetching Shelterluv events")
-    #Run each source to store the output in dropbox and in the container as a CSV
-    sle_count = sl_animal_events.slae_test()
+    sle_count = sl_animal_events.store_all_animals_and_events(session)
     logger.debug("   Finished fetching Shelterluv events - %d records" , sle_count)
 
     logger.debug("Finished fetching raw data from different API sources")
 
-
-    #TODO:   Return object with count for each data source?

src/server/api/API_ingest/shelterluv_db.py

@@ -14,6 +14,9 @@
 from api.file_uploader import validate_and_arrange_upload
 from sqlalchemy.orm import Session, sessionmaker
 
+import structlog
+logger = structlog.get_logger()
+
 
 def insert_animals(animal_list):
     """Insert animal records into shelterluv_animals table and return row count. """
@@ -65,31 +68,28 @@ def truncate_animals():
     return 0
 
 
-def truncate_events():
+def truncate_events(session):
     """Truncate the shelterluv_events table"""
 
-    Session = sessionmaker(engine)
-    session = Session()
     metadata = MetaData()
     sla = Table("sl_animal_events", metadata, autoload=True, autoload_with=engine)
 
     truncate = "TRUNCATE table sl_animal_events;"
     result = session.execute(truncate)
 
-    session.commit()  # Commit all inserted rows
-    session.close()
-
     return 0
 
-
 def insert_events(event_list):
+    Session = sessionmaker(engine)
+    session = Session()
+    insert_events(session,event_list)
+
+def insert_events(session, event_list):
     """Insert event records into sl_animal_events table and return row count. """
 
     # Always a clean insert
-    truncate_events()
+    truncate_events(session)
 
-    Session = sessionmaker(engine)
-    session = Session()
     metadata = MetaData()
     sla = Table("sl_animal_events", metadata, autoload=True, autoload_with=engine)
 
@@ -141,9 +141,7 @@ def insert_events(event_list):
 
     # TODO: Wrap with try/catch
     ret = session.execute(sla.insert(ins_list))
-
-    session.commit()  # Commit all inserted rows
-    session.close()
+    logger.debug("finished inserting events")
 
     return ret.rowcount
 

src/server/api/API_ingest/sl_animal_events.py

@@ -7,7 +7,6 @@
 import requests
 
 from api.API_ingest import shelterluv_db
-from server.api.API_ingest.shelterluv_db import insert_animals
 
 # There are a number of different record types. These are the ones we care about.
 keep_record_types = [
@@ -143,7 +142,7 @@ def get_events_bulk():
             more_records = decoded["has_more"]  # if so, we'll make another pass
             offset += limit
             if offset % 1000 == 0:
-                logger.debug("Reading offset ", str(offset))
+                logger.debug("Reading offset %s", str(offset))
                 if TEST_MODE and offset > 1000:
                     more_records=False  # Break out early 
 
@@ -155,17 +154,30 @@ def get_events_bulk():
 
 def slae_test():
     total_count = get_event_count()
-    logger.debug("Total events:", total_count)
+    logger.debug("Total events: %d", total_count)
 
     b = get_events_bulk()
-    logger.debug("Strored records:", len(b))
+    logger.debug("Stored records: %d", len(b))
 
     # f = filter_events(b)
     # print(f)
 
     count = shelterluv_db.insert_events(b)
     return count
 
+def store_all_animals_and_events(session):
+    total_count = get_event_count()
+    logger.debug("Total events: %d", total_count)
+
+    b = get_events_bulk()
+    logger.debug("Stored records: %d", len(b))
+
+    # f = filter_events(b)
+    # print(f)
+
+    count = shelterluv_db.insert_events(session, b)
+    return count
+
 
 # Query to get last adopt/foster event: