Merge branch 'master' of github.com:CodeForPhilly/paws-data-pipeline into issue-166-360Page

Author: dcstlouis

src/server/api/common_api.py

@@ -15,9 +15,9 @@ def get_contacts(search_text):
 
         names = search_text.split(" ")
         if len(names) == 2:
-            query = text("select * from pdp_contacts WHERE archived_date is null AND \
-                (lower(first_name) like lower(:name1) AND lower(last_name) like lower(:name2) \
-                OR lower(first_name) like lower(:name2) AND lower(last_name) like lower(:name1))")
+            query = text("select * from pdp_contacts where archived_date is null AND\
+                lower(first_name) like lower(:name1) and lower(last_name) like lower(:name2) \
+                OR lower(first_name) like lower(:name2) and lower(last_name) like lower(:name1)")
             query_result = connection.execute(query, name1='{}%'.format(names[0]), name2='{}%'.format(names[1]))
         elif len(names) == 1:
             query = text("select * from pdp_contacts \

src/server/pipeline/clean_and_load_data.py

@@ -6,6 +6,7 @@
 
 from datasource_manager import DATASOURCE_MAPPING, SOURCE_NORMALIZATION_MAPPING
 from flask import current_app
+import sqlalchemy
 from config import CURRENT_SOURCE_FILES_PATH
 
 
@@ -35,7 +36,12 @@ def start(connection, pdp_contacts_df, file_path_list):
                 result = pd.concat([result, source_df])
 
         else:
-            df.to_sql(table_name, connection, index=False, if_exists='append')
+            if table_name in sqlalchemy.inspect(connection).get_table_names():
+                # Only retain new/updated records in secondary tables (shifts, donations, etc.)
+                current_app.logger.info('   - Deduplicating old records')
+                old_data = pd.read_sql_table(table_name, connection)
+                df = old_data.append(df, ignore_index=True).drop_duplicates()
+            df.to_sql(table_name, connection, index=False, if_exists='replace')
 
         current_app.logger.info('   - Finish load_paws_data on: ' + uploaded_file)