added manual matches as parted of automated matching flow

Author: sposerina

src/server/pipeline/clean_and_load_data.py

@@ -17,8 +17,9 @@ def start(connection, pdp_contacts_df, file_path_list):
     for uploaded_file in file_path_list:
         file_path = os.path.join(CURRENT_SOURCE_FILES_PATH, uploaded_file)
         table_name = file_path.split('/')[-1].split('-')[0]
-        if table_name == 'manual_matches':
+        if table_name == 'manualmatches':
             manual_matches_df = pd.read_csv((io.BytesIO(open(file_path, "rb").read())), encoding='iso-8859-1')
+            manual_matches_df[["volgistics", "shelterluvpeople"]] = manual_matches_df[["volgistics", "shelterluvpeople"]].fillna(0).astype(int).astype(str)
             continue
 
         current_app.logger.info('Running load_paws_data on: ' + uploaded_file)

src/server/pipeline/match_data.py

@@ -22,8 +22,6 @@ def start(connection, added_or_updated_rows, manual_matches_df):
     current_app.logger.info('Start record matching')
     # Will need to consider updating the existing row contents (filter by active), deactivate,
     # try to match, and merge previous matching groups if applicable
-    current_app.logger.info('Manual matches DF: ' + manual_matches_df)
-
     job_id = str(int(time.time()))
     log_db.log_exec_status(job_id, {'status': 'starting', 'at_row': 0, 'of_rows': 0})
     current_app.logger.info("***** Running execute job ID " + job_id + " *****")
@@ -73,6 +71,13 @@ def start(connection, added_or_updated_rows, manual_matches_df):
                 ((pdp_contacts["email_normalized"] == row["email_normalized"]) | (pdp_contacts["mobile"] == row["mobile"]))
             )
         ]
+        #collect other linked ids from manual matches source
+        linked_ids = manual_matches_df[(manual_matches_df[row["source_type"]] == row["source_id"])]
+        ids = linked_ids.to_dict(orient="records")
+        for id_num, row_dict in enumerate(ids):
+            for column, value in row_dict.items():
+                row_matches = row_matches.append(pdp_contacts[(pdp_contacts["source_type"] == column) & (pdp_contacts["source_id"] == value)])
+        
         if row_matches.empty:  # new record, no matching rows
             max_matching_group += 1
             row_group = max_matching_group