add reverse first and last names to matching

restore setupProxy and merge in statistics db instead of files

Author: urirot

src/client/src/pages/Admin.js

@@ -39,7 +39,8 @@ class Admin extends Component {
             statistics: undefined,
             filesInput: undefined,
             fileListHtml: undefined,
-            lastExecution: undefined
+            lastExecution: undefined,
+            serverBusy: false
         }
 
         this.handleIndexChange = this.handleIndexChange.bind(this);
@@ -85,15 +86,15 @@ class Admin extends Component {
         this.setState({isLoading: false});
     };
 
-    async handleExecute(event) {
+    handleExecute(event) {
         event.preventDefault();
 
         this.setState({isLoading: true});
         fetch('/api/execute');
 
         setTimeout(() => {
             this.refreshPage();
-        }, 3000);
+        }, 1500);
     }
 
     async handleGetStatistics() {
@@ -184,7 +185,9 @@ class Admin extends Component {
                                                 <b>Last Analysis</b>
                                             </TableCell>
                                             <TableCell align="left">
-                                                <b>{moment(this.state.lastExecution, "dddd MMMM Do h:mm:ss YYYY").local().format("MMMM Do YYYY, h:mm:ss a")}</b>
+                                                <b>
+                                                    {moment(this.state.lastExecution, "dddd MMMM Do h:mm:ss YYYY").local().format("MMMM Do YYYY, h:mm:ss a")}
+                                                </b>
                                             </TableCell>
                                         </TableRow>
                                         {this.state.statistics.map((row, index) => (

src/client/src/setupProxy.js

@@ -1,9 +1,9 @@
 const proxy = require('http-proxy-middleware');
 
+
 module.exports = function(app) {
     app.use(proxy('/api/**', {
-            target: process.env.IS_LOCAL === 'true' ? 'http://localhost:5000' : 'http://server:5000',
-            changeOrigin: true,
+            target: process.env.IS_LOCAL === 'true' ? 'http://localhost:3333' : 'http://server:5000'
         }
     ));
 }

src/server/api/admin_api.py

@@ -6,15 +6,14 @@
 from sqlalchemy.sql import text
 
 from sqlalchemy.dialects.postgresql import insert
-from sqlalchemy import Table, Column, Integer, String, MetaData, ForeignKey, exc, select
+from sqlalchemy import Table, MetaData
 from pipeline import flow_script
 from config import engine
-from flask import request, redirect, jsonify, current_app, abort
+from flask import request, redirect, jsonify, current_app
 from api.file_uploader import validate_and_arrange_upload
 from config import (
     RAW_DATA_PATH,
-    CURRENT_SOURCE_FILES_PATH,
-    LOGS_PATH,
+    CURRENT_SOURCE_FILES_PATH
 )
 
 ALLOWED_EXTENSIONS = {"csv", "xlsx"}
@@ -120,8 +119,10 @@ def list_statistics():
         with engine.connect() as connection:
             s = text("select valcol from kv_unique where keycol = 'last_execution_time';")
             result = connection.execute(s)
-            last_execution_details  = result.fetchone()[0]
 
+            fetch_result = result.fetchone()
+            if fetch_result:
+                last_execution_details = result.fetchone()[0]
 
     except Exception as e:
         current_app.logger.error("Failure reading Last Execution stats from DB")

src/server/pipeline/match_data.py

@@ -24,20 +24,27 @@ def start(connection, added_or_updated_rows):
     items_to_update["matching_id"] = 0  # initializing an int and overwrite in the loop
     items_to_update["archived_date"] = np.nan
     items_to_update["created_date"] = datetime.datetime.now()
-    
+
     rows = items_to_update.to_dict(orient="records")
     row_print_freq = max(1, np.floor_divide(len(rows), 20))  # approx every 5% (or every row if small)
     for row_num, row in enumerate(rows):
         if row_num % row_print_freq == 0:
             current_app.logger.info("- Matching rows {}-{} of {}".format(
-                row_num+1, min(len(rows), row_num+row_print_freq), len(rows))
+                row_num + 1, min(len(rows), row_num + row_print_freq), len(rows))
             )
-        
+
         # Exact matches based on specified columns
         row_matches = pdp_contacts[
-            (pdp_contacts["first_name"] == row["first_name"]) &
-            (pdp_contacts["last_name"] == row["last_name"]) &
-            ((pdp_contacts["email"] == row["email"]) | (pdp_contacts["mobile"] == row["mobile"]))
+            (
+                    ((pdp_contacts["first_name"] == row["first_name"]) &
+                    (pdp_contacts["last_name"] == row["last_name"]))
+                    |
+                    ((pdp_contacts["first_name"] == row["last_name"]) &
+                    (pdp_contacts["last_name"] == row["first_name"]))
+                    &
+                    ((pdp_contacts["email"] == row["email"]) | (pdp_contacts["mobile"] == row["mobile"]))
+            )
+
         ]
         if row_matches.empty:  # new record, no matching rows
             max_matching_group += 1
@@ -47,14 +54,14 @@ def start(connection, added_or_updated_rows):
             if not all(row_matches["matching_id"] == row_group):
                 current_app.logger.warning(
                     "Source {} with ID {} is matching multiple groups in pdp_contacts ({})"
-                    .format(row["source_type"], row["source_id"], str(row_matches["matching_id"].drop_duplicates()))
+                        .format(row["source_type"], row["source_id"], str(row_matches["matching_id"].drop_duplicates()))
                 )
         items_to_update.loc[row_num, "matching_id"] = row_group
         # Updating local pdp_contacts dataframe instead of a roundtrip to postgres within the loop.
         # Indexing by iloc and vector of rows to keep the pd.DataFrame class and avoid implicit
         # casting to a single-typed pd.Series.
         pdp_contacts = pdp_contacts.append(items_to_update.iloc[[row_num], :], ignore_index=True)
-    
+
     # Write new data and matching ID's to postgres in bulk, instead of line-by-line
     current_app.logger.info("- Writing data to pdp_contacts table")
     items_to_update.to_sql('pdp_contacts', connection, index=False, if_exists='append')