Skip to content

Commit 58b1cc5

Browse files
c-simpsonc-simpson
committed
Merge branch 'master' of github.com:CodeForPhilly/paws-data-pipeline
Updates to Uri's b7c4cce merge
2 parents 11f56e5 + b7c4cce commit 58b1cc5

File tree

4 files changed

+136
-127
lines changed

4 files changed

+136
-127
lines changed

‎src/client/src/pages/Admin.js

Lines changed: 115 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -17,25 +17,32 @@ import {
1717
import {withStyles} from '@material-ui/core/styles';
1818
import _ from 'lodash';
1919
import moment from "moment";
20+
import {Alert} from "@material-ui/lab";
2021

2122
const styles = theme => ({
23+
root: {
24+
margin: theme.spacing(2),
25+
},
2226
backdrop: {
2327
zIndex: theme.zIndex.drawer + 1,
2428
color: '#fff',
2529
}
2630
});
2731

2832

33+
const executing = "executing"
34+
2935
class Admin extends Component {
3036
constructor(props) {
3137
super(props);
3238
this.state = {
3339
activeIndex: 0,
34-
isLoading: false,
35-
statistics: [],
40+
isLoading: undefined,
41+
statistics: undefined,
3642
filesInput: undefined,
3743
fileListHtml: undefined,
38-
lastExecution: undefined
44+
lastExecution: undefined,
45+
serverBusy: false
3946
}
4047

4148
this.handleIndexChange = this.handleIndexChange.bind(this);
@@ -45,9 +52,13 @@ class Admin extends Component {
4552
this.handleGetStatistics = this.handleGetStatistics.bind(this);
4653
}
4754

48-
refreshPage() {
49-
this.handleGetFileList();
50-
this.handleGetStatistics();
55+
async refreshPage() {
56+
this.setState({isLoading: true})
57+
58+
await this.handleGetFileList();
59+
await this.handleGetStatistics();
60+
61+
this.setState({isLoading: false})
5162
}
5263

5364
componentDidMount() {
@@ -82,138 +93,130 @@ class Admin extends Component {
8293

8394
this.setState({isLoading: true});
8495

85-
const response = await fetch('/api/execute');
86-
const result = await response.json();
87-
88-
this.setState({isLoading: false});
96+
await fetch('/api/execute');
8997

9098
this.refreshPage();
91-
92-
return result
9399
}
94100

95101
async handleGetStatistics() {
96-
this.setState({isLoading: true})
97-
98-
try {
99-
const statsData = await fetch("/api/statistics");
100-
const statsResponse = await statsData.json();
102+
const statsData = await fetch("/api/statistics");
103+
const statsResponse = await statsData.json()
101104

105+
if (statsResponse !== 'executing') {
102106
this.setState({
103107
statistics: _.toPairsIn(statsResponse.stats),
104108
lastExecution: statsResponse.executionTime
105109
});
106-
107-
this.setState({isLoading: false})
108-
} finally {
109-
this.setState({isLoading: false})
110+
} else {
111+
this.setState({statistics: statsResponse});
110112
}
111-
112113
}
113114

114115
async handleGetFileList() {
115-
this.setState({isLoading: true})
116-
117-
try {
118-
const filesData = await fetch("/api/listCurrentFiles");
119-
const filesResponse = await filesData.json();
120-
121-
this.setState({fileListHtml: filesResponse});
122-
123-
} finally {
124-
this.setState({isLoading: false})
125-
}
116+
const filesData = await fetch("/api/listCurrentFiles");
117+
const filesResponse = await filesData.json();
118+
this.setState({fileListHtml: filesResponse});
126119
}
127120

128121
render() {
129122
const {classes} = this.props;
130123

131124
return (
132-
<div style={{paddingLeft: 20}}>
125+
<div className={classes.root}>
133126
<h1>Admin Portal</h1>
134-
<Backdrop className={classes.backdrop} open={this.state.isLoading === true}>
135-
<CircularProgress size={60}/>
136-
</Backdrop>
137-
<Grid container spacing={3} direction="column" style={{padding: 30}}>
138-
<Grid container spacing={3} direction="row">
139-
<Grid item sm={6}>
140-
<h2>Latest Files</h2>
141-
{_.isEmpty(this.state.fileListHtml) !== true &&
142-
<TableContainer component={Paper} className="statisticsData">
143-
<Table aria-label="simple table" className={classes.table}>
144-
<TableHead>
145-
<TableRow>
146-
<TableCell><b>File Type</b></TableCell>
147-
<TableCell><b>Last Updated</b></TableCell>
148-
</TableRow>
149-
</TableHead>
150-
<TableBody>
151-
{_.map(this.state.fileListHtml, file => {
152-
const fileName = file.split("-")[0];
153-
let fileDate = file.split("-").slice(1).join().split(".")[0];
154-
let fileDateOnlyNumbers = fileDate.replaceAll(",", "");
155-
let fileDateFormatted = moment(fileDateOnlyNumbers, "YYYYMMDDhmmss").local().format("MMMM Do YYYY, h:mm:ss a");
156-
157-
return (
158-
<TableRow>
159-
<TableCell>{fileName}</TableCell>
160-
<TableCell>{fileDateFormatted}</TableCell>
161-
</TableRow>
162-
)
163-
})
164-
}
165-
</TableBody>
166-
</Table>
167-
</TableContainer>}
168-
169-
<Paper style={{padding: 5, marginTop: 10}}>
170-
<CardContent>
171-
<h3 style={{marginTop: 0}}>Upload Files</h3>
172-
<form onSubmit={this.handleUpload}>
173-
<input type="file" value={this.state.filesInput} multiple/>
174-
<Button type="submit" variant="contained" color="primary">Upload</Button>
175-
</form>
176-
</CardContent>
177-
</Paper>
178-
</Grid>
179-
180-
<Grid item sm={6}>
181-
<h2> Last Match Analysis </h2>
182-
{_.isEmpty(this.state.statistics) !== true &&
183-
<TableContainer component={Paper} className="statisticsData">
184-
<Table aria-label="simple table" className={classes.table}>
185-
<TableBody>
186-
<TableRow key='time'>
127+
{this.state.statistics === 'Running' && <Alert severity="info">Execution is in Progress...</Alert>}
128+
<Backdrop className={classes.backdrop} open={this.state.isLoading !== false}>
129+
<CircularProgress size={60}/>
130+
</Backdrop>
131+
<Grid container spacing={3} direction="column" style={{padding: 20}}>
132+
<Grid container spacing={3} direction="row">
133+
<Grid item sm={6}>
134+
<h2>Latest Files</h2>
135+
{_.isEmpty(this.state.fileListHtml) !== true &&
136+
<TableContainer component={Paper} className="statisticsData">
137+
<Table aria-label="simple table" className={classes.table}>
138+
<TableHead>
139+
<TableRow>
140+
<TableCell><b>File Type</b></TableCell>
141+
<TableCell><b>Last Updated</b></TableCell>
142+
</TableRow>
143+
</TableHead>
144+
<TableBody>
145+
{_.map(this.state.fileListHtml, (file, index) => {
146+
const fileName = file.split("-")[0];
147+
let fileDate = file.split("-").slice(1).join().split(".")[0];
148+
let fileDateOnlyNumbers = fileDate.replaceAll(",", "");
149+
let fileDateFormatted = moment(fileDateOnlyNumbers, "YYYYMMDDhmmss").local().format("MMMM Do YYYY, h:mm:ss a");
150+
151+
return (
152+
<TableRow key={index}>
153+
<TableCell>{fileName}</TableCell>
154+
<TableCell>{fileDateFormatted}</TableCell>
155+
</TableRow>
156+
)
157+
})
158+
}
159+
</TableBody>
160+
</Table>
161+
</TableContainer>}
162+
163+
<Paper style={{padding: 5, marginTop: 10}}>
164+
<CardContent>
165+
<h3 style={{marginTop: 0}}>Upload Files</h3>
166+
<form onSubmit={this.handleUpload}>
167+
<input type="file" value={this.state.filesInput} multiple/>
168+
<Button type="submit" variant="contained" color="primary"
169+
disabled={this.state.statistics === 'Running'}>
170+
Upload
171+
</Button>
172+
</form>
173+
</CardContent>
174+
</Paper>
175+
</Grid>
176+
177+
<Grid item sm={6}>
178+
<h2> Last Match Analysis </h2>
179+
{_.isEmpty(this.state.statistics) !== true &&
180+
this.state.statistics !== 'Running' &&
181+
<TableContainer component={Paper} className="statisticsData">
182+
<Table aria-label="simple table" className={classes.table}>
183+
<TableBody>
184+
<TableRow key='time'>
185+
<TableCell align="left" component="th" scope="row">
186+
<b>Last Analysis</b>
187+
</TableCell>
188+
<TableCell align="left">
189+
<b>
190+
{moment(this.state.lastExecution, "dddd MMMM Do h:mm:ss YYYY").local().format("MMMM Do YYYY, h:mm:ss a")}
191+
</b>
192+
</TableCell>
193+
</TableRow>
194+
{this.state.statistics.map((row, index) => (
195+
<TableRow key={index}>
187196
<TableCell align="left" component="th" scope="row">
188-
<b>Last Analysis</b>
189-
</TableCell>
190-
<TableCell align="left">
191-
<b>{moment(this.state.lastExecution, "dddd MMMM Do h:mm:ss YYYY").local().format("MMMM Do YYYY, h:mm:ss a")}</b>
197+
{row[0]}
192198
</TableCell>
199+
<TableCell align="left">{row[1]}</TableCell>
193200
</TableRow>
194-
{this.state.statistics.map((row, index) => (
195-
<TableRow key={index}>
196-
<TableCell align="left" component="th" scope="row">
197-
{row[0]}
198-
</TableCell>
199-
<TableCell align="left">{row[1]}</TableCell>
200-
</TableRow>
201-
))}
202-
</TableBody>
203-
</Table>
204-
</TableContainer>}
205-
<Paper style={{padding: 5, marginTop: 10}}>
206-
<CardContent>
207-
<h3 style={{marginTop: 0}}>Run New Analysis</h3>
208-
<form onSubmit={this.handleExecute}>
209-
<Button type="submit" variant="contained"
210-
color="primary">Run Data Analysis</Button>
211-
</form>
212-
</CardContent>
213-
</Paper>
214-
</Grid>
201+
))}
202+
</TableBody>
203+
</Table>
204+
</TableContainer>
205+
}
206+
<Paper style={{padding: 5, marginTop: 10}}>
207+
<CardContent>
208+
<h3 style={{marginTop: 0}}>Run New Analysis</h3>
209+
<form onSubmit={this.handleExecute}>
210+
<Button type="submit" variant="contained" color="primary"
211+
disabled={this.state.statistics === 'Running'}>
212+
Run Data Analysis
213+
</Button>
214+
</form>
215+
</CardContent>
216+
</Paper>
215217
</Grid>
216218
</Grid>
219+
</Grid>
217220
</div>
218221
);
219222
}

‎src/client/src/setupProxy.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
const proxy = require('http-proxy-middleware');
22

3+
34
module.exports = function(app) {
45
app.use(proxy('/api/**', {
5-
target: process.env.IS_LOCAL === 'true' ? 'http://localhost:5000' : 'http://server:5000',
6-
changeOrigin: true,
6+
target: process.env.IS_LOCAL === 'true' ? 'http://localhost:3333' : 'http://server:5000'
77
}
88
));
99
}

‎src/server/api/admin_api.py

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,8 @@ def __allowed_file(filename):
3131
# file upload tutorial
3232
@admin_api.route("/api/file", methods=["POST"])
3333
def uploadCSV():
34-
current_app.logger.info("Uploading CSV")
3534
if "file" not in request.files:
36-
return jsonify({"error": "no file supplied"});
35+
return redirect(request.url)
3736

3837
for file in request.files.getlist("file"):
3938
if __allowed_file(file.filename):
@@ -44,7 +43,7 @@ def uploadCSV():
4443
finally:
4544
file.close()
4645

47-
return jsonify({"success": "uploaded file"});
46+
return redirect("/")
4847

4948

5049
@admin_api.route("/api/listCurrentFiles", methods=["GET"])
@@ -71,14 +70,14 @@ def execute():
7170
last_execution_details = {"executionTime": current_time, "stats": statistics}
7271
last_ex_json = (json.dumps(last_execution_details))
7372

74-
# Write Last Execution stats to DB
73+
# Write Last Execution stats to DB
7574
# See Alembic Revision ID: 05e0693f8cbb for table definition
7675
with engine.connect() as connection:
77-
ins_stmt = insert(kvt).values( # Postgres-specific insert() supporting ON CONFLICT
76+
ins_stmt = insert(kvt).values( # Postgres-specific insert() supporting ON CONFLICT
7877
keycol = 'last_execution_time',
7978
valcol = last_ex_json,
8079
)
81-
# If key already present in DB, do update instead
80+
# If key already present in DB, do update instead
8281
upsert = ins_stmt.on_conflict_do_update(
8382
constraint='kv_unique_keycol_key',
8483
set_=dict(valcol=last_ex_json)

‎src/server/pipeline/match_data.py

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ def start(connection, added_or_updated_rows):
3030
items_to_update["matching_id"] = 0 # initializing an int and overwrite in the loop
3131
items_to_update["archived_date"] = np.nan
3232
items_to_update["created_date"] = datetime.datetime.now()
33-
33+
3434
rows = items_to_update.to_dict(orient="records")
3535
row_print_freq = max(1, np.floor_divide(len(rows), 20)) # approx every 5% (or every row if small)
3636
for row_num, row in enumerate(rows):
@@ -39,12 +39,19 @@ def start(connection, added_or_updated_rows):
3939
row_num+1, min(len(rows), row_num+row_print_freq), len(rows))
4040
)
4141
log_db.log_exec_status(job_id,{'status': 'executing', 'at_row': row_num+1, 'of_rows':len(rows)})
42-
42+
4343
# Exact matches based on specified columns
4444
row_matches = pdp_contacts[
45-
(pdp_contacts["first_name"] == row["first_name"]) &
46-
(pdp_contacts["last_name"] == row["last_name"]) &
47-
((pdp_contacts["email"] == row["email"]) | (pdp_contacts["mobile"] == row["mobile"]))
45+
(
46+
((pdp_contacts["first_name"] == row["first_name"]) &
47+
(pdp_contacts["last_name"] == row["last_name"]))
48+
|
49+
((pdp_contacts["first_name"] == row["last_name"]) &
50+
(pdp_contacts["last_name"] == row["first_name"]))
51+
&
52+
((pdp_contacts["email"] == row["email"]) | (pdp_contacts["mobile"] == row["mobile"]))
53+
)
54+
4855
]
4956
if row_matches.empty: # new record, no matching rows
5057
max_matching_group += 1
@@ -54,14 +61,14 @@ def start(connection, added_or_updated_rows):
5461
if not all(row_matches["matching_id"] == row_group):
5562
current_app.logger.warning(
5663
"Source {} with ID {} is matching multiple groups in pdp_contacts ({})"
57-
.format(row["source_type"], row["source_id"], str(row_matches["matching_id"].drop_duplicates()))
64+
.format(row["source_type"], row["source_id"], str(row_matches["matching_id"].drop_duplicates()))
5865
)
5966
items_to_update.loc[row_num, "matching_id"] = row_group
6067
# Updating local pdp_contacts dataframe instead of a roundtrip to postgres within the loop.
6168
# Indexing by iloc and vector of rows to keep the pd.DataFrame class and avoid implicit
6269
# casting to a single-typed pd.Series.
6370
pdp_contacts = pdp_contacts.append(items_to_update.iloc[[row_num], :], ignore_index=True)
64-
71+
6572
# Write new data and matching ID's to postgres in bulk, instead of line-by-line
6673
current_app.logger.info("- Writing data to pdp_contacts table")
6774
items_to_update.to_sql('pdp_contacts', connection, index=False, if_exists='append')

0 commit comments

Comments
 (0)