Skip to content

Commit 4f6e582

Browse files
datajohnsondatajohnson
authored
Merge pull request #349 from icefoganalytics/test
CSL Restrictions Fix up
2 parents d8671f7 + 93bea33 commit 4f6e582

File tree

3 files changed

+167
-179
lines changed

3 files changed

+167
-179
lines changed

‎src/api/repositories/csl-restricted-data/csl-restricted-data-repository.ts‎

Lines changed: 105 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -3,137 +3,120 @@ import { BaseRepository } from "../base-repository";
33
import { UploadedFile } from "express-fileupload";
44
import * as fs from "fs";
55
import moment from "moment";
6+
67
interface CSLRestrictedDTO {
7-
sin:string,
8-
amount_disbursed: string,
9-
birth_date: string,
10-
first_name: string,
11-
last_name: string,
12-
over_award: string,
13-
restriction_reason_id: string,
14-
restriction_warn_id: string,
15-
weeks_accumulated: string,
16-
};
8+
sin: string;
9+
amount_disbursed: string;
10+
birth_date: string;
11+
first_name: string;
12+
last_name: string;
13+
over_award: string;
14+
restriction_reason_id: string;
15+
restriction_warn_id: string;
16+
weeks_accumulated: string;
17+
}
1718

1819
export class CSLRestrictedData extends BaseRepository {
20+
constructor(maindb: Knex<any, unknown>) {
21+
super(maindb);
22+
}
1923

20-
constructor(maindb: Knex<any, unknown>) {
21-
super(maindb);
22-
//this.applicationRepo = new ApplicationRepository(maindb);
23-
}
24+
async getResponse(file: UploadedFile): Promise<any> {
25+
const fiteredData: CSLRestrictedDTO[] = this.filterRecords(file.data);
2426

25-
//AfterPForm ()
26-
async getResponse(
27-
file: UploadedFile
28-
): Promise<any> {
29-
30-
const dataFile = file.data;
31-
const fiteredData: CSLRestrictedDTO[] = this.filterRecords(file.data);
32-
const csvData = this.convertToCSV(fiteredData);
33-
34-
fs.writeFile('bulk_insert.csv', csvData, (err) => {
35-
if (err) {
36-
console.error('Error when creating the file:', err);
37-
return;
38-
}
39-
});
40-
console.log('[CSL-Restricted]The INSERT process is started at: ' + moment().toString());
41-
const truncateTable = await this.mainDb('sfa.csl_restricted').truncate();
42-
const resUpdate = await this.insertRecords(fiteredData);
43-
44-
return { success: true, data: {} };
45-
}
27+
console.log("[CSL-Restricted] Started at: " + moment().toString(), fiteredData.length);
28+
await this.mainDb("sfa.csl_restricted").truncate();
4629

47-
async insertRecords(
48-
records: CSLRestrictedDTO[]
49-
): Promise<boolean> {
50-
51-
try {
52-
const self = this;
53-
const chunkSize = 50;
54-
this.mainDb.batchInsert('sfa.csl_restricted', records, chunkSize)
55-
.returning('sin')
56-
.then(function(ids) {
57-
console.log('[CSL-Restricted]The INSERT process is over: ' + moment().toString());
58-
59-
console.log('[CSL-Restricted]The process for updated the applicatios is started at: ' + moment().toString());
60-
61-
const result = self.mainDb.raw(`EXEC sfa.sp_update_csl_fields`).then(function() {
62-
console.log('[CSL-Restricted]The process for updated the applicatios is over: ' + moment().toString());
63-
})
64-
})
65-
.catch(function(error) {
66-
console.log(error);
67-
});
68-
return true;
69-
} catch (error) {
70-
console.log(error);
71-
return false;
72-
}
30+
await this.insertRecords(fiteredData);
7331

74-
}
32+
await this.mainDb.raw(`EXEC sfa.sp_update_csl_fields`);
7533

76-
filterRecords(
77-
dataFile: Buffer
78-
): any[] {
79-
80-
const firstFilter: string[] = dataFile.toString()?.split('\n');
81-
82-
const secondFilter: CSLRestrictedDTO[] = firstFilter
83-
.filter((data: string, index) => data.length > 0)
84-
.map((data: string, index) => {
85-
86-
const restrictedData: any = {
87-
sin: '',
88-
amount_disbursed: '',
89-
birth_date: '',
90-
first_name: '',
91-
last_name: '',
92-
over_award: '',
93-
restriction_reason_id: '',
94-
restriction_warn_id: '',
95-
weeks_accumulated: '',
96-
nslsc_restrict1: '',
97-
nslsc_restrict2: '',
98-
nslsc_restrict3: '',
99-
calsc_restrict1: '',
100-
calsc_restrict2: '',
101-
calsc_restrict3: '',
102-
fi_restrict1: ''
103-
};
104-
105-
//CSL_RESTRICTED_ID -> SIN
106-
restrictedData.sin = data.slice(0, 9).trim();
107-
//AMOUNT_DISBURSED
108-
restrictedData.amount_disbursed = data.slice(86, 92).trim();
109-
//BIRTH_DATE
110-
restrictedData.birth_date = data.slice(69, 77).trim();
111-
//FIRST_NAME
112-
restrictedData.first_name = data.slice(39, 69).trim();
113-
//LAST_NAME
114-
restrictedData.last_name = data.slice(9, 39).trim();
115-
//OVER_AWARD
116-
restrictedData.over_award = data.slice(77, 83).trim();
117-
//RESTRICTION_REASON_ID
118-
restrictedData.restriction_reason_id = data.slice(85, 86).trim();
119-
//RESTRICTION_WARN_ID
120-
restrictedData.restriction_warn_id = data.slice(84, 85).trim();
121-
//WEEKS_ACCUMULATED
122-
restrictedData.weeks_accumulated = data.slice(93, 96).trim();
123-
124-
return restrictedData;
125-
});
126-
127-
return secondFilter;
128-
}
34+
console.log("[CSL-Restricted] Completed at: " + moment().toString());
12935

130-
convertToCSV(arr: any[]) {
131-
const array = [Object.keys(arr[0])].concat(arr)
36+
return { success: true, data: {} };
37+
}
13238

133-
return array.map(it => {
134-
return Object.values(it).toString()
135-
}).join('\n')
136-
}
39+
async insertRecords(records: CSLRestrictedDTO[]): Promise<boolean> {
40+
try {
41+
// break the records into sets of 10000 then loop until all records are inserted
42+
const batchSize = 1000;
43+
const chunkSize = 100;
44+
const batchCount = Math.ceil(records.length / batchSize);
13745

46+
for (let i = 0; i < batchCount; i++) {
47+
const start = i * batchSize;
48+
const end = start + batchSize;
49+
const batch = records.slice(start, end);
50+
console.log("Processing batch " + (i + 1) + " of " + batchCount + ", records: " + batch.length);
13851

139-
}
52+
await this.mainDb.batchInsert("sfa.csl_restricted", batch, chunkSize);
53+
}
54+
55+
return true;
56+
} catch (error) {
57+
console.log("[CSL-Restricted]Error during the INSERT process: " + moment().toString());
58+
console.log(error);
59+
return false;
60+
}
61+
}
62+
63+
filterRecords(dataFile: Buffer): any[] {
64+
const firstFilter: string[] = dataFile.toString()?.split("\n");
65+
66+
const secondFilter: CSLRestrictedDTO[] = firstFilter
67+
.filter((data: string, index) => data.length > 0)
68+
.map((data: string, index) => {
69+
const restrictedData: any = {
70+
sin: "",
71+
amount_disbursed: "",
72+
birth_date: "",
73+
first_name: "",
74+
last_name: "",
75+
over_award: "",
76+
restriction_reason_id: "",
77+
restriction_warn_id: "",
78+
weeks_accumulated: "",
79+
nslsc_restrict1: "",
80+
nslsc_restrict2: "",
81+
nslsc_restrict3: "",
82+
calsc_restrict1: "",
83+
calsc_restrict2: "",
84+
calsc_restrict3: "",
85+
fi_restrict1: "",
86+
};
87+
88+
//CSL_RESTRICTED_ID -> SIN
89+
restrictedData.sin = data.slice(0, 9).trim();
90+
//AMOUNT_DISBURSED
91+
restrictedData.amount_disbursed = data.slice(86, 92).trim();
92+
//BIRTH_DATE
93+
restrictedData.birth_date = data.slice(69, 77).trim();
94+
//FIRST_NAME
95+
restrictedData.first_name = data.slice(39, 69).trim();
96+
//LAST_NAME
97+
restrictedData.last_name = data.slice(9, 39).trim();
98+
//OVER_AWARD
99+
restrictedData.over_award = data.slice(77, 83).trim();
100+
//RESTRICTION_REASON_ID
101+
restrictedData.restriction_reason_id = data.slice(85, 86).trim();
102+
//RESTRICTION_WARN_ID
103+
restrictedData.restriction_warn_id = data.slice(84, 85).trim();
104+
//WEEKS_ACCUMULATED
105+
restrictedData.weeks_accumulated = data.slice(93, 96).trim();
106+
107+
return restrictedData;
108+
});
109+
110+
return secondFilter;
111+
}
112+
113+
convertToCSV(arr: any[]) {
114+
const array = [Object.keys(arr[0])].concat(arr);
115+
116+
return array
117+
.map((it) => {
118+
return Object.values(it).toString();
119+
})
120+
.join("\n");
121+
}
122+
}

‎src/api/routes/admin/csl-restricted-data-router.ts‎

Lines changed: 35 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,42 @@
11
import knex from "knex";
22
import express, { Request, Response } from "express";
3-
import { body, param } from "express-validator";
4-
import { ReturnValidationErrors } from "../../middleware";
53
import { DB_CONFIG } from "../../config";
6-
import * as fs from "fs";
74
import { CSLRestrictedData } from "@/repositories/csl-restricted-data";
85

9-
const db = knex(DB_CONFIG)
6+
const db = knex(DB_CONFIG);
107
export const cslRestrictedData = express.Router();
118

12-
cslRestrictedData.post("/upload-file",
13-
async (req: Request, res: Response) => {
14-
try {
15-
const file: any = req?.files?.file;
16-
17-
const cslRestricted = new CSLRestrictedData(db);
18-
19-
const results = await cslRestricted.getResponse(file);
20-
21-
return res.status(200).json({});
22-
} catch (error) {
23-
console.log(error);
24-
return res.status(404).send();
25-
}
26-
}
27-
);
28-
29-
30-
cslRestrictedData.get("/csl-restricted-count",
31-
async (req: Request, res: Response) => {
32-
try {
33-
const count = await db("sfa.csl_restricted").count('id as CNT').then(function(total) {
34-
res.send({
35-
total: total[0].CNT
36-
});
37-
}).catch(function(error) {
38-
console.log(error);
39-
res.send({
40-
total: 0
41-
});
42-
});
43-
} catch (error) {
44-
console.log(error);
45-
return res.status(404).send();
46-
}
47-
}
48-
);
49-
50-
51-
9+
cslRestrictedData.post("/upload-file", async (req: Request, res: Response) => {
10+
try {
11+
const file: any = req?.files?.file;
12+
13+
const cslRestricted = new CSLRestrictedData(db);
14+
15+
console.log("11")
16+
17+
await cslRestricted.getResponse(file);
18+
console.log("22")
19+
20+
21+
return res.status(200).json({});
22+
} catch (error) {
23+
console.log(error);
24+
return res.status(404).send();
25+
}
26+
});
27+
28+
cslRestrictedData.get("/csl-restricted-count", async (req: Request, res: Response) => {
29+
await db("sfa.csl_restricted")
30+
.count("id as CNT")
31+
.then(function (total) {
32+
res.send({
33+
total: total[0].CNT,
34+
});
35+
})
36+
.catch(function (error) {
37+
console.log(error);
38+
res.send({
39+
total: 0,
40+
});
41+
});
42+
});

0 commit comments

Comments
 (0)