Skip to content

Commit 2c6d6cf

Browse files
datajohnsondatajohnson
authored
Merge pull request ytgov#100 from icefoganalytics/main
CSV Reports
2 parents cde038b + 7efd50b commit 2c6d6cf

File tree

16 files changed

+622
-213
lines changed

16 files changed

+622
-213
lines changed

‎api/src/data/migrations/20250811121453.drop-exclusivity-constraint-from-data-ingestion-mappings.ts‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
import * as knex from "knex";
2+
3+
export async function up(knex: knex.Knex) {
4+
const client = knex.client.config.client;
5+
6+
if (client === "pg" || client === "postgres" || client === "postgresql") {
7+
await knex.raw(`
8+
DROP TRIGGER IF EXISTS "trigger_check_mutual_exclusivity" ON "data_ingestion_mappings";
9+
DROP FUNCTION IF EXISTS "check_unique_mutual_exclusivity"();
10+
`);
11+
} else if (client === "oracledb" || client === "oracle") {
12+
await knex.raw(`
13+
BEGIN
14+
EXECUTE IMMEDIATE 'DROP TRIGGER "trigger_check_mutual_exclusivity"';
15+
EXCEPTION
16+
WHEN OTHERS THEN
17+
IF SQLCODE != -4080 THEN RAISE; END IF;
18+
END;
19+
`);
20+
} else {
21+
throw new Error("Unsupported database client for this migration.");
22+
}
23+
}
24+
25+
26+
export async function down(knex: knex.Knex) {
27+
const client = knex.client.config.client;
28+
29+
if (client === "pg" || client === "postgresql") {
30+
await knex.raw(`
31+
CREATE OR REPLACE FUNCTION check_unique_mutual_exclusivity()
32+
RETURNS trigger AS $$
33+
BEGIN
34+
IF (NEW.source_value IS NULL OR NEW.source_value = '') THEN
35+
IF EXISTS (
36+
SELECT 1 FROM data_ingestion_mappings
37+
WHERE source_id= NEW.source_id
38+
AND source_attribute= NEW.source_attribute
39+
AND source_value IS NOT NULL
40+
AND target_attribute= NEW.target_attribute
41+
) THEN
42+
RAISE EXCEPTION
43+
'Cannot insert NULL mapping when specific mappings exist for (source_id=% , attribute=%, target=%)',
44+
NEW.source_id, NEW.source_attribute, NEW.target_attribute;
45+
END IF;
46+
ELSE
47+
IF EXISTS (
48+
SELECT 1 FROM data_ingestion_mappings
49+
WHERE source_id = NEW.source_id
50+
AND source_attribute= NEW.source_attribute
51+
AND (source_value IS NULL OR source_value = '')
52+
AND target_attribute= NEW.target_attribute
53+
) THEN
54+
RAISE EXCEPTION
55+
'Cannot insert specific mapping when a NULL catch-all exists for (source_id=% , attribute=%, target=%)',
56+
NEW.source_id, NEW.source_attribute, NEW.target_attribute;
57+
END IF;
58+
END IF;
59+
RETURN NEW;
60+
END;
61+
$$ LANGUAGE plpgsql;
62+
`);
63+
64+
await knex.raw(`
65+
DROP TRIGGER IF EXISTS "trigger_check_mutual_exclusivity";
66+
CREATE TRIGGER "trigger_check_mutual_exclusivity"
67+
BEFORE INSERT OR UPDATE ON "data_ingestion_mappings"
68+
FOR EACH ROW
69+
EXECUTE FUNCTION "check_unique_mutual_exclusivity"();
70+
`);
71+
} else if (client === "oracledb" || client === "oracle") {
72+
await knex.raw(`
73+
CREATE OR REPLACE TRIGGER "trigger_check_mutual_exclusivity"
74+
BEFORE INSERT OR UPDATE ON "data_ingestion_mappings"
75+
FOR EACH ROW
76+
DECLARE
77+
v_count NUMBER;
78+
BEGIN
79+
IF :NEW."source_value" IS NULL OR :NEW."source_value" = '' THEN
80+
SELECT COUNT(*) INTO v_count FROM "data_ingestion_mappings"
81+
WHERE "source_id" = :NEW."source_id"
82+
AND "source_attribute" = :NEW."source_attribute"
83+
AND "source_value" IS NOT NULL
84+
AND "target_attribute" = :NEW."target_attribute";
85+
IF v_count > 0 THEN
86+
RAISE_APPLICATION_ERROR(-20001,
87+
'Cannot insert NULL mapping when specific mappings exist for (source_id=' || :NEW.source_id || ', attribute=' || :NEW.source_attribute || ', target=' || :NEW.target_attribute || ')');
88+
END IF;
89+
ELSE
90+
SELECT COUNT(*) INTO v_count FROM "data_ingestion_mappings"
91+
WHERE "source_id" = :NEW."source_id"
92+
AND "source_attribute" = :NEW."source_attribute"
93+
AND ("source_value" IS NULL OR "source_value" = '')
94+
AND "target_attribute" = :NEW."target_attribute";
95+
IF v_count > 0 THEN
96+
RAISE_APPLICATION_ERROR(-20002,
97+
'Cannot insert specific mapping when a NULL catch-all exists for (source_id=' || :NEW.source_id || ', attribute=' || :NEW.source_attribute || ', target=' || :NEW.target_attribute || ')');
98+
END IF;
99+
END IF;
100+
END;
101+
`);
102+
} else {
103+
throw new Error("Unsupported database client for this migration.");
104+
}
105+
}

‎api/src/data/seeds/014.data-ingestion-sources.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
import knex from "knex";
2-
import { DataIngestionSource } from "../../data/models";
2+
import { DataIngestionSource } from "@/data/models";
33

44
export async function seed(knex: knex.Knex) {
55
const data_ingestion_sources = await knex<DataIngestionSource>("data_ingestion_sources");
66

7-
const toInsert = [
7+
const dataIngestionSourcesAttributes: DataIngestionSource[] = [
88
{
99
source_name: "RL6",
1010
description: "HSS's incident system",
@@ -27,15 +27,15 @@ export async function seed(knex: knex.Knex) {
2727
},
2828
] as Array<DataIngestionSource>;
2929

30-
for (const item of toInsert) {
30+
for (const dataIngestionSourcesAttribute of dataIngestionSourcesAttributes) {
3131
const existing = await knex("data_ingestion_sources")
32-
.where({ source_name: item.source_name })
32+
.where({ source_name: dataIngestionSourcesAttribute.source_name })
3333
.first();
3434

3535
if (existing) {
36-
await knex("data_ingestion_sources").where({ source_name: item.source_name }).update(item);
36+
await knex("data_ingestion_sources").where({ source_name: dataIngestionSourcesAttribute.source_name }).update(dataIngestionSourcesAttribute);
3737
} else {
38-
await knex("data_ingestion_sources").insert(item);
38+
await knex("data_ingestion_sources").insert(dataIngestionSourcesAttribute);
3939
}
4040
}
4141
}

‎api/src/data/seeds/015.data-ingestion-mappings-hss.ts‎

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
import knex from "knex";
2+
import { DataIngestionMapping } from "@/data/models";
23

34
export async function seed(knex: knex.Knex) {
4-
const mappings = [
5+
const dataIngestionMappingsAttributes: DataIngestionMapping[] = [
56
{
67
source_id: 1,
78
source_attribute: "File ID",
@@ -130,23 +131,27 @@ export async function seed(knex: knex.Knex) {
130131
},
131132
];
132133

133-
for (const row of mappings) {
134-
const { source_id, source_attribute, source_value, target_attribute, target_value } = row;
135-
const key = { source_id, source_attribute, target_attribute };
136-
if (row.source_value == null) {
137-
await knex("data_ingestion_mappings").where(key).whereNotNull("source_value").del();
138-
} else {
139-
await knex("data_ingestion_mappings").where(key).whereNull("source_value").del();
140-
}
134+
for (const dataIngestionMappingsAttribute of dataIngestionMappingsAttributes) {
135+
const { source_id, source_attribute, source_value, target_attribute, target_value } =
136+
dataIngestionMappingsAttribute;
137+
138+
const existingIds = await knex("data_ingestion_mappings")
139+
.where({ source_id, source_attribute, target_attribute })
140+
.andWhere((queryBuilder) => {
141+
if (source_value == null || source_value === "") {
142+
queryBuilder.whereNull("source_value");
143+
} else {
144+
queryBuilder.where("source_value", source_value);
145+
}
146+
})
147+
.select("id");
141148

142-
const exists = await knex("data_ingestion_mappings")
143-
.where({ ...key, source_value })
144-
.first();
149+
const ids = existingIds.map((existing) => existing.id);
145150

146-
if (exists) {
147-
await knex("data_ingestion_mappings").where({ id: exists.id }).update({ target_value });
151+
if (ids.length != 0) {
152+
await knex("data_ingestion_mappings").whereIn("id", ids).update({ target_value });
148153
} else {
149-
await knex("data_ingestion_mappings").insert(row);
154+
await knex("data_ingestion_mappings").insert(dataIngestionMappingsAttribute);
150155
}
151156
}
152157
}

‎api/src/data/seeds/016.data-ingestion-mappings-airport.ts‎

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
import knex from "knex";
2+
import { DataIngestionMapping } from "@/data/models";
23

34
export async function seed(knex: knex.Knex) {
4-
const mappings = [
5+
const dataIngestionMappingsAttributes: DataIngestionMapping[] = [
56
{
67
source_id: 2,
78
source_attribute: "ID",
@@ -95,23 +96,27 @@ export async function seed(knex: knex.Knex) {
9596
},
9697
];
9798

98-
for (const row of mappings) {
99-
const { source_id, source_attribute, source_value, target_attribute, target_value } = row;
100-
const key = { source_id, source_attribute, target_attribute };
101-
if (row.source_value == null) {
102-
await knex("data_ingestion_mappings").where(key).whereNotNull("source_value").del();
103-
} else {
104-
await knex("data_ingestion_mappings").where(key).whereNull("source_value").del();
105-
}
99+
for (const dataIngestionMappingsAttribute of dataIngestionMappingsAttributes) {
100+
const { source_id, source_attribute, source_value, target_attribute, target_value } =
101+
dataIngestionMappingsAttribute;
102+
103+
const existingIds = await knex("data_ingestion_mappings")
104+
.where({ source_id, source_attribute, target_attribute })
105+
.andWhere((queryBuilder) => {
106+
if (source_value == null || source_value === "") {
107+
queryBuilder.whereNull("source_value");
108+
} else {
109+
queryBuilder.where("source_value", source_value);
110+
}
111+
})
112+
.select("id");
106113

107-
const exists = await knex("data_ingestion_mappings")
108-
.where({ ...key, source_value })
109-
.first();
114+
const ids = existingIds.map((existing) => existing.id);
110115

111-
if (exists) {
112-
await knex("data_ingestion_mappings").where({ id: exists.id }).update({ target_value });
116+
if (ids.length != 0) {
117+
await knex("data_ingestion_mappings").whereIn("id", ids).update({ target_value });
113118
} else {
114-
await knex("data_ingestion_mappings").insert(row);
119+
await knex("data_ingestion_mappings").insert(dataIngestionMappingsAttribute);
115120
}
116121
}
117122
}

‎api/src/data/seeds/017.data-ingestion-mappings-hpw.ts‎

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
import knex from "knex";
2+
import { DataIngestionMapping } from "@/data/models";
23

34
export async function seed(knex: knex.Knex) {
4-
const mappings = [
5+
const dataIngestionMappingsAttributes: DataIngestionMapping[] = [
56
{
67
source_id: 3,
78
source_attribute: "Number",
@@ -74,23 +75,27 @@ export async function seed(knex: knex.Knex) {
7475
},
7576
];
7677

77-
for (const row of mappings) {
78-
const { source_id, source_attribute, source_value, target_attribute, target_value } = row;
79-
const key = { source_id, source_attribute, target_attribute };
80-
if (row.source_value == null) {
81-
await knex("data_ingestion_mappings").where(key).whereNotNull("source_value").del();
82-
} else {
83-
await knex("data_ingestion_mappings").where(key).whereNull("source_value").del();
84-
}
78+
for (const dataIngestionMappingsAttribute of dataIngestionMappingsAttributes) {
79+
const { source_id, source_attribute, source_value, target_attribute, target_value } =
80+
dataIngestionMappingsAttribute;
81+
82+
const existingIds = await knex("data_ingestion_mappings")
83+
.where({ source_id, source_attribute, target_attribute })
84+
.andWhere((queryBuilder) => {
85+
if (source_value == null || source_value === "") {
86+
queryBuilder.whereNull("source_value");
87+
} else {
88+
queryBuilder.where("source_value", source_value);
89+
}
90+
})
91+
.select("id");
8592

86-
const exists = await knex("data_ingestion_mappings")
87-
.where({ ...key, source_value })
88-
.first();
93+
const ids = existingIds.map((existing) => existing.id);
8994

90-
if (exists) {
91-
await knex("data_ingestion_mappings").where({ id: exists.id }).update({ target_value });
95+
if (ids.length != 0) {
96+
await knex("data_ingestion_mappings").whereIn("id", ids).update({ target_value });
9297
} else {
93-
await knex("data_ingestion_mappings").insert(row);
98+
await knex("data_ingestion_mappings").insert(dataIngestionMappingsAttribute);
9499
}
95100
}
96101
}

‎api/src/routes/data-ingestion-router.ts‎

Lines changed: 10 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,9 @@
11
import express, { Request, Response } from "express";
2+
import { isNil } from "lodash";
23

34
import { db } from "@/data/db-client";
45
import { RequireAdmin } from "../middleware";
5-
import {
6-
CreateService as DataIngestionCreateService,
7-
DataIngestionSourceService,
8-
UserService,
9-
} from "@/services";
10-
import { isNil } from "lodash";
6+
import { CreateService as DataIngestionCreateService } from "@/services";
117

128
export const dataIngestionRouter = express.Router();
139

@@ -17,30 +13,20 @@ dataIngestionRouter.get("/", async (req: Request, res: Response) => {
1713
});
1814

1915
dataIngestionRouter.post("/", async (req: Request, res: Response) => {
20-
const users = new UserService();
21-
const dataIngestionSource = new DataIngestionSourceService();
22-
const { source_id, user_id } = req.body;
23-
24-
if (isNil(req.files?.csvFile)) return res.status(400).json({ error: "Missing file" });
25-
if (isNil(user_id)) return res.status(422).json({ error: "Missing user_id" });
26-
if (isNil(source_id)) return res.status(422).json({ error: "Missing source_id" });
27-
28-
if (isNil(users.getById(Number(user_id)))) {
29-
return res.status(422).json({ error: "Invalid user_id" });
16+
const { sourceId, userId } = req.body;
17+
const uploaded = Array.isArray(req.files?.csvFile) ? req.files!.csvFile[0] : req.files?.csvFile;
18+
if (isNil(uploaded)) {
19+
return res.status(422).json({ error: "No file uploaded" });
3020
}
3121

32-
if (isNil(dataIngestionSource.getById(Number(source_id)))) {
33-
return res.status(422).json({ error: "Invalid source_id" });
34-
}
35-
36-
const uploaded = Array.isArray(req.files.csvFile) ? req.files.csvFile[0] : req.files.csvFile;
37-
3822
try {
39-
await DataIngestionCreateService.perform(uploaded.data, Number(source_id), Number(user_id));
23+
await DataIngestionCreateService.perform(uploaded.data, Number(sourceId), Number(userId));
4024
return res.json({ success: true });
4125
} catch (err: any) {
4226
console.error(" DataIngestionService Error:", err);
43-
return res.status(500).json({ error: err.message });
27+
return res.status(422).json({
28+
message: `Assessment deletion failed: ${err}`,
29+
});
4430
}
4531
});
4632

0 commit comments

Comments
 (0)