Merge pull request #263 from commonknowledge/fix/migrantdemo-fixes

fix: add import data test and ignore spaces in numeric data

Author: joaquimds

src/server/jobs/importDataSource.ts

@@ -207,10 +207,10 @@ const parseNumber = (value: unknown): number => {
 };
 
 /**
- * Replace terminating % and commas
+ * Replace terminating % and all commas and spaces
  */
 const cleanNumber = (value: string): string => {
-  return value.trim().replace(/%$/, "").replace(/,/g, "");
+  return value.trim().replace(/%$/, "").replace(/,/g, "").replace(/ /g, "");
 };
 
 const addColumnDefs = (

tests/feature/importDataSource.test.ts

@@ -1,20 +1,25 @@
-import { describe, expect, test } from "vitest";
+import { afterAll, describe, expect, test } from "vitest";
 import importDataSource from "@/server/jobs/importDataSource";
 import { AreaSetCode } from "@/server/models/AreaSet";
 import {
+  ColumnType,
   DataSourceRecordType,
   DataSourceType,
   GeocodingType,
 } from "@/server/models/DataSource";
-import { FilterType } from "@/server/models/MapView";
+import { CalculationType, FilterType } from "@/server/models/MapView";
 import { streamDataRecordsByDataSource } from "@/server/repositories/DataRecord";
 import {
   createDataSource,
   deleteDataSource,
+  findDataSourceById,
 } from "@/server/repositories/DataSource";
 import { upsertOrganisation } from "@/server/repositories/Organisation";
+import { getAreaStats } from "@/server/stats";
 
 describe("importDataSource tests", () => {
+  const toRemove: string[] = [];
+
   test("importDataSource imports records from CSV", async () => {
     // 1. Create test organisation
     const org = await upsertOrganisation({
@@ -26,7 +31,7 @@ describe("importDataSource tests", () => {
       name: "Test Import CSV Source",
       autoEnrich: false,
       autoImport: false,
-      recordType: DataSourceRecordType.Data,
+      recordType: DataSourceRecordType.Members,
       config: {
         type: DataSourceType.CSV,
         url: "file://tests/resources/members.csv",
@@ -43,6 +48,8 @@ describe("importDataSource tests", () => {
       public: false,
     });
 
+    toRemove.push(dataSource.id);
+
     // 3. Call importDataSource
     await importDataSource({ dataSourceId: dataSource.id });
 
@@ -126,4 +133,86 @@ describe("importDataSource tests", () => {
       },
     ]);
   });
+
+  test("importDataSource imports stats from CSV", async () => {
+    // 1. Create test organisation
+    const org = await upsertOrganisation({
+      name: "Test Import Org",
+    });
+
+    // 2. Create test data source with CSV file
+    const dataSource = await createDataSource({
+      name: "Test Import Stats CSV Source",
+      autoEnrich: false,
+      autoImport: false,
+      recordType: DataSourceRecordType.Data,
+      config: {
+        type: DataSourceType.CSV,
+        url: "file://tests/resources/stats.csv",
+      },
+      columnDefs: [],
+      columnRoles: { nameColumns: [] },
+      enrichments: [],
+      geocodingConfig: {
+        type: GeocodingType.Code,
+        column: "Code",
+        areaSetCode: AreaSetCode.WMC24,
+      },
+      organisationId: org.id,
+      public: false,
+    });
+
+    toRemove.push(dataSource.id);
+
+    // 3. Call importDataSource
+    await importDataSource({ dataSourceId: dataSource.id });
+
+    const importedDataSource = await findDataSourceById(dataSource.id);
+    const columnDefs = importedDataSource?.columnDefs;
+    columnDefs?.sort((a, b) => (a.name < b.name ? -1 : 1));
+
+    expect(columnDefs).toEqual([
+      { name: "Code", type: ColumnType.String },
+      { name: "Electorate", type: ColumnType.Number },
+      { name: "Name", type: ColumnType.String },
+    ]);
+
+    const areaStats = await getAreaStats({
+      areaSetCode: AreaSetCode.WMC24,
+      dataSourceId: dataSource.id,
+      calculationType: CalculationType.Sum,
+      column: "Electorate",
+      excludeColumns: [],
+    });
+
+    const stats = areaStats.primary?.stats;
+    stats?.sort((a, b) => (a.areaCode < b.areaCode ? -1 : 1));
+
+    // Remove stats array from main equality check
+    areaStats.primary = undefined;
+
+    expect(areaStats).toEqual({
+      areaSetCode: AreaSetCode.WMC24,
+      calculationType: CalculationType.Sum,
+      dataSourceId: dataSource.id,
+      primary: undefined,
+    });
+
+    expect(stats).toEqual([
+      {
+        areaCode: "E14001070",
+        value: 71002,
+      },
+      {
+        areaCode: "E14001071",
+        value: 79169,
+      },
+    ]);
+  });
+
+  afterAll(async () => {
+    for (const id of toRemove) {
+      await deleteDataSource(id);
+    }
+  });
 }, 10000);