Merge pull request #262 from commonknowledge/fix/migrantdemo-fixes

fix: minor bugs seen in the migrantdemo onboarding

Author: joaquimds

src/app/(private)/data-sources/[id]/components/ConfigurationForm.tsx

@@ -155,7 +155,7 @@ export default function ConfigurationForm({
         variant="secondary"
         type="submit"
       >
-        Save and import
+        {loading ? "Saving (this can take a minute...)" : "Save and import"}
       </Button>
 
       {error && <p className="text-xs text-red-500">{error}</p>}

src/app/map/[id]/colors.ts

@@ -246,11 +246,7 @@ export const useFillColor = ({
       .fill(null)
       .flatMap((_, i) => {
         const step = stepScale(i);
-        const color = isReversed
-          ? colorScheme.colorScale(
-              colorScheme.maxValue - (step - colorScheme.minValue),
-            )
-          : colorScheme.colorScale(step);
+        const color = colorScheme.colorScale(step);
         return [step, color];
       });
     return [

src/app/map/[id]/components/Markers.tsx

@@ -14,19 +14,19 @@ import type { FeatureCollection } from "geojson";
 
 const MARKER_CLIENT_EXCLUDED_KEY = "__clientExcluded";
 
-function hexToRgb(hex: string) {
-  const normalized = hex.replace("#", "");
-  const bigint = parseInt(normalized, 16);
-  const r = (bigint >> 16) & 255;
-  const g = (bigint >> 8) & 255;
-  const b = bigint & 255;
-  return { r, g, b };
-}
+// function hexToRgb(hex: string) {
+//   const normalized = hex.replace("#", "");
+//   const bigint = parseInt(normalized, 16);
+//   const r = (bigint >> 16) & 255;
+//   const g = (bigint >> 8) & 255;
+//   const b = bigint & 255;
+//   return { r, g, b };
+// }
 
-function rgbaString(hex: string, alpha: number) {
-  const { r, g, b } = hexToRgb(hex);
-  return `rgba(${r}, ${g}, ${b}, ${alpha})`;
-}
+// function rgbaString(hex: string, alpha: number) {
+//   const { r, g, b } = hexToRgb(hex);
+//   return `rgba(${r}, ${g}, ${b}, ${alpha})`;
+// }
 
 export default function Markers() {
   const { viewConfig } = useMapViews();
@@ -154,57 +154,53 @@ function DataSourceMarkers({
         ],
       }}
     >
-      {publicMap ? (
-        [
-          <Layer
-            id={`${sourceId}-circles`}
-            key={`${sourceId}-circles`}
-            type="circle"
-            source={sourceId}
-            filter={["has", "point_count"]}
-            paint={{
-              // Circle radius based on point_count
-              "circle-radius": [
-                "interpolate",
-                ["linear"],
-                ["get", "point_count"],
-                1,
-                15,
-                10,
-                25,
-                100,
-                35,
-                1000,
-                50,
-                10000,
-                70,
-              ],
-              // Circle color
-              "circle-color": color,
-              // Opacity based on matched_count
-              "circle-opacity": [
-                "case",
-                ["==", ["get", "matched_count"], 0],
-                0.5,
-                0.8,
-              ],
-            }}
-          />,
-
-          <Layer
-            id={`${sourceId}-counts`}
-            key={`${sourceId}-counts`}
-            type="symbol"
-            source={sourceId}
-            filter={["has", "point_count"]}
-            layout={{
-              "text-field": ["get", "point_count"],
-              "text-font": ["DIN Pro Medium", "Arial Unicode MS Bold"],
-              "text-size": 12,
-            }}
-          />,
-        ]
-      ) : (
+      <Layer
+        id={`${sourceId}-circles`}
+        key={`${sourceId}-circles`}
+        type="circle"
+        source={sourceId}
+        filter={["has", "point_count"]}
+        paint={{
+          // Circle radius based on point_count
+          "circle-radius": [
+            "interpolate",
+            ["linear"],
+            ["get", "point_count"],
+            1,
+            15,
+            10,
+            25,
+            100,
+            35,
+            1000,
+            50,
+            10000,
+            70,
+          ],
+          // Circle color
+          "circle-color": color,
+          // Opacity based on matched_count
+          "circle-opacity": [
+            "case",
+            ["==", ["get", "matched_count"], 0],
+            0.5,
+            0.8,
+          ],
+        }}
+      />
+      <Layer
+        id={`${sourceId}-counts`}
+        key={`${sourceId}-counts`}
+        type="symbol"
+        source={sourceId}
+        filter={["has", "point_count"]}
+        layout={{
+          "text-field": ["get", "point_count"],
+          "text-font": ["DIN Pro Medium", "Arial Unicode MS Bold"],
+          "text-size": 12,
+        }}
+      />
+      {/* TODO: Restore this with a switch
         <Layer
           id={`${sourceId}-heatmap`}
           type="heatmap"
@@ -273,8 +269,7 @@ function DataSourceMarkers({
             ],
             "heatmap-opacity": 0.7,
           }}
-        />
-      )}
+        /> */}
       <Layer
         id={`${sourceId}-pins`}
         type="circle"

src/server/jobs/importDataSource.ts

@@ -182,7 +182,7 @@ const getType = (value: unknown): ColumnType => {
   }
 
   if (typeof value === "string") {
-    const trimmedValue = value.trim().replace(/%$/, "");
+    const trimmedValue = cleanNumber(value);
     if (/^[-+]?(\d+\.?\d*|\.\d+)([eE][-+]?\d+)?$/.test(trimmedValue)) {
       return ColumnType.Number;
     }
@@ -203,7 +203,14 @@ const parseNumber = (value: unknown): number => {
   if (typeof value !== "string") {
     return Number(value);
   }
-  return Number(value.replace(/%$/, ""));
+  return Number(cleanNumber(value));
+};
+
+/**
+ * Replace terminating % and commas
+ */
+const cleanNumber = (value: string): string => {
+  return value.trim().replace(/%$/, "").replace(/,/g, "");
 };
 
 const addColumnDefs = (

src/server/trpc/routers/dataSource.ts

@@ -53,7 +53,7 @@ export const dataSourceRouter = router({
         return eb.or(filter);
       })
       .selectAll("dataSource")
-      .select(db.fn.count("dataRecord.id").as("recordCount"))
+      .select(db.fn.count("dataRecord.id").distinct().as("recordCount"))
       .groupBy("dataSource.id")
       .execute();
 
@@ -65,31 +65,20 @@ export const dataSourceRouter = router({
       .leftJoin("dataRecord", "dataRecord.dataSourceId", "dataSource.id")
       .where("organisationId", "=", ctx.organisation.id)
       .selectAll("dataSource")
-      .select(db.fn.count("dataRecord.id").as("recordCount"))
+      .select(db.fn.count("dataRecord.id").distinct().as("recordCount"))
       .groupBy("dataSource.id")
       .execute();
 
     return addImportInfo(dataSources);
   }),
   byId: dataSourceOwnerProcedure.query(async ({ ctx }) => {
-    const dataSource = await db
-      .selectFrom("dataSource")
-      .leftJoin("dataRecord", "dataRecord.dataSourceId", "dataSource.id")
-      .where("organisationId", "=", ctx.organisation.id)
-      .where("dataSource.id", "=", ctx.dataSource.id)
-      .selectAll("dataSource")
-      .select(db.fn.count("dataRecord.id").as("recordCount"))
-      .groupBy("dataSource.id")
+    const recordCount = await db
+      .selectFrom("dataRecord")
+      .where("dataSourceId", "=", ctx.dataSource.id)
+      .select(db.fn.countAll().as("count"))
       .executeTakeFirst();
 
-    if (!dataSource) {
-      throw new TRPCError({
-        code: "NOT_FOUND",
-        message: "Data source not found",
-      });
-    }
-
-    const dataSourceIds = dataSource.enrichments
+    const dataSourceIds = ctx.dataSource.enrichments
       .filter((e) => e.sourceType === EnrichmentSourceType.DataSource)
       .map((e) => e.dataSourceId)
       .filter((id) => typeof id === "string");
@@ -103,15 +92,15 @@ export const dataSourceRouter = router({
         ),
       ]);
     return {
-      ...dataSource,
+      ...ctx.dataSource,
       config: {
-        ...dataSource.config,
+        ...ctx.dataSource.config,
         __SERIALIZE_CREDENTIALS: true,
       },
       enrichmentInfo,
       importInfo,
       enrichmentDataSources,
-      recordCount: Number(dataSource.recordCount) || 0,
+      recordCount: Number(recordCount?.count) || 0,
     };
   }),