feat: strongly typed csv parsing (#52)

Co-authored-by: wujekbogdan <[email protected]>

Author: dsonyy

.changeset/lucky-kings-mix.md

@@ -0,0 +1,6 @@
+---
+"map_of_science": minor
+---
+
+- Introduce strongly-typed `keys.tsv`, `data.tsv`, and `labels.tsv"` files parsing.
+- Refactor `points.js` to use th newly introduced parser

src/csv/collector.spec.ts

@@ -0,0 +1,44 @@
+import { describe, it, expect } from "vitest";
+import { arrayCollector, mapCollector, setCollector } from "./collector.ts";
+
+describe("collectors", () => {
+  describe("arrayCollector", () => {
+    it("should collect items in an array", () => {
+      const collector = arrayCollector();
+      collector.add("Alice");
+      collector.add("Bob");
+
+      expect(collector.getResults()).toEqual(["Alice", "Bob"]);
+    });
+  });
+
+  describe("setCollector", () => {
+    it("should collect unique items in a set", () => {
+      const collector = setCollector();
+      collector.add("Alice");
+      collector.add("Bob");
+      collector.add("Alice"); // Duplicate
+
+      expect(collector.getResults()).toEqual(new Set(["Alice", "Bob"]));
+    });
+  });
+
+  describe("mapCollector", () => {
+    it("should collect items in a map using the provided key function", () => {
+      const collector = mapCollector<string, { id: string; name: string }>(
+        (item) => item.id,
+      );
+
+      collector.add({ id: "1", name: "Alice" });
+      collector.add({ id: "2", name: "Bob" });
+      collector.add({ id: "1", name: "Charlie" }); // Overwrites "Alice"
+
+      expect(collector.getResults()).toEqual(
+        new Map([
+          ["1", { id: "1", name: "Charlie" }], // Last value wins
+          ["2", { id: "2", name: "Bob" }],
+        ]),
+      );
+    });
+  });
+});

src/csv/collector.ts

@@ -0,0 +1,33 @@
+import { Collector } from "./parse.ts";
+
+export const arrayCollector = <T>(): Collector<T, T[]> => {
+  const collection: T[] = [];
+  return {
+    add: (item) => {
+      collection.push(item);
+      return collection;
+    },
+    getResults: () => collection,
+  };
+};
+
+export const setCollector = <T>(): Collector<T, Set<T>> => {
+  const collection = new Set<T>();
+  return {
+    add: (item) => {
+      collection.add(item);
+      return collection;
+    },
+    getResults: () => collection,
+  };
+};
+
+export const mapCollector = <K, V>(
+  getKey: (item: V) => K,
+): Collector<V, Map<K, V>> => {
+  const collection = new Map<K, V>();
+  return {
+    add: (item) => collection.set(getKey(item), item),
+    getResults: () => collection,
+  };
+};

src/csv/csv.spec.ts

@@ -1,118 +1,60 @@
 import { describe, it, expect, vi } from "vitest";
-import { parse, parseFromUrl } from "./parse";
+import { parse, withHttpProvider } from "./parse";
 import { withRequestInterception } from "../test-utils/request-interception.ts";
 
 const CSV = "name\tage\nAlice\t30\nBob\t40";
 
 describe("csv", () => {
   describe("parse", () => {
-    it("should parse CSV", async () => {
-      const fetcher = vi.fn(() => CSV);
-      const asyncFetcher = vi.fn(() => Promise.resolve(CSV));
-      const transformer = (row: unknown) => row;
+    it("should parse CSV with non-async provider", async () => {
+      const provider = vi.fn(() => CSV);
+      const onItem = vi.fn();
 
-      const resultNonAsync = await parse(fetcher, transformer);
-      const resultAsync = await parse(asyncFetcher, transformer);
-      const expected = new Set([
-        { name: "Alice", age: "30" },
-        { name: "Bob", age: "40" },
-      ]);
+      await parse(provider, onItem);
 
-      expect(fetcher).toHaveBeenCalledOnce();
-      expect(asyncFetcher).toHaveBeenCalledOnce();
-      expect(resultNonAsync).toEqual(expected);
-      expect(resultAsync).toEqual(expected);
+      expect(onItem).toHaveBeenCalledTimes(2);
+      expect(onItem).toHaveBeenNthCalledWith(1, { name: "Alice", age: "30" });
+      expect(onItem).toHaveBeenNthCalledWith(2, { name: "Bob", age: "40" });
     });
 
-    it("should transform rows", async () => {
-      const fetcher = () => CSV;
+    it("should parse CSV with async provider", async () => {
+      const provider = vi.fn(() => Promise.resolve(CSV));
+      const onItem = vi.fn();
 
-      const result = await parse(fetcher, (row) => ({
-        name: `transformed ${row.name}`,
-        age: parseInt(row.age),
-      }));
-      const expected = new Set([
-        { name: "transformed Alice", age: 30 },
-        { name: "transformed Bob", age: 40 },
-      ]);
+      await parse(provider, onItem);
 
-      expect(result).toEqual(expected);
+      expect(onItem).toHaveBeenCalledTimes(2);
+      expect(onItem).toHaveBeenNthCalledWith(1, { name: "Alice", age: "30" });
+      expect(onItem).toHaveBeenNthCalledWith(2, { name: "Bob", age: "40" });
     });
 
-    it("should fail to parse CSV if transformer fails", async () => {
-      const transformer = () => {
-        throw new Error("Failed to transform");
+    it("should fail to parse CSV if onItem fails", async () => {
+      const provider = vi.fn(() => Promise.resolve(CSV));
+      const onItem = () => {
+        throw new Error("onItem failed");
       };
 
-      return expect(parse(() => CSV, transformer)).rejects.toThrow(
-        "Failed to transform",
-      );
+      return expect(parse(provider, onItem)).rejects.toThrow("onItem failed");
     });
   });
 
-  describe("parseFromUrl", () => {
+  describe("withHttpProvider", () => {
     it(
-      "should parse CSV from URL",
+      "should fetch CSV from URL and process each row",
       withRequestInterception(
         ({ http, HttpResponse }) => [
-          http.get("https://example.com/csv", () => {
-            return HttpResponse.text(CSV);
-          }),
+          http.get("https://example.com/csv", () => HttpResponse.text(CSV)),
         ],
         async () => {
-          const result = await parseFromUrl(
-            "https://example.com/csv",
-            (row: unknown) => row,
-          );
-
-          expect(result).toEqual(
-            new Set([
-              { name: "Alice", age: "30" },
-              { name: "Bob", age: "40" },
-            ]),
-          );
-        },
-      ),
-    );
-
-    it(
-      "should fail if request fails",
-      withRequestInterception(
-        ({ http, HttpResponse }) => [
-          http.get("https://example.com/csv", () => {
-            return HttpResponse.error();
-          }),
-        ],
-        async () => {
-          try {
-            await parseFromUrl("https://example.com/csv", (row) => row);
-          } catch (error) {
-            expect(error).toEqual(new Error("Failed to fetch"));
-          }
-          expect.hasAssertions();
-        },
-      ),
-    );
-
-    it(
-      "should fail if request doesn't fail, but response is not 200",
-      withRequestInterception(
-        ({ http, HttpResponse }) => [
-          http.get("https://example.com/csv", () => {
-            return new HttpResponse(null, {
-              status: 404,
-            });
-          }),
-        ],
-        async () => {
-          try {
-            await parseFromUrl("https://example.com/csv", (row) => row);
-          } catch (error) {
-            expect(error).toEqual(
-              new Error("Failed to fetch csv from: https://example.com/csv"),
-            );
-          }
-          expect.hasAssertions();
+          const onItem = vi.fn();
+          await withHttpProvider("https://example.com/csv", onItem);
+
+          expect(onItem).toHaveBeenCalledTimes(2);
+          expect(onItem).toHaveBeenNthCalledWith(1, {
+            name: "Alice",
+            age: "30",
+          });
+          expect(onItem).toHaveBeenNthCalledWith(2, { name: "Bob", age: "40" });
         },
       ),
     );

src/csv/parse.ts

@@ -1,33 +1,36 @@
 import { parse as csvParse } from "csv-parse/browser/esm";
+import { ZodTypeDef, ZodSchema } from "zod";
 
-type Fetcher = () => Promise<string> | string;
-type Transformer<T> = (data: Record<string, string>) => T;
-
-export const parse = async <T>(
-  fetch: Fetcher,
-  transform: Transformer<T>,
-): Promise<Set<T>> => {
-  const csv = await fetch();
+type CsvSource = string | Buffer;
+type Provider = () => Promise<CsvSource> | CsvSource;
+type OnItem<T, R> = (item: T) => R;
+type CSVRecord = Record<string, string>;
+export type Collector<T, R> = {
+  add: (item: T) => R;
+  getResults: () => R;
+};
 
-  console.time("parse");
+export const parse = async <T extends CSVRecord, R>(
+  providerCsv: Provider,
+  onItem: OnItem<T, R>,
+): Promise<void> => {
+  const csv = await providerCsv();
 
   return new Promise((resolve, reject) => {
-    const result = new Set<T>();
     const parser = csvParse({
       delimiter: "\t",
       columns: true,
       bom: true,
     });
 
     const onReadable = () => {
-      const record = parser.read() as Record<string, string> | null;
+      const record = parser.read() as T | null;
 
       if (record === null) {
         return;
       }
 
-      const transformed = transform(record);
-      result.add(transformed);
+      onItem(record);
       onReadable();
     };
 
@@ -36,26 +39,40 @@ export const parse = async <T>(
         reject(error);
       })
       .on("readable", onReadable)
-      .on("end", () => {
-        console.timeEnd("parse");
-        resolve(result);
-      });
+      .on("end", resolve);
 
     stream.write(csv);
     stream.end();
   });
 };
 
-export const parseFromUrl = <T>(url: string, transform: Transformer<T>) => {
-  const fetcher = async () => {
-    const response = await fetch(url);
+const httpProvider = async (url: string) => {
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch CSV from: ${url}`);
+  }
+  return response.text();
+};
 
-    if (!response.ok) {
-      throw new Error(`Failed to fetch csv from: ${url}`);
-    }
+export const withHttpProvider = async <T extends CSVRecord, R>(
+  url: string,
+  onItem: OnItem<T, R>,
+) => parse(() => httpProvider(url), onItem);
 
-    return response.text();
-  };
+export const validateWithSchema =
+  <T>(schema: ZodSchema<T, ZodTypeDef, unknown>) =>
+  (data: unknown): T =>
+    schema.parse(data);
 
-  return parse(fetcher, transform);
+export const createProcessor = <T, R>(
+  schema: ZodSchema<T, ZodTypeDef, unknown>,
+  collector: Collector<T, R>,
+) => {
+  return {
+    process: (data: unknown) => {
+      const parsed = validateWithSchema(schema)(data);
+      collector.add(parsed);
+    },
+    getResults: () => collector.getResults(),
+  };
 };

src/js/main.js

@@ -17,6 +17,5 @@ export const init = () => {
   }
 
   enableLoadingScreen();
-  points.loadConcepts();
-  points.loadDataPoints();
+  points.load();
 };