perf: offload all data processing to web workers with filter-stage caching and debounced compute

tom-draper · tom-draper · commit 5ebf1ad14032 · 2026-03-10T09:13:51.000Z
diff --git a/dashboard/lib/analytics.worker.ts b/dashboard/lib/analytics.worker.ts
@@ -63,6 +63,14 @@ let logs: NginxLog[] = [];
 let logsWithoutBots: NginxLog[] = [];
 let cachedBaseKey = '';
 let cachedFilteredData: NginxLog[] = [];
+let cachedVersionKey = '';
+let cachedVersionFilteredData: NginxLog[] = [];
+let cachedDeviceKey = '';
+let cachedDeviceFilteredData: NginxLog[] = [];
+let cachedHourKey = '';
+let cachedHourFilteredData: NginxLog[] = [];
+let cachedDayKey = '';
+let cachedDayFilteredData: NginxLog[] = [];
 
 function getDayId(date: Date): number {
     return new Date(date).setHours(0, 0, 0, 0);
@@ -118,7 +126,11 @@ self.onmessage = (e: MessageEvent<ComputeMessage | ParseAndStoreMessage>) => {
         const parsed = parseNginxLogs(data.rawLogs, data.logFormat);
         logs = [...logs, ...parsed];
         logsWithoutBots = [...logsWithoutBots, ...parsed.filter(row => !isBotOrCrawler(row.userAgent))];
-        cachedBaseKey = ''; // invalidate cache when logs change
+        cachedBaseKey = '';
+        cachedVersionKey = '';
+        cachedDeviceKey = '';
+        cachedHourKey = '';
+        cachedDayKey = '';
         return;
     }
 
@@ -167,26 +179,59 @@ self.onmessage = (e: MessageEvent<ComputeMessage | ParseAndStoreMessage>) => {
         cachedBaseKey = baseKey;
     }
 
-    // Version filter
-    const versionFilteredData = filter.version === null
-        ? filteredData
-        : filteredData.filter(row => getVersion(row.path) === filter.version);
-
-    // Device filter
-    let deviceFilteredData = versionFilteredData;
-    if (filter.client !== null) deviceFilteredData = deviceFilteredData.filter(row => getClient(row.userAgent) === filter.client);
-    if (filter.os !== null) deviceFilteredData = deviceFilteredData.filter(row => getOS(row.userAgent) === filter.os);
-    if (filter.deviceType !== null) deviceFilteredData = deviceFilteredData.filter(row => getDevice(row.userAgent) === filter.deviceType);
-
-    // Hour filter
-    const hourFilteredData = filter.hour === null
-        ? deviceFilteredData
-        : deviceFilteredData.filter(row => row.timestamp?.getHours() === filter.hour);
-
-    // Day filter
-    const dayFilteredData = filter.dayOfWeek === null
-        ? hourFilteredData
-        : hourFilteredData.filter(row => row.timestamp?.getDay() === filter.dayOfWeek);
+    // Version filter — cached
+    const versionKey = `${baseKey}|${filter.version ?? ''}`;
+    let versionFilteredData: NginxLog[];
+    if (versionKey === cachedVersionKey) {
+        versionFilteredData = cachedVersionFilteredData;
+    } else {
+        versionFilteredData = filter.version === null
+            ? filteredData
+            : filteredData.filter(row => getVersion(row.path) === filter.version);
+        cachedVersionFilteredData = versionFilteredData;
+        cachedVersionKey = versionKey;
+    }
+
+    // Device filter — cached
+    const deviceKey = `${versionKey}|${filter.client ?? ''}|${filter.os ?? ''}|${filter.deviceType ?? ''}`;
+    let deviceFilteredData: NginxLog[];
+    if (deviceKey === cachedDeviceKey) {
+        deviceFilteredData = cachedDeviceFilteredData;
+    } else {
+        let result = versionFilteredData;
+        if (filter.client !== null) result = result.filter(row => getClient(row.userAgent) === filter.client);
+        if (filter.os !== null) result = result.filter(row => getOS(row.userAgent) === filter.os);
+        if (filter.deviceType !== null) result = result.filter(row => getDevice(row.userAgent) === filter.deviceType);
+        deviceFilteredData = result;
+        cachedDeviceFilteredData = result;
+        cachedDeviceKey = deviceKey;
+    }
+
+    // Hour filter — cached
+    const hourKey = `${deviceKey}|${filter.hour ?? ''}`;
+    let hourFilteredData: NginxLog[];
+    if (hourKey === cachedHourKey) {
+        hourFilteredData = cachedHourFilteredData;
+    } else {
+        hourFilteredData = filter.hour === null
+            ? deviceFilteredData
+            : deviceFilteredData.filter(row => row.timestamp?.getHours() === filter.hour);
+        cachedHourFilteredData = hourFilteredData;
+        cachedHourKey = hourKey;
+    }
+
+    // Day filter — cached
+    const dayKey = `${hourKey}|${filter.dayOfWeek ?? ''}`;
+    let dayFilteredData: NginxLog[];
+    if (dayKey === cachedDayKey) {
+        dayFilteredData = cachedDayFilteredData;
+    } else {
+        dayFilteredData = filter.dayOfWeek === null
+            ? hourFilteredData
+            : hourFilteredData.filter(row => row.timestamp?.getDay() === filter.dayOfWeek);
+        cachedDayFilteredData = dayFilteredData;
+        cachedDayKey = dayKey;
+    }
 
     // Aggregations
     const endpointCounts = new Map<string, number>();
diff --git a/dashboard/lib/components/dashboard.tsx b/dashboard/lib/components/dashboard.tsx
@@ -10,9 +10,9 @@ import Users from "@/lib/components/users";
 import { Version, getVersion } from "@/lib/components/version";
 import { Location } from "@/lib/components/location";
 import { type Location as LocationType } from "@/lib/location"
-import { parseNginxLogs } from "@/lib/parse";
 import { useCallback, useEffect, useMemo, useRef, useState, useTransition } from "react";
 import type { WorkerResult } from '@/lib/analytics.worker';
+import type { ParseWorkerResult } from '@/lib/parse.worker';
 import { Device } from "@/lib/components/device/device";
 import { type Filter, newFilter } from "@/lib/filter";
 import { Period, periodStart } from "@/lib/period";
@@ -32,13 +32,9 @@ import dynamic from "next/dynamic";
 const NetworkBackground = dynamic(() => import("./network-background"), { ssr: false });
 const FileUpload = dynamic(() => import("./file-upload"));
 
-const PARSE_CHUNK_SIZE = 5000;
-
 export default function Dashboard({ fileUpload, demo, logFormat }: { fileUpload: boolean, demo: boolean, logFormat?: string }) {
     const [accessLogs, setAccessLogs] = useState<string[]>([]);
     const [logs, setLogs] = useState<NginxLog[]>([]);
-    const parsedAccessCount = useRef(0);
-    const parseCancelRef = useRef(false);
 
     const [errorLogs, setErrorLogs] = useState<string[]>([]);
 
@@ -96,13 +92,15 @@ export default function Dashboard({ fileUpload, demo, logFormat }: { fileUpload:
         startTransition(() => setFilter((previous) => ({ ...previous, dayOfWeek })))
     }, [])
 
-    // Worker ref and result state
+    // Worker refs and result state
     const workerRef = useRef<Worker | null>(null);
     const workerSeqRef = useRef(0);
     const workerRawCountRef = useRef(0);
+    const computeDebounceRef = useRef<ReturnType<typeof setTimeout> | null>(null);
+    const parseWorkerRef = useRef<Worker | null>(null);
     const [workerResult, setWorkerResult] = useState<WorkerResult | null>(null);
 
-    // Create worker once on mount
+    // Create workers once on mount
     useEffect(() => {
         const worker = new Worker(new URL('../analytics.worker.ts', import.meta.url));
         workerRef.current = worker;
@@ -111,25 +109,49 @@ export default function Dashboard({ fileUpload, demo, logFormat }: { fileUpload:
                 setWorkerResult(e.data);
             }
         };
-        return () => worker.terminate();
+
+        const parseWorker = new Worker(new URL('../parse.worker.ts', import.meta.url));
+        parseWorkerRef.current = parseWorker;
+        parseWorker.onmessage = (e: MessageEvent<ParseWorkerResult>) => {
+            const { logs: newLogs, maxTimestamp, isFirstBatch } = e.data;
+            if (newLogs.length === 0) return;
+            if (isFirstBatch && maxTimestamp !== null) {
+                const maxDate = new Date(maxTimestamp);
+                if (inPeriod(maxDate, 'week')) setPeriod('week');
+                else if (inPeriod(maxDate, 'month')) setPeriod('month');
+                else if (inPeriod(maxDate, '6 months')) setPeriod('6 months');
+                else setPeriod('all time');
+            }
+            setLogs(prev => [...prev, ...newLogs]);
+        };
+
+        return () => {
+            worker.terminate();
+            parseWorker.terminate();
+        };
     }, []);
 
-    // Send raw logs to worker for parsing (fires before compute effect)
+    // Send raw logs to both workers for parsing (fires before compute effect)
     useEffect(() => {
-        if (!workerRef.current || accessLogs.length <= workerRawCountRef.current) return;
+        if (accessLogs.length <= workerRawCountRef.current) return;
         const newRawLogs = accessLogs.slice(workerRawCountRef.current);
+        const isFirstBatch = workerRawCountRef.current === 0;
         workerRawCountRef.current = accessLogs.length;
-        workerRef.current.postMessage({ type: 'parseAndStore', rawLogs: newRawLogs, logFormat });
+        workerRef.current?.postMessage({ type: 'parseAndStore', rawLogs: newRawLogs, logFormat });
+        parseWorkerRef.current?.postMessage({ rawLogs: newRawLogs, logFormat, isFirstBatch });
     }, [accessLogs]);
 
-    // Trigger computation when inputs change
+    // Trigger computation when inputs change — debounced to avoid redundant work on rapid changes
     useEffect(() => {
         if (!workerRef.current) return;
-        const seq = ++workerSeqRef.current;
-        const locationMapEntries: [string, string][] = filter.location !== null
-            ? Array.from(locationMap.entries()).map(([ip, loc]) => [ip, loc.country])
-            : [];
-        workerRef.current.postMessage({ type: 'compute', seq, filter, settings, locationMap: locationMapEntries });
+        if (computeDebounceRef.current) clearTimeout(computeDebounceRef.current);
+        computeDebounceRef.current = setTimeout(() => {
+            const seq = ++workerSeqRef.current;
+            const locationMapEntries: [string, string][] = filter.location !== null
+                ? Array.from(locationMap.entries()).map(([ip, loc]) => [ip, loc.country])
+                : [];
+            workerRef.current?.postMessage({ type: 'compute', seq, filter, settings, locationMap: locationMapEntries });
+        }, 50);
     }, [accessLogs, filter, settings, locationMap]);
 
     useEffect(() => {
@@ -196,63 +218,6 @@ export default function Dashboard({ fileUpload, demo, logFormat }: { fileUpload:
         return url;
     }
 
-    useEffect(() => {
-        if (accessLogs.length <= parsedAccessCount.current) return;
-
-        const newRawLogs = accessLogs.slice(parsedAccessCount.current);
-        const isFirstBatch = parsedAccessCount.current === 0;
-        parsedAccessCount.current = accessLogs.length;
-
-        const initPeriod = (parsed: ReturnType<typeof parseNginxLogs>) => {
-            let maxDate = parsed[0].timestamp;
-            for (const log of parsed) {
-                if (log.timestamp && (!maxDate || log.timestamp > maxDate)) {
-                    maxDate = log.timestamp;
-                }
-            }
-            if (maxDate) {
-                if (inPeriod(maxDate, 'week')) setPeriod('week');
-                else if (inPeriod(maxDate, 'month')) setPeriod('month');
-                else if (inPeriod(maxDate, '6 months')) setPeriod('6 months');
-                else setPeriod('all time');
-            }
-        };
-
-        // Small batches: parse synchronously
-        if (newRawLogs.length <= PARSE_CHUNK_SIZE) {
-            const newParsed = parseNginxLogs(newRawLogs, logFormat);
-            if (newParsed.length === 0) return;
-            if (isFirstBatch) initPeriod(newParsed);
-            setLogs(prev => [...prev, ...newParsed]);
-            return;
-        }
-
-        // Large batches: chunk with setTimeout to avoid blocking the main thread,
-        // but accumulate all results and update state only once at the end.
-        parseCancelRef.current = false;
-        let offset = 0;
-        const allParsed: ReturnType<typeof parseNginxLogs> = [];
-        const processChunk = () => {
-            if (parseCancelRef.current) return;
-            const chunk = newRawLogs.slice(offset, offset + PARSE_CHUNK_SIZE);
-            if (chunk.length === 0) return;
-            const parsed = parseNginxLogs(chunk, logFormat);
-            if (parsed.length > 0) allParsed.push(...parsed);
-            offset += PARSE_CHUNK_SIZE;
-            if (offset < newRawLogs.length) {
-                setTimeout(processChunk, 0);
-            } else {
-                // All chunks done — single state update
-                if (allParsed.length > 0) {
-                    if (isFirstBatch) initPeriod(allParsed);
-                    setLogs(prev => [...prev, ...allParsed]);
-                }
-            }
-        };
-        processChunk();
-
-        return () => { parseCancelRef.current = true; };
-    }, [accessLogs])
 
 
 
diff --git a/dashboard/lib/components/location.tsx b/dashboard/lib/components/location.tsx
@@ -1,4 +1,4 @@
-import { Dispatch, SetStateAction, useEffect, useState } from "react";
+import { Dispatch, SetStateAction, useEffect, useRef, useState } from "react";
 import { type Location } from '@/lib/location'
 import { generateDemoLocations } from "../demo";
 
@@ -26,6 +26,7 @@ export function Location({
 }) {
     const [loading, setLoading] = useState(false);
     const [endpointDisabled, setEndpointDisabled] = useState(false);
+    const attemptedIPsRef = useRef(new Set<string>());
 
     const fetchLocations = async (ipAddresses: string[]) => {
         const response = await fetch('/api/location', {
@@ -77,15 +78,18 @@ export function Location({
 
     useEffect(() => {
         if (noFetch || endpointDisabled || unknownIPs.length === 0) return;
+        const toFetch = unknownIPs.filter(ip => !attemptedIPsRef.current.has(ip));
+        if (toFetch.length === 0) return;
+        toFetch.forEach(ip => attemptedIPsRef.current.add(ip));
 
         const fetchData = async () => {
             setLoading(true);
             try {
                 let fetchedLocations: Location[];
                 if (demo) {
-                    fetchedLocations = generateDemoLocations(unknownIPs);
+                    fetchedLocations = generateDemoLocations(toFetch);
                 } else {
-                    fetchedLocations = await fetchLocations(unknownIPs);
+                    fetchedLocations = await fetchLocations(toFetch);
                 }
                 if (fetchedLocations.length > 0) {
                     setLocationMap((prevMap) => {
diff --git a/dashboard/lib/parse.worker.ts b/dashboard/lib/parse.worker.ts
@@ -0,0 +1,34 @@
+/// <reference lib="webworker" />
+export type {};
+
+import { parseNginxLogs } from './parse';
+import type { NginxLog } from './types';
+
+type ParseMessage = {
+    rawLogs: string[];
+    logFormat?: string;
+    isFirstBatch: boolean;
+};
+
+export type ParseWorkerResult = {
+    logs: NginxLog[];
+    maxTimestamp: number | null;
+    isFirstBatch: boolean;
+};
+
+self.onmessage = (e: MessageEvent<ParseMessage>) => {
+    const { rawLogs, logFormat, isFirstBatch } = e.data;
+    const parsed = parseNginxLogs(rawLogs, logFormat);
+
+    let maxTimestamp: number | null = null;
+    if (isFirstBatch) {
+        for (const log of parsed) {
+            if (log.timestamp) {
+                const t = log.timestamp.getTime();
+                if (maxTimestamp === null || t > maxTimestamp) maxTimestamp = t;
+            }
+        }
+    }
+
+    self.postMessage({ logs: parsed, maxTimestamp, isFirstBatch } satisfies ParseWorkerResult);
+};
