diff --git a/src/lib/workers/parquetWorker.ts b/src/lib/workers/parquetWorker.ts index 6b654e1b..d31712ec 100644 --- a/src/lib/workers/parquetWorker.ts +++ b/src/lib/workers/parquetWorker.ts @@ -1,8 +1,23 @@ import type { ColumnData } from 'hyparquet' -import { parquetRead } from 'hyparquet' +import { AsyncBuffer, asyncBufferFromUrl, cachedAsyncBuffer, parquetRead } from 'hyparquet' import { compressors } from 'hyparquet-compressors' -import { asyncBufferFrom } from '../utils.js' -import type { ChunkMessage, ClientMessage, ErrorMessage, ResultMessage } from './types.js' +import type { AsyncBufferFrom, ChunkMessage, ClientMessage, ErrorMessage, ResultMessage } from './types.js' + +const cache = new Map>() + +export function asyncBufferFrom(from: AsyncBufferFrom): Promise { + if ('url' in from) { + // Cached asyncBuffer for urls only + const key = JSON.stringify(from) + const cached = cache.get(key) + if (cached) return cached + const asyncBuffer = asyncBufferFromUrl(from).then(cachedAsyncBuffer) + cache.set(key, asyncBuffer) + return asyncBuffer + } else { + return from.file.arrayBuffer() + } +} function postChunkMessage ({ chunk, queryId }: ChunkMessage) { self.postMessage({ chunk, queryId }) diff --git a/src/lib/workers/parquetWorkerClient.ts b/src/lib/workers/parquetWorkerClient.ts index f9702308..835a8a2c 100644 --- a/src/lib/workers/parquetWorkerClient.ts +++ b/src/lib/workers/parquetWorkerClient.ts @@ -43,7 +43,6 @@ function getWorker() { * to be serialized to the worker. */ export function parquetQueryWorker({ metadata, from, rowStart, rowEnd, columns, onChunk }: WorkerOptions): Promise { - // TODO(SL) Support passing columns? return new Promise((resolve, reject) => { const queryId = nextQueryId++ pending.set(queryId, { resolve, reject, onChunk }) diff --git a/src/lib/workers/types.ts b/src/lib/workers/types.ts index be883077..98dcec2c 100644 --- a/src/lib/workers/types.ts +++ b/src/lib/workers/types.ts @@ -30,6 +30,5 @@ export interface WorkerOptions { rowEnd?: number, columns?: string[], onChunk: (chunk: ColumnData) => void - // TODO(SL): support onPage too? } export type ClientMessage = Omit & ResultMessage diff --git a/vite.lib.config.ts b/vite.lib.config.ts index 7ad8065c..95d49148 100644 --- a/vite.lib.config.ts +++ b/vite.lib.config.ts @@ -9,6 +9,7 @@ const __dirname = dirname (fileURLToPath (import.meta.url )) export default defineConfig({ plugins: [react()], build: { + minify: true, outDir: resolve(__dirname, 'lib'), copyPublicDir: false, lib: {