Use cachedAsyncBuffer from hyparquet

Author: platypii

package.json

@@ -24,7 +24,7 @@
   "dependencies": {
     "highlight.js": "11.10.0",
     "hightable": "0.5.3",
-    "hyparquet": "1.4.1",
+    "hyparquet": "1.5.0",
     "hyparquet-compressors": "0.1.4"
   },
   "devDependencies": {

public/build/app.min.js.map

@@ -1,9 +1,10 @@
-import { asyncBufferFromUrl, parquetQuery } from 'hyparquet'
+import { parquetQuery } from 'hyparquet'
 import { compressors } from 'hyparquet-compressors'
+import { asyncBufferFrom } from './parquetWorkerClient.js'
 
 self.onmessage = async ({ data }) => {
   const { metadata, asyncBuffer, rowStart, rowEnd, orderBy } = data
-  const file = await asyncBufferFromUrl(asyncBuffer.url, asyncBuffer.byteLength)
+  const file = await asyncBufferFrom(asyncBuffer)
   try {
     const result = await parquetQuery({
       metadata, file, rowStart, rowEnd, orderBy, compressors,

public/build/worker.min.js

@@ -1,4 +1,4 @@
-import type { FileMetaData } from 'hyparquet'
+import { asyncBufferFromUrl, cachedAsyncBuffer, AsyncBuffer, FileMetaData } from 'hyparquet'
 
 // Serializable constructor for AsyncBuffers
 export interface AsyncBufferFrom {
@@ -45,3 +45,16 @@ export function parquetQueryWorker({
     worker.postMessage({ metadata, asyncBuffer, rowStart, rowEnd, orderBy })
   })
 }
+
+/**
+ * Convert AsyncBufferFrom to AsyncBuffer and cache results.
+ */
+export async function asyncBufferFrom(from: AsyncBufferFrom): Promise<AsyncBuffer> {
+  const key = JSON.stringify(from)
+  const cached = cache.get(key)
+  if (cached) return cached
+  const asyncBuffer = asyncBufferFromUrl(from.url, from.byteLength)
+  cache.set(key, asyncBuffer.then(cachedAsyncBuffer))
+  return asyncBuffer
+}
+const cache = new Map<string, Promise<AsyncBuffer>>()